Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

refactor(eve): integrate the shared agent tracing engine (#4164)

Chad Hietala76f68542dbfa
diff --git a/.changeset/clean-agent-tracing.md b/.changeset/clean-agent-tracing.md
new file mode 100644
index 000000000..a7a5ab6de
--- /dev/null
+++ b/.changeset/clean-agent-tracing.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+eve's agent spans now come from the shared tracing library, and durable turns recover from one checkpoint per turn. Each tool call is still one `execute_tool` span, a local subagent's first turn still nests under its caller, and trace schema version 4 is unchanged.
diff --git a/packages/eve/scripts/build-rolldown.mjs b/packages/eve/scripts/build-rolldown.mjs
index 875492253..bb7725480 100644
--- a/packages/eve/scripts/build-rolldown.mjs
+++ b/packages/eve/scripts/build-rolldown.mjs
@@ -162,7 +162,6 @@ const EXCLUDED_DIRECTORIES = new Set([join("internal", "testing")]);
*/
const EXTERNAL_PACKAGES = new Set([
"@nuxt/kit",
- "@opentelemetry/api",
"@sveltejs/kit",
"ai",
"braintrust",
@@ -259,6 +258,20 @@ function createVendoredConnectPlugin() {
};
}
+function createVendoredTelemetryPlugin() {
+ const packages = new Set([
+ "@opentelemetry/api",
+ "@opentelemetry/otlp-transformer",
+ "@vercel/otel",
+ ]);
+ return {
+ name: "eve:vendored-telemetry",
+ resolveId(source) {
+ return packages.has(source) ? { id: `#compiled/${source}/index.js`, external: true } : null;
+ },
+ };
+}
+
async function collectSourceFiles(directory, relativeRoot = "") {
const entries = await readdir(directory, { withFileTypes: true });
const sourceFiles = [];
@@ -315,6 +328,7 @@ await buildWithNitroRolldown({
plugins: [
createVendoredZodPlugin(),
createVendoredConnectPlugin(),
+ createVendoredTelemetryPlugin(),
createStripUnusedRolldownRuntimeImportPlugin(),
createDynamicToolTransformPlugin(),
createWorkflowMetadataTransformPlugin(),
diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
index e7e5aa0ac..aa62360c8 100644
--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts
@@ -10,7 +10,7 @@ import { formatElapsed } from "#cli/format-elapsed.js";
import { clipVisible, stripTerminalControls, visibleLength } from "#cli/ui/terminal-text.js";
import type { LocalTrace, LocalTraceSpan } from "#tracing/local/trace-reader.js";
import { compareLocalTraceSpans, isAgentTurnSpan } from "#tracing/local/trace-reader.js";
-import { agentTurnIdentity } from "#tracing/agent-span-contract.js";
+import { agentTurnIdentity } from "#tracing/local/inspection.js";
import { localTraceSpanCostUsd } from "#tracing/local/trace-summary.js";
import { traceStringAttribute } from "#tracing/local/trace-operations.js";
diff --git a/packages/eve/src/eve-channel/index.ts b/packages/eve/src/eve-channel/index.ts
index f4997fd74..180abea59 100644
--- a/packages/eve/src/eve-channel/index.ts
+++ b/packages/eve/src/eve-channel/index.ts
@@ -1,6 +1,7 @@
import { InvalidToolStubTargetError } from "#tool-stubs/validate-targets.js";
import { readStubFailure, readMatchedStubRules } from "#execution/tool-stubs/steps.js";
import { getRun } from "#internal/workflow/runtime.js";
+import { proxyStreamHeaders } from "#eve-channel/stream-headers.js";
import { handleExpiredLegacyAuthorization } from "#execution/legacy-session/authorization.js";
import { EVE_ROUTE_PREFIX } from "#protocol/routes.js";
import type { SessionAuthContext, SessionParent, SessionTraceContext } from "#channel/types.js";
@@ -18,13 +19,7 @@ import {
readRemoteAgentStreamHeadersResolver,
readRouteSessionCreator,
} from "#internal/nitro/routes/channel-route-context.js";
-import {
- EVE_SESSION_ID_HEADER,
- EVE_STREAM_CONTROL_VERSION_QUERY,
- EVE_STREAM_FORMAT_HEADER,
- EVE_STREAM_TAIL_INDEX_HEADER,
- EVE_STREAM_VERSION_HEADER,
-} from "#protocol/message.js";
+import { EVE_SESSION_ID_HEADER, EVE_STREAM_CONTROL_VERSION_QUERY } from "#protocol/message.js";
import { legacyTaskInputRoute } from "#execution/legacy-remote-agent/protocol.js";
import {
EVE_CALLBACK_ROUTE_PATTERN,
@@ -52,7 +47,7 @@ import {
readForwardedAudienceBaggage,
readForwardedParentSessionBaggage,
} from "#protocol/baggage.js";
-import { readConversationBaggage } from "#tracing/eve/conversation-context.js";
+import { agentDelegation } from "#tracing/eve/delegation.js";
import {
FAIL_CLOSED_FORWARDED_TRACE_ASSERTION,
formatTraceContentCeiling,
@@ -336,40 +331,49 @@ export function eveChannel(input: EveChannelInput): EveChannel {
let handle: Awaited<ReturnType<typeof createSession>>;
try {
- handle = await createSession({
- audienceAuth: auth,
- toolStubs:
- body.stubs === undefined
- ? undefined
- : {
- rules: body.stubs,
- token: crypto.randomUUID(),
+ handle = await agentDelegation.receive(
+ {
+ headers: req.headers,
+ delegated: body.callback !== undefined,
+ trusted: forwarded.accepted,
+ parent,
+ callerTrace: parsedParentTraceContext,
+ parentTraceContext,
+ },
+ ({ conversationId, parentTraceContext: inheritedTrace }) => {
+ return createSession({
+ audienceAuth: auth,
+ toolStubs:
+ body.stubs === undefined
+ ? undefined
+ : {
+ rules: body.stubs,
+ token: crypto.randomUUID(),
+ },
+ auth: messageResult.auth,
+ capabilities: body.capabilities ?? { requestInput: true },
+ callback: body.callback,
+ legacyRemoteAgentCaller: body.legacyRemoteAgentCaller,
+ continuationToken: operationToken,
+ initiatorAuth: forwarded.accepted ? forwarded.initiatorAuth : undefined,
+ input: attachClientContext(
+ {
+ message: body.message,
+ context: messageResult.context,
+ outputSchema: body.outputSchema,
},
- auth: messageResult.auth,
- capabilities: body.capabilities ?? { requestInput: true },
- callback: body.callback,
- legacyRemoteAgentCaller: body.legacyRemoteAgentCaller,
- continuationToken: operationToken,
- initiatorAuth: forwarded.accepted ? forwarded.initiatorAuth : undefined,
- input: attachClientContext(
- {
- message: body.message,
- context: messageResult.context,
- outputSchema: body.outputSchema,
- },
- body.context,
- ),
- conversationId:
- body.callback === undefined
- ? undefined
- : readConversationBaggage(req.headers.get("baggage")),
- parent,
- parentTraceContext,
- // A remote agent's spans form their own session in this
- // deployment; its lineage still names the caller's root.
- traceRoot: parent === undefined ? undefined : { kind: "own" },
- title: messageResult.title,
- });
+ body.context,
+ ),
+ conversationId,
+ parent,
+ parentTraceContext: inheritedTrace,
+ // A remote agent's spans form their own session in this
+ // deployment; its lineage still names the caller's root.
+ traceRoot: parent === undefined ? undefined : { kind: "own" },
+ title: messageResult.title,
+ });
+ },
+ );
} catch (error) {
if (error instanceof InvalidToolStubTargetError) {
return Response.json({ error: error.message, ok: false }, { status: 400 });
@@ -723,19 +727,7 @@ export function eveChannel(input: EveChannelInput): EveChannel {
redirect: "manual",
signal: req.signal,
});
- const responseHeaders = new Headers();
- for (const name of [
- "cache-control",
- "content-type",
- "x-accel-buffering",
- EVE_SESSION_ID_HEADER,
- EVE_STREAM_FORMAT_HEADER,
- EVE_STREAM_TAIL_INDEX_HEADER,
- EVE_STREAM_VERSION_HEADER,
- ]) {
- const value = upstream.headers.get(name);
- if (value !== null) responseHeaders.set(name, value);
- }
+ const responseHeaders = proxyStreamHeaders(upstream.headers);
return new Response(upstream.body, {
headers: responseHeaders,
status: upstream.status,
diff --git a/packages/eve/src/eve-channel/stream-headers.ts b/packages/eve/src/eve-channel/stream-headers.ts
new file mode 100644
index 000000000..a008211a2
--- /dev/null
+++ b/packages/eve/src/eve-channel/stream-headers.ts
@@ -0,0 +1,19 @@
+import {
+ EVE_SESSION_ID_HEADER,
+ EVE_STREAM_FORMAT_HEADER,
+ EVE_STREAM_TAIL_INDEX_HEADER,
+ EVE_STREAM_VERSION_HEADER,
+} from "#protocol/message.js";
+
+export function proxyStreamHeaders(headers: Headers): Headers {
+ const allowed = [
+ "cache-control",
+ "content-type",
+ "x-accel-buffering",
+ EVE_SESSION_ID_HEADER,
+ EVE_STREAM_FORMAT_HEADER,
+ EVE_STREAM_TAIL_INDEX_HEADER,
+ EVE_STREAM_VERSION_HEADER,
+ ];
+ return new Headers([...headers].filter(([name]) => allowed.includes(name)));
+}
diff --git a/packages/eve/src/execution/agent-sessions/remote.test.ts b/packages/eve/src/execution/agent-sessions/remote.test.ts
index 0b06c18d9..d8a882e2e 100644
--- a/packages/eve/src/execution/agent-sessions/remote.test.ts
+++ b/packages/eve/src/execution/agent-sessions/remote.test.ts
@@ -166,6 +166,64 @@ describe("resolveRemoteAgentStreamHeaders", () => {
});
describe("startRemoteAgentSession", () => {
+ it("uses the library transport handoff for a worker dispatch", async () => {
+ const { AsyncLocalStorageContextManager } = await import("@opentelemetry/context-async-hooks");
+ const { context } = await import("@opentelemetry/api");
+ const { agentDelegation } = await import("#tracing/eve/delegation.js");
+ const manager = new AsyncLocalStorageContextManager().enable();
+ context.setGlobalContextManager(manager);
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValue(
+ Response.json(
+ { ok: true, protocolVersion: 2, sessionId: "remote-session", status: "accepted" },
+ { status: 202 },
+ ),
+ );
+ vi.stubGlobal("fetch", fetchMock);
+ try {
+ await agentDelegation.resume(
+ {
+ context: {
+ trace: {
+ conversationId: "conversation",
+ originAudience: "public",
+ parentTraceContext: {
+ traceId: "a".repeat(32),
+ spanId: "b".repeat(16),
+ traceFlags: 1,
+ decision: { action: "record", recordInputs: false, recordOutputs: false },
+ },
+ },
+ parent: {
+ sessionId: "parent",
+ rootSessionId: "parent",
+ callId: "call",
+ turn: { id: "turn", sequence: 0 },
+ },
+ },
+ agentName: "research",
+ },
+ () =>
+ startRemoteAgentSession({
+ action: createAction(),
+ callbackBaseUrl: "https://caller.example.com",
+ remote: createRemoteAgent(),
+ session: { continuationToken: "eve:parent-token" },
+ }),
+ );
+ const options = fetchMock.mock.calls[0]?.[1];
+ expect(JSON.parse(new Headers(options.headers).get("x-agent-tracing")!)).toMatchObject({
+ conversationId: "conversation",
+ parentRunId: "parent",
+ parentCallId: "call",
+ });
+ expect(options.redirect).toBe("error");
+ } finally {
+ context.disable();
+ manager.disable();
+ }
+ });
afterEach(() => {
vi.unstubAllGlobals();
vi.unstubAllEnvs();
diff --git a/packages/eve/src/execution/agent-sessions/remote.ts b/packages/eve/src/execution/agent-sessions/remote.ts
index 3411a3fc1..51d2811e9 100644
--- a/packages/eve/src/execution/agent-sessions/remote.ts
+++ b/packages/eve/src/execution/agent-sessions/remote.ts
@@ -44,6 +44,7 @@ import {
} from "#protocol/baggage.js";
import { decisionToTraceContentCeiling } from "#shared/forwarded-trace-policy.js";
import { writeConversationBaggage } from "#tracing/eve/conversation-context.js";
+import { agentDelegation } from "#tracing/eve/delegation.js";
const CreateSessionResponseSchema = z.object({
ok: z.literal(true),
@@ -151,7 +152,7 @@ export async function startRemoteAgentSession(input: {
"baggage",
writeForwardedParentSessionBaggage(conversationBaggage, input.parent?.lineage),
);
- const response = await fetch(createRemoteAgentSessionUrl(input.remote), {
+ const response = await agentDelegation.fetch(createRemoteAgentSessionUrl(input.remote), {
body: JSON.stringify(requestBody),
headers: {
"content-type": "application/json",
diff --git a/packages/eve/src/execution/agent-sessions/steps.ts b/packages/eve/src/execution/agent-sessions/steps.ts
index f4e293dda..34db0b8e7 100644
--- a/packages/eve/src/execution/agent-sessions/steps.ts
+++ b/packages/eve/src/execution/agent-sessions/steps.ts
@@ -39,6 +39,7 @@ import {
import { buildSubagentRunInput } from "#subagents/tool.js";
import { resolveConversationId } from "#shared/conversation-identity.js";
import { stubToolPath } from "#tool-stubs/target.js";
+import { agentDelegation } from "#tracing/eve/delegation.js";
const log = createLogger("execution.agent-sessions");
@@ -115,10 +116,11 @@ export async function openAgentSessionStep(
isRootSession: context.parent.rootSessionId === context.parent.sessionId,
});
const start = { auth: input.auth, bundle, context, key: input.key, replyTo: input.replyTo };
- if (target.kind === "remote") {
- return await startRemoteSession({ ...start, target });
- }
- return await startLocalSession({ ...start, target });
+ const execute = () =>
+ target.kind === "remote"
+ ? startRemoteSession({ ...start, target })
+ : startLocalSession({ ...start, target });
+ return await agentDelegation.resume({ context, agentName: action.name }, execute);
}
/**
diff --git a/packages/eve/src/execution/cancelled-model-call-batch.ts b/packages/eve/src/execution/cancelled-model-call-batch.ts
index a3bb98960..2a099935d 100644
--- a/packages/eve/src/execution/cancelled-model-call-batch.ts
+++ b/packages/eve/src/execution/cancelled-model-call-batch.ts
@@ -11,7 +11,7 @@ import type { HarnessSession, StepInput, StepResult } from "#harness/types.js";
import { appliedSession, saveSessionProjection } from "#harness/session-machine/current.js";
import { takeDeferredMessage } from "#harness/hitl/index.js";
import { preserveSerializedInstrumentationState } from "#instrumentation/state.js";
-import { preserveSerializedAgentTraceState } from "#tracing/agent-trace-context-store.js";
+import { preserveSerializedAgentTraceState } from "#tracing/eve/agent-trace-context-store.js";
const log = createLogger("execution.cancelled-turn");
diff --git a/packages/eve/src/execution/eve-workflow-attributes.ts b/packages/eve/src/execution/eve-workflow-attributes.ts
index b29622e02..f00a82c3a 100644
--- a/packages/eve/src/execution/eve-workflow-attributes.ts
+++ b/packages/eve/src/execution/eve-workflow-attributes.ts
@@ -53,7 +53,7 @@ import {
ConversationContextKey,
normalizeConversationContext,
} from "#shared/conversation-context.js";
-import { isSampledTrace } from "#tracing/sampled-trace.js";
+import { isSampledTrace } from "#shared/trace-policy.js";
import { resolveForwardedTraceSeed } from "#shared/forwarded-trace-policy.js";
/** eve version recorded on every run eve starts; handoff successors and legacy importers re-record it at boot. */
diff --git a/packages/eve/src/execution/node-step.test.ts b/packages/eve/src/execution/node-step.test.ts
index 4336b5d77..eea51697e 100644
--- a/packages/eve/src/execution/node-step.test.ts
+++ b/packages/eve/src/execution/node-step.test.ts
@@ -1,3 +1,4 @@
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
import { ToolLoopAgent } from "ai";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { Runtime } from "#channel/types.js";
@@ -330,6 +331,7 @@ describe("createExecutionNodeStep", () => {
setupMockAgentForToolExecution("regular-tool", { question: "Run the tool." });
const forceFlush = vi.fn(async () => undefined);
const runtime: InstrumentationRuntime = {
+ idGenerator: new AgentSpanIdGenerator(),
forceFlush,
hooks: createInstrumentationHooks([]),
otelSettings: undefined,
diff --git a/packages/eve/src/execution/session/turn-step.test.ts b/packages/eve/src/execution/session/turn-step.test.ts
index 666f1329f..d76c79c2d 100644
--- a/packages/eve/src/execution/session/turn-step.test.ts
+++ b/packages/eve/src/execution/session/turn-step.test.ts
@@ -1,3 +1,4 @@
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
import { createTestSessionState } from "#internal/testing/session-state.js";
import { jsonSchema, type ModelMessage } from "ai";
import { MockLanguageModelV4 } from "ai/test";
@@ -2229,6 +2230,7 @@ describe("turnStep", () => {
});
instrumentationRuntimeOverride.current = {
forceFlush: async () => undefined,
+ idGenerator: new AgentSpanIdGenerator(),
hooks: {
capturesContent: false,
publish: async (event) => {
diff --git a/packages/eve/src/execution/session/turn-step.ts b/packages/eve/src/execution/session/turn-step.ts
index 111ade392..8d5e32b9d 100644
--- a/packages/eve/src/execution/session/turn-step.ts
+++ b/packages/eve/src/execution/session/turn-step.ts
@@ -32,7 +32,7 @@ import { BundleKey, ChannelKey } from "#runtime/sessions/runtime-context-keys.js
import { deserializeContext, serializeContext } from "#context/serialize.js";
import { bindSessionInstrumentation } from "#instrumentation/runtime.js";
import { RuntimeActionSettlementTimesKey } from "#harness/runtime-action-settlement-state.js";
-import * as agentTraceState from "#tracing/agent-trace-context-store.js";
+import * as agentTraceState from "#tracing/eve/agent-trace-context-store.js";
import { matchAuthorizationCallbacks } from "#execution/authorization-callback-match.js";
import { isTurnCancellation, throwIfTurnAborted } from "#harness/turn-cancellation.js";
import { setChannelContext } from "#execution/channel-context.js";
diff --git a/packages/eve/src/execution/workflow-runtime.test.ts b/packages/eve/src/execution/workflow-runtime.test.ts
index 0d50b6d30..b1a03e2e5 100644
--- a/packages/eve/src/execution/workflow-runtime.test.ts
+++ b/packages/eve/src/execution/workflow-runtime.test.ts
@@ -27,7 +27,7 @@ import {
sessionHandoffMarkerToken,
} from "#execution/session-inbox/address.js";
import { registerInstrumentationRuntime } from "#instrumentation/runtime.js";
-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
import type { RuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js";
import { getCompiledRuntimeAgentBundle } from "#runtime/sessions/compiled-agent-cache.js";
import { markAgentTraceContext } from "#tracing/eve/agent-trace-context.js";
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index 740899ce4..43fdc92eb 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -1,3 +1,4 @@
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
import { context as otelContext, trace } from "#compiled/@opentelemetry/api/index.js";
import {
type FilePart,
@@ -195,6 +196,7 @@ function declareTelemetry(
? undefined
: {
forceFlush: async () => undefined,
+ idGenerator: new AgentSpanIdGenerator(),
hooks: createInstrumentationHooks([]),
otelSettings: {
...config,
@@ -226,6 +228,7 @@ function bindHookInstrumentation(
return bindInstrumentationRuntime(
{
forceFlush: async () => undefined,
+ idGenerator: new AgentSpanIdGenerator(),
hooks,
otelSettings: useDeclaredRuntime ? declaredRuntime?.otelSettings : undefined,
runtimeContextResolvers: useDeclaredRuntime
@@ -10176,6 +10179,7 @@ describe("createToolLoopHarness", () => {
};
const authoredSpan = trace.wrapSpanContext(authoredTrace);
const getTracerSpy = vi.spyOn(trace, "getTracer").mockReturnValue({
+ startActiveSpan: vi.fn(),
startSpan: vi.fn(() => authoredSpan),
} as ReturnType<typeof trace.getTracer>);
setupMockAgent({
diff --git a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts
index 80c8006f9..9c93920f0 100644
--- a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts
+++ b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts
@@ -3,14 +3,12 @@ import type { Telemetry } from "ai";
import type {
InstrumentationAttemptScope,
InstrumentationStepAttemptStartedEvent,
- InstrumentationContentPart,
InstrumentationContextRunner,
InstrumentationHooks,
InstrumentationModelCallCompletedEvent,
InstrumentationModelCallStartedEvent,
InstrumentationOperationRef,
InstrumentationToolCallStartedEvent,
- InstrumentationUsage,
} from "#instrumentation/lifecycle.js";
import {
attemptIdempotencyKey,
@@ -18,6 +16,7 @@ import {
toolCallIdempotencyKey,
} from "#instrumentation/lifecycle.js";
import { structuralProviderMetadata } from "#instrumentation/content.js";
+import { modelContent, modelUsage } from "#compiled/@eve/tracing/ai-sdk.js";
type TelemetryEvent<TKey extends keyof Telemetry> = Parameters<NonNullable<Telemetry[TKey]>>[0];
@@ -224,78 +223,19 @@ function toModelCallCompleted(
source: TelemetryEvent<"onLanguageModelCallEnd">,
): InstrumentationModelCallCompletedEvent {
return Object.freeze({
- content: state.capturesOutputs ? toContentParts(source.content) : undefined,
+ content: state.capturesOutputs
+ ? Object.freeze(modelContent(source.content).map((part) => Object.freeze(part)))
+ : undefined,
finishReason: source.finishReason,
idempotencyKey,
responseModelId: source.modelId,
responseId: source.responseId,
scope: state.scope,
type: "model.call.completed",
- usage: toUsage(source.usage),
- });
-}
-
-function toUsage(usage: TelemetryEvent<"onLanguageModelCallEnd">["usage"]): InstrumentationUsage {
- return Object.freeze({
- inputTokenDetails: Object.freeze({
- cacheReadTokens: usage.inputTokenDetails?.cacheReadTokens,
- cacheWriteTokens: usage.inputTokenDetails?.cacheWriteTokens,
- }),
- inputTokens: usage.inputTokens,
- outputTokens: usage.outputTokens,
+ usage: Object.freeze(modelUsage(source.usage)),
});
}
-/** Drops kinds eve does not record; see {@link InstrumentationContentPart}. */
-function toContentParts(
- content: TelemetryEvent<"onLanguageModelCallEnd">["content"],
-): readonly InstrumentationContentPart[] {
- const parts: InstrumentationContentPart[] = [];
- for (const part of content) {
- switch (part.type) {
- case "text":
- case "reasoning":
- parts.push(Object.freeze({ text: part.text, type: part.type }));
- break;
- case "tool-call":
- parts.push(
- Object.freeze({
- callId: part.toolCallId,
- input: part.input,
- toolName: part.toolName,
- type: "tool-call",
- }),
- );
- break;
- case "tool-result":
- parts.push(
- Object.freeze({
- callId: part.toolCallId,
- input: part.input,
- output: part.output,
- toolName: part.toolName,
- type: "tool-result",
- }),
- );
- break;
- case "tool-error":
- parts.push(
- Object.freeze({
- callId: part.toolCallId,
- error: part.error,
- input: part.input,
- toolName: part.toolName,
- type: "tool-error",
- }),
- );
- break;
- default:
- break;
- }
- }
- return Object.freeze(parts);
-}
-
function toToolCallStarted(
state: AttemptState,
idempotencyKey: string,
diff --git a/packages/eve/src/instrumentation/channel-delivery.test.ts b/packages/eve/src/instrumentation/channel-delivery.test.ts
index e4ae24eb5..77514f8d2 100644
--- a/packages/eve/src/instrumentation/channel-delivery.test.ts
+++ b/packages/eve/src/instrumentation/channel-delivery.test.ts
@@ -1,3 +1,4 @@
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
import { describe, expect, it, vi } from "vitest";
import { ContextContainer, contextStorage } from "#context/container.js";
@@ -34,6 +35,7 @@ function bindHooks(
return bindInstrumentationRuntime(
{
forceFlush: async () => undefined,
+ idGenerator: new AgentSpanIdGenerator(),
hooks,
otelSettings: undefined,
runInContext: (_operation, execute) => execute(),
diff --git a/packages/eve/src/instrumentation/memory.ts b/packages/eve/src/instrumentation/memory.ts
index e1f56883f..f6aed1333 100644
--- a/packages/eve/src/instrumentation/memory.ts
+++ b/packages/eve/src/instrumentation/memory.ts
@@ -59,10 +59,7 @@ export type InstrumentationMemoryOperationTerminalEvent =
| InstrumentationMemoryOperationCompletedEvent
| InstrumentationMemoryOperationFailedEvent;
-export interface InstrumentationMemoryExecutionOperation {
- readonly idempotencyKey: string;
- readonly sessionId: string;
- readonly turnId?: string;
+export interface InstrumentationMemoryExecutionOperation extends InstrumentationMemoryOperationEvent {
readonly type: "memory.operation";
}
@@ -98,9 +95,7 @@ export function createMemoryInstrumentation(input: {
try {
const result = await input.runInContext(
{
- idempotencyKey: operation.idempotencyKey,
- sessionId: input.sessionId,
- turnId: operation.turnId,
+ ...event,
type: "memory.operation",
},
execute,
diff --git a/packages/eve/src/instrumentation/principal-summary.ts b/packages/eve/src/instrumentation/principal-summary.ts
index 6cba63e4e..e04dd2cc6 100644
--- a/packages/eve/src/instrumentation/principal-summary.ts
+++ b/packages/eve/src/instrumentation/principal-summary.ts
@@ -11,7 +11,7 @@ import {
} from "#shared/instrumentation-content.js";
import type { InstrumentationDecision } from "#shared/instrumentation-decision.js";
import { resolveTracePolicyDecision } from "#shared/trace-policy.js";
-import { boundedPrincipalId } from "#tracing/telemetry-budget.js";
+import { boundedPrincipalId } from "#compiled/@eve/tracing/otel.js";
export function summarizeInstrumentationPrincipal(
principal: SessionAuthContext | null | undefined,
diff --git a/packages/eve/src/instrumentation/providers-otel-policy.test.ts b/packages/eve/src/instrumentation/providers-otel-policy.test.ts
index aff40d504..1f934121e 100644
--- a/packages/eve/src/instrumentation/providers-otel-policy.test.ts
+++ b/packages/eve/src/instrumentation/providers-otel-policy.test.ts
@@ -8,20 +8,16 @@ import {
import { defineInstrumentation } from "#public/instrumentation/index.js";
import { otel, type TracePolicyDecision } from "#public/instrumentation/otel.js";
+const REGISTRY_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers");
vi.mock("#tracing/eve/otel-registration.js", () => ({
- registerOtelPipeline: () => ({
- forceFlush: async () => undefined,
- idGenerator: {
- deriveSpanId: () => "2".repeat(16),
- generateTraceId: () => "1".repeat(32),
- withSpanId: (_spanId: string, run: () => unknown) => run(),
- withTraceId: (_traceId: string, run: () => unknown) => run(),
- },
- shutdown: async () => undefined,
+ registerOtel: () => ({
+ provider: { getTracer: () => ({}) },
+ idGenerator: { deriveSpanId: () => "2".repeat(16), generateTraceId: () => "1".repeat(32) },
+ samplesTrace: () => true,
+ forceFlush: async () => {},
+ shutdown: async () => {},
}),
}));
-
-const REGISTRY_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers");
const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime");
const traceContext = (audience: "public" | "private" | "unknown") => ({
diff --git a/packages/eve/src/instrumentation/runtime.test.ts b/packages/eve/src/instrumentation/runtime.test.ts
index 795ca3b01..d28134475 100644
--- a/packages/eve/src/instrumentation/runtime.test.ts
+++ b/packages/eve/src/instrumentation/runtime.test.ts
@@ -26,8 +26,8 @@ import {
type InstrumentationRuntime,
type InstrumentationStepScope,
} from "#instrumentation/runtime.js";
-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
-import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
+import { ContextAgentTraceStateStore } from "#tracing/eve/agent-trace-context-store.js";
import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";
import { readForwardedAudienceBaggage, writeForwardedAudienceBaggage } from "#protocol/baggage.js";
import { ConversationContextKey } from "#shared/conversation-context.js";
@@ -45,6 +45,7 @@ function createRuntime(
): InstrumentationRuntime {
return {
forceFlush: async () => undefined,
+ idGenerator: new AgentSpanIdGenerator(),
hooks,
otelSettings: {
recordInputs: true,
@@ -354,6 +355,7 @@ describe("initializeSessionInstrumentation", () => {
});
it("does not widen an unsampled parent with a record decision", () => {
+ registerInstrumentationRuntime(createRuntime({ capturesContent: true, publish: vi.fn() }));
const ctx = createContext("public");
ctx.set(ParentTraceContextKey, {
decision: { action: "record", recordInputs: true, recordOutputs: true },
@@ -999,7 +1001,7 @@ describe("bindSessionInstrumentation", () => {
);
const ctx = createContext();
contextStorage.run(ctx, () => {
- new ContextAgentTraceStateStore().setSession("session-1", {
+ new ContextAgentTraceStateStore().set("session", "session-1", {
context: { spanId: "1".repeat(16), traceFlags: 1, traceId: "2".repeat(32) },
decision: { action: "record", recordInputs: false, recordOutputs: true },
rootSessionId: "session-1",
diff --git a/packages/eve/src/instrumentation/runtime.ts b/packages/eve/src/instrumentation/runtime.ts
index aa337665c..03b9939c9 100644
--- a/packages/eve/src/instrumentation/runtime.ts
+++ b/packages/eve/src/instrumentation/runtime.ts
@@ -43,7 +43,7 @@ import {
} from "#instrumentation/prepare-trace-context.js";
import type { RuntimeTraceContext } from "#protocol/message.js";
import type { OtelHarnessSettings, RuntimeContextResolver } from "#tracing/eve/otel-declaration.js";
-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
import { contextStorage, type ContextContainer } from "#context/container.js";
import {
createMemoryInstrumentation,
@@ -51,15 +51,15 @@ import {
} from "#instrumentation/memory.js";
import { ConversationIdKey, ParentSessionKey } from "#context/keys.js";
import type { ConversationContext } from "#shared/conversation-context.js";
-import { withErrorContent } from "#tracing/error-content-context.js";
-import type { AgentSamplingOperation } from "#tracing/agent-span-contract.js";
+import { withCapture } from "#compiled/@eve/tracing/otel.js";
+import type { SamplingOperation as AgentSamplingOperation } from "#tracing/eve/otel-registration.js";
import {
isSampledTrace,
resolveTracePolicy,
resolveTracePolicyDecision,
-} from "#tracing/sampled-trace.js";
+} from "#shared/trace-policy.js";
import { readInstrumentationSessionContext } from "#instrumentation/session-context.js";
-import { readSessionTraceDecision } from "#tracing/agent-trace-context-store.js";
+import { readSessionTraceDecision } from "#tracing/eve/agent-trace-context-store.js";
import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";
import { applyLiveDeliveryAudienceCeiling } from "#shared/forwarded-trace-policy.js";
import { readConversationId } from "#shared/conversation-identity.js";
@@ -498,7 +498,11 @@ export function bindInstrumentationRuntime(
});
try {
return await otelContext.with(
- withErrorContent(parentContext ?? otelContext.active(), content.recordOutputs),
+ withCapture(parentContext ?? otelContext.active(), {
+ emit: true,
+ recordInputs: content.recordInputs,
+ recordOutputs: content.recordOutputs,
+ }),
run,
);
} finally {
diff --git a/packages/eve/src/instrumentation/session-init.test.ts b/packages/eve/src/instrumentation/session-init.test.ts
index b4a51c89c..1c378350c 100644
--- a/packages/eve/src/instrumentation/session-init.test.ts
+++ b/packages/eve/src/instrumentation/session-init.test.ts
@@ -10,7 +10,7 @@ import {
import { initializeSessionInstrumentation } from "#instrumentation/session-init.js";
import { registerInstrumentationRuntime } from "#instrumentation/runtime-global.js";
import type { InstrumentationRuntime } from "#instrumentation/runtime.js";
-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";
afterEach(() => {
diff --git a/packages/eve/src/instrumentation/session-init.ts b/packages/eve/src/instrumentation/session-init.ts
index 6ec447ba7..ce2500a2f 100644
--- a/packages/eve/src/instrumentation/session-init.ts
+++ b/packages/eve/src/instrumentation/session-init.ts
@@ -29,8 +29,8 @@ import {
isSampledTrace,
resolveTracePolicy,
resolveTracePolicyDecision,
-} from "#tracing/sampled-trace.js";
-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
+} from "#shared/trace-policy.js";
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
import { getInstrumentationRuntime } from "#instrumentation/runtime-global.js";
import type { InstrumentationRuntime } from "#instrumentation/runtime.js";
diff --git a/packages/eve/src/internal/logging.test.ts b/packages/eve/src/internal/logging.test.ts
index d6298b308..e0d373f2f 100644
--- a/packages/eve/src/internal/logging.test.ts
+++ b/packages/eve/src/internal/logging.test.ts
@@ -6,7 +6,7 @@ import {
trace,
type Span,
} from "#compiled/@opentelemetry/api/index.js";
-import { withErrorContent } from "#tracing/error-content-context.js";
+import { withErrorContent } from "#compiled/@eve/tracing/otel.js";
import {
createErrorId,
@@ -36,6 +36,11 @@ describe("span error content policy", () => {
function mockSpan() {
return {
+ setAttributes: vi.fn(),
+ addLink: vi.fn(),
+ addLinks: vi.fn(),
+ updateName: vi.fn(),
+ isRecording: vi.fn(() => true),
addEvent: vi.fn(),
end: vi.fn(),
recordException: vi.fn(),
diff --git a/packages/eve/src/internal/logging.ts b/packages/eve/src/internal/logging.ts
index 2f8dc4fd9..1ef855487 100644
--- a/packages/eve/src/internal/logging.ts
+++ b/packages/eve/src/internal/logging.ts
@@ -10,8 +10,8 @@ import { inspect } from "node:util";
import { isNonEmptyString, isObject } from "#shared/guards.js";
import type { JsonObject, JsonValue } from "#shared/json.js";
-import { capturesErrorContent } from "#tracing/error-content-context.js";
-import { truncateTelemetryText } from "#tracing/telemetry-budget.js";
+import { currentCapture } from "#compiled/@eve/tracing/index.js";
+import { truncateTelemetryText } from "#compiled/@eve/tracing/otel.js";
const MAX_INSPECT_STRING_LENGTH = 8 * 1024;
const MAX_DETAIL_BYTES = 16 * 1024;
@@ -230,7 +230,7 @@ function truncateForDisplay(value: string, maxChars = 160): string {
* non-active span should be annotated.
*/
export function recordErrorOnSpan(span: Span, error: unknown): void {
- if (!capturesErrorContent()) {
+ if (currentCapture()?.recordOutputs === false) {
span.setStatus({ code: SpanStatusCode.ERROR });
return;
}
@@ -365,7 +365,7 @@ function recordOnActiveSpan(message: string, fields?: LogFields): void {
}
const error = fields?.error;
- if (!capturesErrorContent()) {
+ if (currentCapture()?.recordOutputs === false) {
if (error instanceof Error || isFormattedError(error)) {
span.setStatus({ code: SpanStatusCode.ERROR });
}
diff --git a/packages/eve/src/internal/nitro/routes/channel-dispatch.test.ts b/packages/eve/src/internal/nitro/routes/channel-dispatch.test.ts
index 2d55abd30..b8d783465 100644
--- a/packages/eve/src/internal/nitro/routes/channel-dispatch.test.ts
+++ b/packages/eve/src/internal/nitro/routes/channel-dispatch.test.ts
@@ -1,3 +1,4 @@
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
import type { H3Event } from "nitro";
import {
context as apiContext,
@@ -94,6 +95,7 @@ function createDeferred<T>() {
function registerChannelRequestTracing(enabled: boolean): void {
registerInstrumentationRuntime({
forceFlush: async () => undefined,
+ idGenerator: new AgentSpanIdGenerator(),
hooks: createInstrumentationHooks([]),
otelSettings: {
functionId: undefined,
diff --git a/packages/eve/src/internal/nitro/routes/channel-dispatch.ts b/packages/eve/src/internal/nitro/routes/channel-dispatch.ts
index 959040936..cef93717d 100644
--- a/packages/eve/src/internal/nitro/routes/channel-dispatch.ts
+++ b/packages/eve/src/internal/nitro/routes/channel-dispatch.ts
@@ -65,7 +65,7 @@ export async function dispatchChannelRequest(
// Correlation does not require an eve-owned request span. Preserve any
// active platform request or function span before route resolution.
const requestTraceContext =
- span?.spanContext() ?? otelTrace.getSpan(otelContext.active())?.spanContext();
+ span?.reference ?? otelTrace.getSpan(otelContext.active())?.spanContext();
const bundle = await resolveNitroChannelRuntimeBundle(config);
const matchedChannel = bundle.channels.find(
@@ -85,11 +85,11 @@ export async function dispatchChannelRequest(
// (`channel:<name>`) over the raw adapter kind — behaviorless authored
// channels keep adapter kind `"http"`, so the adapter alone would report
// `"http"` where the rest of the trace reports `channel:<name>`.
- span?.setAttribute("eve.channel.name", matchedChannel.name);
+ span?.channel({ channelName: matchedChannel.name });
const channelKind =
getChannelInstrumentationKind(matchedChannel.definition) ?? matchedChannel.adapter?.kind;
if (channelKind !== undefined) {
- span?.setAttribute("eve.channel.kind", channelKind);
+ span?.channel({ channelKind });
}
const routeArgs = await buildRouteArgs(
diff --git a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts
index 12bd57bc3..6a2909d88 100644
--- a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts
+++ b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts
@@ -1,102 +1,71 @@
import {
- type Attributes,
context,
propagation,
- type Span,
+ trace,
SpanKind,
SpanStatusCode,
type TextMapGetter,
- trace,
} from "#compiled/@opentelemetry/api/index.js";
import { getInstrumentationRuntime } from "#instrumentation/runtime.js";
+import { withErrorContent } from "#compiled/@eve/tracing/otel.js";
import { markAgentTraceContext } from "#tracing/eve/agent-trace-context.js";
-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";
-import { AGENT_SPAN_NAMES } from "#tracing/agent-span-contract.js";
-import { withErrorContent } from "#tracing/error-content-context.js";
-/**
- * Stable tracer name for every inbound eve channel HTTP request. Kept
- * low-cardinality and independent of the agent so dashboards can group
- * request spans across deployments.
- */
-const TRACER_NAME = "eve.channel";
-
-/**
- * Reads OTel context from an inbound request's `Headers`. Header lookups
- * are case-insensitive, so this is a thin adapter over `Headers.get`.
- */
+export interface ChannelRequestTrace {
+ readonly reference: import("@opentelemetry/api").SpanContext;
+ channel(input: { channelName?: string; channelKind?: string }): void;
+}
const headersGetter: TextMapGetter<Headers> = {
- get(carrier, key) {
- return carrier.get(key) ?? undefined;
- },
- keys(carrier) {
- return [...carrier.keys()];
- },
+ get: (headers, key) => headers.get(key) ?? undefined,
+ keys: (headers) => [...headers.keys()],
};
-/** Inputs for {@link traceChannelRequest}. */
-interface TraceChannelRequestInput {
- readonly request: Request;
- readonly routeKey: string;
-}
-
-/**
- * Wraps one inbound channel HTTP request in an OTel `SERVER` span.
- *
- * The span is named for the low-cardinality registered `routeKey`
- * (`"POST /eve/v1/session/:sessionId"`), never the concrete URL, while the
- * `http.route` attribute carries only the path template and
- * `http.request.method` carries the method. It starts from context extracted
- * off the incoming request headers so an upstream `traceparent` becomes its
- * parent and nested channel operations become its descendants.
- *
- * The handler runs inside the span's active context. When it returns, the
- * response status is recorded and a `>= 500` status marks the span as an
- * error. A thrown handler marks failure status and is rethrown; request
- * spans never record exception content. The span always ends in `finally`, without
- * waiting for `event.waitUntil()` work or streamed response bodies.
- *
- * Emitting these spans is opt-in: unless authored instrumentation enables it
- * via `traceChannelRequests: true`, the handler runs with no span (`undefined`)
- * and performs no header extraction. The channel dispatcher can still use an
- * already-active platform span for the activation's cross-trace link.
- *
- * This is observability-only: it never changes the response and performs no
- * synchronous span export in the request path, adding only minimal in-process
- * tracing overhead.
- */
+/** The request lifetime ends at handler return, not response-body consumption. */
export async function traceChannelRequest<T extends Response>(
- input: TraceChannelRequestInput,
- handler: (span: Span | undefined) => Promise<T>,
+ input: { readonly request: Request; readonly routeKey: string },
+ handler: (operation: ChannelRequestTrace | undefined) => Promise<T>,
): Promise<T> {
- if (getInstrumentationRuntime()?.otelSettings?.traceChannelRequests !== true) {
- return await handler(undefined);
- }
-
+ if (getInstrumentationRuntime()?.otelSettings?.traceChannelRequests !== true)
+ return handler(undefined);
const { request, routeKey } = input;
- const parentContext = propagation.extract(context.active(), request.headers, headersGetter);
- const spanName = AGENT_SPAN_NAMES.channelRequest;
- const span = trace.getTracer(TRACER_NAME).startSpan(
- spanName,
+ const parent = propagation.extract(context.active(), request.headers, headersGetter);
+ const separator = routeKey.indexOf(" ");
+ let url: URL | undefined;
+ try {
+ url = new URL(request.url);
+ } catch {}
+ const span = trace.getTracer("eve.channel").startSpan(
+ "agent.channel.request",
{
+ kind: SpanKind.SERVER,
attributes: {
- ...baseAttributes(request, routeKey),
- ...(spanName === "agent.channel.request" ? agentSpanNamingAttributes(spanName) : undefined),
+ "operation.name": "agent.channel.request",
+ "resource.name": "agent.channel.request",
+ "http.request.method": request.method,
+ "http.route": separator === -1 ? routeKey : routeKey.slice(separator + 1),
+ "url.scheme": url?.protocol.replace(/:$/, ""),
+ "server.address": url?.hostname,
},
- kind: SpanKind.SERVER,
},
- parentContext,
+ parent,
);
- const activeContext = markAgentTraceContext(
- withErrorContent(trace.setSpan(parentContext, span), false),
- );
-
+ const operation: ChannelRequestTrace = {
+ reference: span.spanContext(),
+ channel(input) {
+ if (input.channelName !== undefined)
+ span.setAttribute("agent.channel.name", input.channelName);
+ if (input.channelName !== undefined) span.setAttribute("eve.channel.name", input.channelName);
+ if (input.channelKind !== undefined)
+ span.setAttribute("agent.channel.kind", input.channelKind);
+ if (input.channelKind !== undefined) span.setAttribute("eve.channel.kind", input.channelKind);
+ },
+ };
try {
- const response = await context.with(activeContext, () => handler(span));
+ const response = await context.with(
+ markAgentTraceContext(withErrorContent(trace.setSpan(parent, span), false)),
+ () => handler(operation),
+ );
span.setAttribute("http.response.status_code", response.status);
- if (response.status >= 500) {
- span.setStatus({ code: SpanStatusCode.ERROR });
- }
+ if (response.status >= 500) span.setStatus({ code: SpanStatusCode.ERROR });
return response;
} catch (error) {
span.setStatus({ code: SpanStatusCode.ERROR });
@@ -105,43 +74,3 @@ export async function traceChannelRequest<T extends Response>(
span.end();
}
}
-
-/**
- * Standard, non-sensitive request attributes known before the channel is
- * resolved. `eve.channel.name` / `eve.channel.kind` are attached later by
- * the caller once the matching channel is known. Never records session
- * ids, tokens, auth or cookie headers, bodies, or query parameters.
- */
-function baseAttributes(request: Request, routeKey: string): Attributes {
- const attributes: Attributes = {
- "http.request.method": request.method,
- "http.route": routeTemplate(routeKey),
- };
-
- const url = parseRequestUrl(request.url);
- if (url !== undefined) {
- attributes["url.scheme"] = url.protocol.replace(/:$/, "");
- attributes["server.address"] = url.hostname;
- }
-
- return attributes;
-}
-
-/**
- * Extracts the path template from a `"METHOD /path/:param"` route key. The
- * OTel HTTP convention keeps `http.route` free of the method so route
- * facets align with other server spans; the method lives in
- * `http.request.method` and the full key remains the span name.
- */
-function routeTemplate(routeKey: string): string {
- const separator = routeKey.indexOf(" ");
- return separator === -1 ? routeKey : routeKey.slice(separator + 1);
-}
-
-function parseRequestUrl(rawUrl: string): URL | undefined {
- try {
- return new URL(rawUrl);
- } catch {
- return undefined;
- }
-}
diff --git a/packages/eve/src/internal/testing/in-memory-agent-trace-state-store.ts b/packages/eve/src/internal/testing/in-memory-agent-trace-state-store.ts
index a259acc9f..a5651baf0 100644
--- a/packages/eve/src/internal/testing/in-memory-agent-trace-state-store.ts
+++ b/packages/eve/src/internal/testing/in-memory-agent-trace-state-store.ts
@@ -1,98 +1,30 @@
-import type {
- AgentActionTraceState,
- AgentSessionTraceState,
- AgentTraceStateStore,
- AgentTurnTraceState,
-} from "#tracing/agent-trace-state.js";
+import type { AgentTraceStateStore, AgentTraceValues } from "#tracing/eve/agent-trace-state.js";
export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {
- readonly #actionAnchors = new Map<string, AgentActionTraceState>();
- readonly #actions = new Map<string, AgentActionTraceState>();
- readonly #sessions = new Map<string, AgentSessionTraceState>();
- readonly #turns = new Map<string, AgentTurnTraceState>();
-
- deleteAction(idempotencyKey: string): void {
- this.#actions.delete(idempotencyKey);
- }
-
- deleteActionAnchors(sessionId: string): void {
- for (const [key, state] of this.#actionAnchors) {
- if (state.sessionId === sessionId) this.#actionAnchors.delete(key);
- }
- }
-
- deleteActions(sessionId: string, turnId?: string): void {
- for (const [key, state] of this.#actions) {
- if (state.sessionId === sessionId && (turnId === undefined || state.turnId === turnId)) {
- this.#actions.delete(key);
- }
- }
- }
-
- deleteSession(sessionId: string): void {
- this.#sessions.delete(sessionId);
+ readonly #entries = new Map<string, AgentTraceValues[keyof AgentTraceValues]>();
+ get<K extends keyof AgentTraceValues>(kind: K, key: string): AgentTraceValues[K] | undefined {
+ return this.#entries.get(JSON.stringify([kind, key])) as AgentTraceValues[K] | undefined;
}
-
- deleteTurn(sessionId: string, turnId: string): void {
- this.#turns.delete(turnKey(sessionId, turnId));
+ set<K extends keyof AgentTraceValues>(kind: K, key: string, value: AgentTraceValues[K]): void {
+ this.#entries.set(JSON.stringify([kind, key]), value);
}
-
- findAction(sessionId: string, callId: string): AgentActionTraceState | undefined {
- return [...this.#actions.values()].find(
- (state) => state.sessionId === sessionId && state.callId === callId,
- );
- }
-
- findActionAnchor(
- sessionId: string,
- turnId: string,
- callId: string,
- ): AgentActionTraceState | undefined {
- return [...this.#actionAnchors.values()].find(
- (state) =>
- state.sessionId === sessionId && state.turnId === turnId && state.callId === callId,
- );
- }
-
- getAction(idempotencyKey: string): AgentActionTraceState | undefined {
- return this.#actions.get(idempotencyKey);
- }
-
- getSession(sessionId: string): AgentSessionTraceState | undefined {
- return this.#sessions.get(sessionId);
+ delete(kind: keyof AgentTraceValues, key: string): void {
+ this.#entries.delete(JSON.stringify([kind, key]));
}
-
- getTurn(sessionId: string, turnId: string): AgentTurnTraceState | undefined {
- return this.#turns.get(turnKey(sessionId, turnId));
- }
-
- setAction(idempotencyKey: string, state: AgentActionTraceState): void {
- this.#actions.set(idempotencyKey, state);
- }
-
- setActionAnchor(idempotencyKey: string, state: AgentActionTraceState): void {
- this.#actionAnchors.set(idempotencyKey, state);
- }
-
- setSession(sessionId: string, state: AgentSessionTraceState): void {
- this.#sessions.set(sessionId, state);
- }
-
- setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void {
- this.#turns.set(turnKey(sessionId, turnId), state);
- }
-
- updateTurn(
- sessionId: string,
- turnId: string,
- update: (state: AgentTurnTraceState) => AgentTurnTraceState,
+ update<K extends keyof AgentTraceValues>(
+ kind: K,
+ key: string,
+ update: (value: AgentTraceValues[K]) => AgentTraceValues[K],
): void {
- const key = turnKey(sessionId, turnId);
- const state = this.#turns.get(key);
- if (state !== undefined) this.#turns.set(key, update(state));
+ const value = this.get(kind, key);
+ if (value !== undefined) this.set(kind, key, update(value));
+ }
+ entries<K extends keyof AgentTraceValues>(kind: K): [string, AgentTraceValues[K]][] {
+ return [...this.#entries].flatMap(([key, value]) => {
+ const [storedKind, id] = JSON.parse(key);
+ return storedKind === kind
+ ? [[id, value as AgentTraceValues[K]] as [string, AgentTraceValues[K]]]
+ : [];
+ });
}
-}
-
-function turnKey(sessionId: string, turnId: string): string {
- return `${sessionId}:${turnId}`;
}
diff --git a/packages/eve/src/public/channels/eve.test.ts b/packages/eve/src/public/channels/eve.test.ts
index 26b75cbf6..87760e430 100644
--- a/packages/eve/src/public/channels/eve.test.ts
+++ b/packages/eve/src/public/channels/eve.test.ts
@@ -1010,6 +1010,77 @@ describe("eveChannel — onMessage", () => {
});
describe("eveChannel — create session idempotency", () => {
+ it.each([true, false])(
+ "accepts library handoff only with trusted forwarded provenance (%s)",
+ async (trusted) => {
+ const { AsyncLocalStorageContextManager } =
+ await import("@opentelemetry/context-async-hooks");
+ const { context } = await import("@opentelemetry/api");
+ const manager = new AsyncLocalStorageContextManager().enable();
+ context.setGlobalContextManager(manager);
+ try {
+ const handler = createEveCreateHandler({
+ auth: () => ACCEPTED_AUTH,
+ trustedForwarders: () => trusted,
+ });
+ const request = createJsonMessageRequest({
+ callback: {
+ callId: "call-1",
+ subagentName: "research",
+ token: "tok123",
+ url: "https://caller.example.com/eve/v1/callback/tok123",
+ },
+ message: "hi",
+ protocolVersion: REMOTE_AGENT_PROTOCOL_VERSION,
+ ...(trusted
+ ? {
+ forwardedPrincipal: {
+ current: {
+ attributes: {},
+ authenticator: "api-key",
+ principalId: "user",
+ principalType: "user",
+ },
+ },
+ }
+ : {}),
+ });
+ request.headers.set("traceparent", `00-${"a".repeat(32)}-${"b".repeat(16)}-01`);
+ request.headers.set(
+ "baggage",
+ writeForwardedParentSessionBaggage(undefined, {
+ callId: "call-1",
+ rootSessionId: "root",
+ sessionId: "parent",
+ turn: { id: "turn", sequence: 0 },
+ })!,
+ );
+ request.headers.set(
+ "x-agent-tracing",
+ JSON.stringify({
+ caller: { traceId: "a".repeat(32), spanId: "b".repeat(16), traceFlags: 1 },
+ conversationId: "handoff-conversation",
+ parentRunId: "parent",
+ parentCallId: "call-1",
+ agentName: "research",
+ capture: { emit: true, recordInputs: false, recordOutputs: false },
+ }),
+ );
+ expect((await handler.fetch(request)).status).toBe(202);
+ const options = handler.createSession.mock.calls[0]?.[0];
+ expect(options?.conversationId).toBe(trusted ? "handoff-conversation" : undefined);
+ if (trusted)
+ expect(options?.parentTraceContext?.decision).toEqual({
+ action: "record",
+ recordInputs: false,
+ recordOutputs: false,
+ });
+ } finally {
+ context.disable();
+ manager.disable();
+ }
+ },
+ );
it("creates once for an operation id and uses it as the continuation token", async () => {
const handler = createEveCreateHandler({ auth: () => ACCEPTED_AUTH });
diff --git a/packages/eve/src/runtime/connections/mcp-tracing.test.ts b/packages/eve/src/runtime/connections/mcp-tracing.test.ts
index 11d5185dc..034e56143 100644
--- a/packages/eve/src/runtime/connections/mcp-tracing.test.ts
+++ b/packages/eve/src/runtime/connections/mcp-tracing.test.ts
@@ -23,10 +23,7 @@ import {
withMcpToolCallSpan,
withMcpToolsListSpan,
} from "#runtime/connections/mcp-tracing.js";
-import {
- withAgentToolContentPolicy,
- withAgentToolSpanContext,
-} from "#tracing/agent-tool-span-context.js";
+import { otelTelemetry } from "#compiled/@eve/tracing/index.js";
describe("MCP trace propagation", () => {
let exporter: InMemorySpanExporter;
@@ -51,11 +48,20 @@ describe("MCP trace propagation", () => {
it("injects configured propagation into params._meta and annotates the tool span", async () => {
const setAttributes = vi.fn();
- const toolContext = withAgentToolSpanContext(ROOT_CONTEXT, {
- recordInputs: false,
- recordOutputs: false,
- setAttributes,
- });
+ const capture = { emit: true, recordInputs: false, recordOutputs: false };
+ const telemetry = otelTelemetry({ provider });
+ let toolContext = ROOT_CONTEXT;
+ telemetry.run(
+ {
+ type: "tool",
+ reference: { traceId: "1".repeat(32), spanId: "2".repeat(16), traceFlags: 1 },
+ capture,
+ mcp: { update: setAttributes, error() {}, arguments() {}, result() {} },
+ },
+ () => {
+ toolContext = runtimeContext.active();
+ },
+ );
const fetcher = vi.fn(
async (_request: Parameters<typeof fetch>[0], _init?: Parameters<typeof fetch>[1]) =>
new Response(null, { headers: { "mcp-session-id": "session-1" } }),
@@ -102,17 +108,14 @@ describe("MCP trace propagation", () => {
expect(requestBody.params._meta).not.toHaveProperty("recordOutputs");
expect(setAttributes).toHaveBeenCalledWith(
expect.objectContaining({
- "eve.connection.name": "linear",
- "gen_ai.operation.name": "execute_tool",
- "gen_ai.tool.name": "get_issue",
- "jsonrpc.request.id": "42",
- "mcp.method.name": "tools/call",
- "mcp.protocol.version": "2025-11-25",
- "network.protocol.name": "http",
- "network.transport": "tcp",
+ connectionName: "linear",
+ toolName: "get_issue",
+ requestId: "42",
+ method: "tools/call",
+ protocolVersion: "2025-11-25",
}),
);
- expect(setAttributes).toHaveBeenCalledWith({ "mcp.session.id": "session-1" });
+ expect(setAttributes).toHaveBeenCalledWith({ sessionId: "session-1" });
});
it("leaves oversized JSON-RPC requests untouched", async () => {
@@ -174,10 +177,17 @@ describe("MCP trace propagation", () => {
it("propagates the fallback CLIENT span context for tools/call", async () => {
const parent = runtimeTrace.getTracer("test.mcp").startSpan("parent");
- const parentContext = withAgentToolContentPolicy(runtimeTrace.setSpan(ROOT_CONTEXT, parent), {
- recordInputs: true,
- recordOutputs: true,
- });
+ let parentContext = ROOT_CONTEXT;
+ otelTelemetry({ provider }).run(
+ {
+ type: "activation",
+ reference: parent.spanContext(),
+ capture: { emit: true, recordInputs: true, recordOutputs: true },
+ },
+ () => {
+ parentContext = runtimeContext.active();
+ },
+ );
const fetcher = vi.fn(
async (_request: Parameters<typeof fetch>[0], _init?: Parameters<typeof fetch>[1]) =>
new Response(null, { headers: { "mcp-session-id": "session-1" } }),
diff --git a/packages/eve/src/runtime/connections/mcp-tracing.ts b/packages/eve/src/runtime/connections/mcp-tracing.ts
index d7df9a7ad..32f20d28f 100644
--- a/packages/eve/src/runtime/connections/mcp-tracing.ts
+++ b/packages/eve/src/runtime/connections/mcp-tracing.ts
@@ -1,380 +1,34 @@
-import {
- context as otelContext,
- createContextKey,
- propagation,
- SpanKind,
- trace,
- type Attributes,
- type Context,
- type Span,
- type TextMapSetter,
-} from "#compiled/@opentelemetry/api/index.js";
-
-import { contentAttribute } from "#tracing/agent-otel-content.js";
-import { recordAgentSpanError } from "#tracing/agent-span-error.js";
-import {
- agentToolContentPolicy,
- agentToolSpanContext,
- annotateAgentToolSpan,
- recordAgentToolSpanError,
- withAgentToolSpanContext,
-} from "#tracing/agent-tool-span-context.js";
-import { truncateTelemetryText } from "#tracing/telemetry-budget.js";
+import { createMcpTraceFetch as traceFetch } from "#tracing/eve/mcp.js";
+import { createMcpTracing } from "#tracing/eve/mcp.js";
import { replaceBaggageMember } from "#protocol/baggage.js";
-import { isObject } from "#shared/guards.js";
-
-const MAX_MCP_TRACE_REQUEST_BYTES = 1024 * 1024;
-const MAX_MCP_TRACE_CONTEXT_BYTES = 8192;
-const utf8Encoder = new TextEncoder();
-const EVE_TRACE_BAGGAGE_KEYS = [
- "eve.audience",
- "eve.conversation.id",
- "eve.parent_session",
-] as const;
-const traceContextSetter: TextMapSetter<Record<string, string>> = {
- set(carrier, key, value) {
- carrier[key] = value;
- },
-};
-const MCP_METHOD_NAME_CONTEXT_KEY = createContextKey("eve.mcp.method-name");
-
-export function createMcpTraceFetch(input: {
- readonly connectionName: string;
- readonly fetcher?: typeof fetch;
- readonly getActiveContext?: () => Context;
- readonly getProtocolVersion: () => string | undefined;
- readonly injectContext?: McpTraceContextInjector;
-}): typeof fetch {
- const fetcher = input.fetcher ?? ((request, init) => globalThis.fetch(request, init));
- const getActiveContext = input.getActiveContext ?? (() => otelContext.active());
- const injectContext = input.injectContext ?? injectOpenTelemetryTraceContext;
-
- return async (request, init) => {
- const message = readJsonRpcRequest(init?.body);
- if (message === undefined) return await fetcher(request, init);
-
- const activeContext = getActiveContext();
- const propagated = injectMcpTraceContext(message, activeContext, injectContext);
- const annotateRequest =
- message.method === "tools/call" || mcpMethodName(activeContext) === message.method;
- if (annotateRequest) {
- annotateAgentToolSpan(
- mcpRequestAttributes({
- connectionName: input.connectionName,
- message,
- params: isObject(message.params) ? message.params : {},
- protocolVersion:
- headerValue(init?.headers, "mcp-protocol-version") ?? input.getProtocolVersion(),
- }),
- activeContext,
- );
- }
-
- const response = await fetcher(
- request,
- propagated === undefined ? init : { ...init, body: JSON.stringify(propagated) },
- );
- if (annotateRequest) {
- const sessionId = response.headers.get("mcp-session-id");
- if (sessionId !== null) {
- annotateAgentToolSpan(
- { "mcp.session.id": truncateTelemetryText(sessionId, 512) },
- activeContext,
- );
- }
- }
- return response;
- };
-}
-function injectMcpTraceContext(
- message: JsonRpcRequest,
- activeContext: Context,
- injectContext: McpTraceContextInjector = injectOpenTelemetryTraceContext,
-): JsonRpcRequest | undefined {
- if (!withinTraceRequestLimit(JSON.stringify(message))) return undefined;
-
- if (message.params !== undefined && !isObject(message.params)) return undefined;
- const propagated: Record<string, string> = {};
- injectContext(activeContext, propagated);
- if (!withinTraceContextLimit(propagated)) return undefined;
- stripEveTraceBaggage(propagated);
-
- const params = isObject(message.params) ? message.params : {};
- const existingMeta = isObject(params["_meta"]) ? params["_meta"] : {};
- return {
- ...message,
- params: {
- ...params,
- _meta: { ...existingMeta, ...propagated },
+export function createMcpTraceFetch(input: Parameters<typeof traceFetch>[0]): typeof fetch {
+ return traceFetch({
+ ...input,
+ filterCarrier(carrier) {
+ let baggage = carrier.baggage;
+ for (const key of ["eve.audience", "eve.conversation.id", "eve.parent_session"])
+ baggage = replaceBaggageMember(baggage, key, undefined);
+ if (baggage === undefined) delete carrier.baggage;
+ else carrier.baggage = baggage;
},
- };
-}
-
-export async function withMcpToolsListSpan<T>(input: {
- readonly connectionName: string;
- readonly execute: () => Promise<T>;
- readonly protocolVersion?: string;
-}): Promise<T> {
- const parent = otelContext.active();
- const attributes = mcpRequestAttributes({
- connectionName: input.connectionName,
- method: "tools/list",
- protocolVersion: input.protocolVersion,
});
- const span = trace.getTracer("eve.mcp").startSpan(
- "tools/list",
- {
- attributes,
- kind: SpanKind.CLIENT,
- },
- parent,
- );
- const spanContext = withMcpMethodName(
- withAgentToolSpanContext(trace.setSpan(parent, span), {
- ...agentToolContentPolicy(parent),
- recordError: (error, errorType) => recordAgentSpanError(span, error, errorType),
- setAttributes: (spanAttributes) => setSpanAttributes(span, spanAttributes),
- }),
- "tools/list",
- );
- try {
- return await otelContext.with(spanContext, async () => {
- try {
- return await input.execute();
- } catch (error) {
- const code = jsonRpcErrorCode(error);
- if (code !== undefined) {
- annotateAgentToolSpan({ "rpc.response.status_code": code }, spanContext);
- }
- recordAgentToolSpanError(error, errorType(error, code), spanContext);
- throw error;
- }
- });
- } finally {
- span.end();
- }
}
-export async function withMcpToolCallSpan<T>(input: {
- readonly arguments: unknown;
- readonly connectionName: string;
- readonly execute: () => Promise<T>;
- readonly protocolVersion?: string;
- readonly toolName: string;
+export function withMcpToolsListSpan<T>(input: {
+ connectionName: string;
+ protocolVersion?: string;
+ execute: () => Promise<T>;
}): Promise<T> {
- const parent = otelContext.active();
- const existing = agentToolSpanContext(parent);
- const policy = agentToolContentPolicy(parent);
- if (existing?.setAttributes !== undefined) {
- annotateAgentToolSpan(mcpToolCallAttributes(input), parent);
- return await runMcpToolCall(input, parent, undefined, policy);
- }
-
- const span = trace.getTracer("eve.mcp").startSpan(
- `tools/call ${truncateTelemetryText(input.toolName, 128)}`,
- {
- attributes: mcpToolCallAttributes(input),
- kind: SpanKind.CLIENT,
- },
- parent,
- );
- const spanContext = withMcpMethodName(
- withAgentToolSpanContext(trace.setSpan(parent, span), {
- ...policy,
- recordError: (error, errorType) => recordAgentSpanError(span, error, errorType),
- setAttributes: (attributes) => setSpanAttributes(span, attributes),
- }),
- "tools/call",
- );
-
- try {
- return await otelContext.with(spanContext, () =>
- runMcpToolCall(input, spanContext, span, policy),
- );
- } finally {
- span.end();
- }
-}
-
-function runMcpToolCall<T>(
- input: {
- readonly arguments: unknown;
- readonly execute: () => Promise<T>;
- },
- context: Context,
- fallbackSpan: Span | undefined,
- policy: ReturnType<typeof agentToolContentPolicy>,
-): Promise<T> {
- if (fallbackSpan !== undefined && policy.recordInputs) {
- const argumentsAttribute = contentAttribute(input.arguments);
- if (argumentsAttribute !== undefined) {
- fallbackSpan.setAttribute("gen_ai.tool.call.arguments", argumentsAttribute);
- }
- }
-
- return input
- .execute()
- .then((result) => {
- if (isToolErrorResult(result)) {
- recordAgentToolSpanError(undefined, "tool_error", context);
- }
- if (fallbackSpan !== undefined && policy.recordOutputs) {
- const resultAttribute = contentAttribute(result);
- if (resultAttribute !== undefined) {
- fallbackSpan.setAttribute("gen_ai.tool.call.result", resultAttribute);
- }
- }
- return result;
- })
- .catch((error: unknown) => {
- const code = jsonRpcErrorCode(error);
- if (code !== undefined) {
- annotateAgentToolSpan({ "rpc.response.status_code": code }, context);
- }
- if (fallbackSpan !== undefined) {
- recordAgentToolSpanError(error, errorType(error, code), context);
- }
- throw error;
- });
-}
-
-function mcpToolCallAttributes(input: {
- readonly connectionName: string;
- readonly protocolVersion?: string;
- readonly toolName: string;
-}): Attributes {
- return mcpRequestAttributes({
- connectionName: input.connectionName,
- method: "tools/call",
- protocolVersion: input.protocolVersion,
- toolName: input.toolName,
- });
-}
-
-function mcpRequestAttributes(input: {
- readonly connectionName: string;
- readonly message?: JsonRpcRequest;
- readonly method?: string;
- readonly params?: Record<string, unknown>;
- readonly protocolVersion?: string;
- readonly toolName?: string;
-}): Attributes {
- const method = input.method ?? input.message?.method;
- const params = input.params ?? (isObject(input.message?.params) ? input.message.params : {});
- const attributes: Attributes = {
- "eve.connection.name": input.connectionName,
- "mcp.method.name": method ?? "unknown",
- "network.protocol.name": "http",
- "network.transport": "tcp",
- };
- const toolName =
- input.toolName ?? (typeof params["name"] === "string" ? params["name"] : undefined);
- if (method === "tools/call") {
- attributes["gen_ai.operation.name"] = "execute_tool";
- if (toolName !== undefined) attributes["gen_ai.tool.name"] = toolName;
- }
- const requestId = input.message?.id;
- if (typeof requestId === "number" || typeof requestId === "string") {
- attributes["jsonrpc.request.id"] = String(requestId);
- }
- const meta = params["_meta"];
- const protocolVersion =
- input.protocolVersion ??
- (isObject(meta) && typeof meta["io.modelcontextprotocol/protocolVersion"] === "string"
- ? meta["io.modelcontextprotocol/protocolVersion"]
- : undefined);
- if (protocolVersion !== undefined) {
- attributes["mcp.protocol.version"] = protocolVersion;
- }
- return attributes;
-}
-
-interface JsonRpcRequest extends Record<string, unknown> {
- readonly method: string;
- readonly params?: unknown;
-}
-
-type McpTraceContextInjector = (context: Context, carrier: Record<string, string>) => void;
-
-function injectOpenTelemetryTraceContext(context: Context, carrier: Record<string, string>): void {
- propagation.inject(context, carrier, traceContextSetter);
+ return createMcpTracing().list(input);
}
-function stripEveTraceBaggage(carrier: Record<string, string>): void {
- let baggage = carrier["baggage"];
- if (baggage === undefined) return;
- for (const key of EVE_TRACE_BAGGAGE_KEYS) {
- baggage = replaceBaggageMember(baggage, key, undefined);
- }
- if (baggage === undefined) delete carrier["baggage"];
- else carrier["baggage"] = baggage;
-}
-
-function readJsonRpcRequest(body: RequestInit["body"]): JsonRpcRequest | undefined {
- if (typeof body !== "string" || !withinTraceRequestLimit(body)) return undefined;
- try {
- const value: unknown = JSON.parse(body);
- return isObject(value) && typeof value["method"] === "string"
- ? (value as JsonRpcRequest)
- : undefined;
- } catch {
- return undefined;
- }
-}
-
-function withinTraceRequestLimit(value: string): boolean {
- return (
- value.length <= MAX_MCP_TRACE_REQUEST_BYTES &&
- utf8Encoder.encode(value).byteLength <= MAX_MCP_TRACE_REQUEST_BYTES
- );
-}
-
-function withinTraceContextLimit(carrier: Readonly<Record<string, string>>): boolean {
- let bytes = 0;
- for (const [key, value] of Object.entries(carrier)) {
- if (key.length + value.length > MAX_MCP_TRACE_CONTEXT_BYTES) return false;
- bytes += utf8Encoder.encode(key).byteLength + utf8Encoder.encode(value).byteLength + 2;
- if (bytes > MAX_MCP_TRACE_CONTEXT_BYTES) return false;
- }
- return bytes > 0;
-}
-
-function headerValue(headers: RequestInit["headers"], name: string): string | undefined {
- if (headers === undefined) return undefined;
- try {
- return new Headers(headers).get(name) ?? undefined;
- } catch {
- return undefined;
- }
-}
-
-function isToolErrorResult(value: unknown): boolean {
- return isObject(value) && value["isError"] === true;
-}
-
-function jsonRpcErrorCode(error: unknown): string | undefined {
- if (!isObject(error)) return undefined;
- const code = error["code"];
- return typeof code === "number" || typeof code === "string" ? String(code) : undefined;
-}
-
-function errorType(error: unknown, code: string | undefined): string | undefined {
- if (code !== undefined) return code;
- return error instanceof Error ? error.name || "Error" : undefined;
-}
-
-function setSpanAttributes(span: Span, attributes: Attributes): void {
- for (const [name, value] of Object.entries(attributes)) {
- if (value !== undefined) span.setAttribute(name, value);
- }
-}
-
-function withMcpMethodName(context: Context, methodName: string): Context {
- return context.setValue(MCP_METHOD_NAME_CONTEXT_KEY, methodName);
-}
-
-function mcpMethodName(context: Context): string | undefined {
- const methodName = context.getValue(MCP_METHOD_NAME_CONTEXT_KEY);
- return typeof methodName === "string" ? methodName : undefined;
+export function withMcpToolCallSpan<T>(input: {
+ connectionName: string;
+ protocolVersion?: string;
+ toolName: string;
+ arguments: unknown;
+ execute: () => Promise<T>;
+}): Promise<T> {
+ return createMcpTracing().call(input);
}
diff --git a/packages/eve/src/tracing/sampled-trace.test.ts b/packages/eve/src/shared/trace-policy.test.ts
similarity index 99%
rename from packages/eve/src/tracing/sampled-trace.test.ts
rename to packages/eve/src/shared/trace-policy.test.ts
index e48709178..d5642c503 100644
--- a/packages/eve/src/tracing/sampled-trace.test.ts
+++ b/packages/eve/src/shared/trace-policy.test.ts
@@ -1,6 +1,6 @@
import { afterEach, describe, expect, it, vi } from "vitest";
-import { resolveTracePolicy, resolveTracePolicyDecision } from "#tracing/sampled-trace.js";
+import { resolveTracePolicy, resolveTracePolicyDecision } from "#shared/trace-policy.js";
const contentContext = (
audience: "public" | "private" | "unknown",
diff --git a/packages/eve/src/shared/trace-policy.ts b/packages/eve/src/shared/trace-policy.ts
index 1d554180c..3610cd440 100644
--- a/packages/eve/src/shared/trace-policy.ts
+++ b/packages/eve/src/shared/trace-policy.ts
@@ -1,5 +1,10 @@
import { shouldCaptureInstrumentationContent } from "#shared/instrumentation-content.js";
import type { ConversationContext } from "#shared/conversation-context.js";
+
+// Keep sampled-flag checks free of OTel imports for workflow driver bundles.
+export function isSampledTrace(context: { readonly traceFlags: number }): boolean {
+ return (context.traceFlags & 1) !== 0;
+}
import {
DROP_INSTRUMENTATION,
type InstrumentationDecision,
diff --git a/packages/eve/src/tracing/agent-approval-instrumentation.ts b/packages/eve/src/tracing/agent-approval-instrumentation.ts
deleted file mode 100644
index c3ef5df8b..000000000
--- a/packages/eve/src/tracing/agent-approval-instrumentation.ts
+++ /dev/null
@@ -1,193 +0,0 @@
-import {
- ROOT_CONTEXT,
- type SpanContext,
- type Tracer,
- trace,
-} from "#compiled/@opentelemetry/api/index.js";
-
-import type {
- InstrumentationHandlerContext,
- InstrumentationInputRequestedEvent,
- InstrumentationInputResolvedEvent,
- InstrumentationProviderDefinition,
-} from "#instrumentation/lifecycle.js";
-import type { JsonValue } from "#shared/json.js";
-import { contentAttribute } from "#tracing/agent-otel-content.js";
-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";
-import { agentTraceIdentityAttributes, traceSessionIdOf } from "#tracing/agent-otel-attributes.js";
-import { decodeTraceSessionId } from "#tracing/agent-trace-context-codec.js";
-import { AGENT_SPAN_NAMES } from "#tracing/agent-span-contract.js";
-import { recordAgentSpanError as recordError } from "#tracing/agent-span-error.js";
-import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";
-import type { AgentToolContext } from "#tracing/agent-tool-instrumentation.js";
-import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
-import { normalizeChannelAudience, type ChannelAudience } from "#shared/channel-audience.js";
-
-interface AgentApprovalSpanState {
- readonly traceSessionId: string;
- readonly actionCallId: string;
- readonly actionName: string;
- readonly attemptIndex: number;
- readonly channelAudience: ChannelAudience;
- readonly parent: SpanContext;
- readonly requestAttribute?: string;
- readonly requestId: string;
- readonly rootSessionId: string;
- readonly sessionId: string;
- readonly startTimeMs: number;
- readonly stepIndex: number;
- readonly turnId: string;
-}
-
-/** Builds durable approval wait spans under their originating runtime action. */
-export function createAgentApprovalInstrumentation(input: {
- readonly actionContextFor: (
- sessionId: string,
- turnId: string,
- callId: string,
- ) => Promise<AgentToolContext | undefined>;
- readonly frameworkVersion: string;
- readonly idGenerator: AgentSpanIdGenerator;
- readonly tracer: Tracer;
-}): Pick<
- NonNullable<InstrumentationProviderDefinition["events"]>,
- "input.requested" | "input.resolved"
-> {
- const onRequested = async (
- event: InstrumentationInputRequestedEvent,
- ctx: InstrumentationHandlerContext,
- ): Promise<void> => {
- if (event.kind !== "tool-approval" || readState(ctx.state.get()) !== undefined) return;
- const parent = await input.actionContextFor(
- event.scope.sessionId,
- event.scope.turnId,
- event.action.callId,
- );
- if (parent === undefined) return;
- const state: Record<string, JsonValue> = {
- actionCallId: event.action.callId,
- actionName: event.action.name,
- attemptIndex: event.scope.attemptIndex,
- channelAudience: normalizeChannelAudience(event.scope.channelAudience),
- parent: {
- spanId: parent.spanContext.spanId,
- traceFlags: parent.spanContext.traceFlags,
- traceId: parent.spanContext.traceId,
- },
- requestId: event.requestId,
- rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,
- traceSessionId: traceSessionIdOf(event.scope),
- sessionId: event.scope.sessionId,
- startTimeMs: Date.now(),
- stepIndex: event.scope.stepIndex,
- turnId: event.scope.turnId,
- };
- const requestAttribute = contentAttribute(event.request);
- if (requestAttribute !== undefined) state["requestAttribute"] = requestAttribute;
- ctx.state.set(state);
- };
-
- const onResolved = (
- event: InstrumentationInputResolvedEvent,
- ctx: InstrumentationHandlerContext,
- ): void => {
- const state = readState(ctx.state.get());
- if (state === undefined) return;
- const span = input.idGenerator.withSpanId(
- input.idGenerator.deriveSpanId(`approval:${event.idempotencyKey}`),
- () =>
- input.tracer.startSpan(
- AGENT_SPAN_NAMES.approval,
- {
- attributes: {
- "gen_ai.tool.call.id": state.actionCallId,
- "agent.action.name": state.actionName,
- "agent.approval.kind": "tool-approval",
- "agent.approval.outcome": event.outcome,
- "agent.approval.request_id": state.requestId,
- "agent.framework.name": "eve",
- "agent.framework.version": input.frameworkVersion,
- "agent.step.attempt": state.attemptIndex,
- "agent.step.index": state.stepIndex,
- "agent.turn.id": state.turnId,
- "gen_ai.operation.name": "workflow",
- ...agentSpanNamingAttributes("agent.approval", "workflow"),
- ...agentTraceIdentityAttributes({
- rootSessionId: state.rootSessionId,
- traceSessionId: state.traceSessionId,
- sessionId: state.sessionId,
- }),
- },
- startTime: state.startTimeMs,
- },
- withChannelAudience(
- trace.setSpan(
- ROOT_CONTEXT,
- trace.wrapSpanContext({ ...state.parent, isRemote: false }),
- ),
- state.channelAudience,
- ),
- ),
- );
- if (state.requestAttribute !== undefined) {
- span.setAttribute("agent.approval.request", state.requestAttribute);
- }
- if (event.response !== undefined) {
- const response = contentAttribute(event.response);
- if (response !== undefined) span.setAttribute("agent.approval.response", response);
- }
- if (event.outcome === "failed") recordError(span, event.error);
- span.end();
- };
-
- return {
- "input.requested": onRequested,
- "input.resolved": onResolved,
- };
-}
-
-function readState(value: unknown): AgentApprovalSpanState | undefined {
- if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;
- const state = value as Record<string, unknown>;
- const parent = state["parent"];
- if (typeof parent !== "object" || parent === null || Array.isArray(parent)) return undefined;
- const parentRecord = parent as Record<string, unknown>;
- if (
- typeof state["actionCallId"] !== "string" ||
- typeof state["actionName"] !== "string" ||
- typeof state["attemptIndex"] !== "number" ||
- typeof parentRecord["spanId"] !== "string" ||
- typeof parentRecord["traceFlags"] !== "number" ||
- typeof parentRecord["traceId"] !== "string" ||
- typeof state["requestId"] !== "string" ||
- typeof state["rootSessionId"] !== "string" ||
- typeof state["sessionId"] !== "string" ||
- typeof state["startTimeMs"] !== "number" ||
- typeof state["stepIndex"] !== "number" ||
- typeof state["turnId"] !== "string"
- ) {
- return undefined;
- }
- const requestAttribute = state["requestAttribute"];
- if (requestAttribute !== undefined && typeof requestAttribute !== "string") return undefined;
- return {
- actionCallId: state["actionCallId"],
- actionName: state["actionName"],
- attemptIndex: state["attemptIndex"],
- channelAudience: normalizeChannelAudience(state["channelAudience"]),
- parent: {
- isRemote: false,
- spanId: parentRecord["spanId"],
- traceFlags: parentRecord["traceFlags"],
- traceId: parentRecord["traceId"],
- },
- requestAttribute,
- requestId: state["requestId"],
- rootSessionId: state["rootSessionId"],
- traceSessionId: decodeTraceSessionId(state),
- sessionId: state["sessionId"],
- startTimeMs: state["startTimeMs"],
- stepIndex: state["stepIndex"],
- turnId: state["turnId"],
- };
-}
diff --git a/packages/eve/src/tracing/agent-memory-instrumentation.ts b/packages/eve/src/tracing/agent-memory-instrumentation.ts
deleted file mode 100644
index 5a31a500a..000000000
--- a/packages/eve/src/tracing/agent-memory-instrumentation.ts
+++ /dev/null
@@ -1,195 +0,0 @@
-import {
- ROOT_CONTEXT,
- SpanKind,
- context,
- trace,
- type Context,
- type Span,
- type Tracer,
-} from "#compiled/@opentelemetry/api/index.js";
-
-import { contextStorage } from "#context/container.js";
-import { SessionTraceSeedKey } from "#context/keys.js";
-import type { InstrumentationProviderDefinition } from "#instrumentation/lifecycle.js";
-import type {
- InstrumentationMemoryExecutionOperation,
- InstrumentationMemoryOperationEvent,
- InstrumentationMemoryOperationStartedEvent,
- InstrumentationMemoryOperationTerminalEvent,
-} from "#instrumentation/memory.js";
-import { normalizeChannelAudience } from "#shared/channel-audience.js";
-import {
- applyLiveDeliveryAudienceCeiling,
- resolveForwardedTraceSeed,
-} from "#shared/forwarded-trace-policy.js";
-import { genAiMemoryRecordsAttribute } from "#tracing/agent-otel-content.js";
-import { agentTraceIdentityAttributes, traceSessionIdOf } from "#tracing/agent-otel-attributes.js";
-import { isAgentTraceContext, markAgentTraceContext } from "#tracing/eve/agent-trace-context.js";
-import { recordAgentSpanError } from "#tracing/agent-span-error.js";
-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";
-import type { AgentTraceStateStore } from "#tracing/agent-trace-state.js";
-import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";
-import { withErrorContent } from "#tracing/error-content-context.js";
-import { isSampledTrace } from "#tracing/sampled-trace.js";
-import { suppressTracing } from "#tracing/suppress-tracing.js";
-import { withOperationConversation } from "#tracing/eve/conversation-context.js";
-import type { ConversationEnvironment } from "#shared/conversation-context.js";
-
-type SpanState = { readonly context: Context; readonly span: Span };
-
-interface AgentMemoryInstrumentation {
- readonly events: Pick<
- NonNullable<InstrumentationProviderDefinition["events"]>,
- "memory.operation.completed" | "memory.operation.failed" | "memory.operation.started"
- >;
- runInContext<T>(
- operation: InstrumentationMemoryExecutionOperation,
- execute: () => PromiseLike<T>,
- ): Promise<T>;
-}
-
-export function createAgentMemoryInstrumentation(input: {
- readonly environment: ConversationEnvironment;
- readonly recordOutputs?: boolean;
- readonly stateStore: AgentTraceStateStore;
- readonly tracer: Tracer;
-}): AgentMemoryInstrumentation {
- const recordOutputs = input.recordOutputs ?? false;
- const spans = new Map<string, SpanState>();
-
- const parentContext = async (
- event: InstrumentationMemoryOperationEvent,
- ): Promise<Context | undefined> => {
- const active = context.active();
- if (isAgentTraceContext(active)) return active;
-
- const session = await input.stateStore.getSession(event.sessionId);
- if (event.turnId !== undefined) {
- const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);
- if (turn !== undefined) {
- return withChannelAudience(contextFromSpanContext(turn.context), session?.channelAudience);
- }
- }
- return session === undefined
- ? undefined
- : withChannelAudience(contextFromSpanContext(session.context), session.channelAudience);
- };
-
- const onStarted = async (event: InstrumentationMemoryOperationStartedEvent): Promise<void> => {
- if (spans.has(event.idempotencyKey)) return;
- const parent = await parentContext(event);
- const parentSpan = parent === undefined ? undefined : trace.getSpan(parent)?.spanContext();
- if (parent === undefined || parentSpan === undefined || !isSampledTrace(parentSpan)) return;
- const span = input.tracer.startSpan(
- event.operationName,
- {
- attributes: memorySpanAttributes(event),
- kind: SpanKind.CLIENT,
- },
- parent,
- );
- spans.set(event.idempotencyKey, {
- context: trace.setSpan(parent, span),
- span,
- });
- };
-
- const onTerminal = (event: InstrumentationMemoryOperationTerminalEvent): void => {
- const state = spans.get(event.idempotencyKey);
- if (state === undefined) return;
- spans.delete(event.idempotencyKey);
- updateMemorySpan(state.span, event);
- state.span.end();
- };
-
- return {
- events: {
- "memory.operation.completed": onTerminal,
- "memory.operation.failed": onTerminal,
- "memory.operation.started": onStarted,
- },
- async runInContext(operation, execute) {
- const session = await input.stateStore.getSession(operation.sessionId);
- let parent = spans.get(operation.idempotencyKey)?.context;
- if (parent === undefined && operation.turnId !== undefined) {
- const turn = await input.stateStore.getTurn(operation.sessionId, operation.turnId);
- if (turn !== undefined) {
- parent = withChannelAudience(
- contextFromSpanContext(turn.context),
- session?.channelAudience,
- );
- if (!isSampledTrace(turn.context)) parent = suppressTracing(parent);
- }
- }
- if (parent === undefined && session !== undefined) {
- parent = withChannelAudience(
- contextFromSpanContext(session.context),
- session.channelAudience,
- );
- if (!isSampledTrace(session.context)) parent = suppressTracing(parent);
- }
- const seed = resolveForwardedTraceSeed(contextStorage.getStore()?.get(SessionTraceSeedKey));
- const decision = seed?.decision ?? session?.decision;
- const effective =
- decision === undefined
- ? undefined
- : applyLiveDeliveryAudienceCeiling(
- decision,
- normalizeChannelAudience(session?.channelAudience),
- seed?.forwardedTracePolicy,
- input.environment,
- );
- return parent === undefined
- ? await execute()
- : await context.with(
- markAgentTraceContext(
- withErrorContent(
- withOperationConversation(parent, {
- rootSessionId: session?.rootSessionId,
- sessionId: operation.sessionId,
- }),
- recordOutputs && effective?.action === "record" && effective.recordOutputs,
- ),
- ),
- execute,
- );
- },
- };
-}
-
-function memorySpanAttributes(
- event: InstrumentationMemoryOperationEvent,
-): Record<string, string | number> {
- const attributes: Record<string, string | number> = {
- "agent.memory.phase": event.phase,
- "agent.memory.slot": event.slot,
- "gen_ai.memory.store.id": event.storeId,
- "gen_ai.operation.name": event.operationName,
- ...agentSpanNamingAttributes(event.operationName, event.operationName),
- ...agentTraceIdentityAttributes({
- rootSessionId: event.rootSessionId,
- traceSessionId: traceSessionIdOf(event),
- sessionId: event.sessionId,
- }),
- };
- if (event.turnId !== undefined) attributes["agent.turn.id"] = event.turnId;
- return attributes;
-}
-
-function updateMemorySpan(span: Span, event: InstrumentationMemoryOperationTerminalEvent): void {
- if (event.type === "memory.operation.failed") {
- recordAgentSpanError(span, event.error);
- } else {
- if (event.recordCount !== undefined) {
- span.setAttribute("gen_ai.memory.record.count", event.recordCount);
- }
- const records = genAiMemoryRecordsAttribute(event.outputRecords ?? []);
- if (records !== undefined && event.outputRecords !== undefined) {
- span.setAttribute("gen_ai.memory.records", records);
- }
- }
-}
-
-function contextFromSpanContext(spanContext: Parameters<typeof trace.wrapSpanContext>[0]): Context {
- return trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(spanContext));
-}
diff --git a/packages/eve/src/tracing/agent-otel-attributes.test.ts b/packages/eve/src/tracing/agent-otel-attributes.test.ts
deleted file mode 100644
index 77ed5e893..000000000
--- a/packages/eve/src/tracing/agent-otel-attributes.test.ts
+++ /dev/null
@@ -1,74 +0,0 @@
-import { afterEach, describe, expect, it, vi } from "vitest";
-
-import { ContextContainer, contextStorage } from "#context/container.js";
-import { ConversationIdKey, TraceRootKey } from "#context/keys.js";
-import { agentTraceIdentityAttributes } from "#tracing/agent-otel-attributes.js";
-
-afterEach(() => {
- vi.unstubAllEnvs();
-});
-
-describe("agentTraceIdentityAttributes", () => {
- it("uses the lineage fallback when a remote entry has no conversation in context", () => {
- vi.stubEnv("VERCEL_ENV", "preview");
- const context = new ContextContainer();
- context.set(TraceRootKey, { kind: "own" });
- contextStorage.run(context, () => {
- expect(
- agentTraceIdentityAttributes({
- rootSessionId: "caller-root",
- sessionId: "remote-session",
- traceSessionId: "remote-session",
- }),
- ).toMatchObject({
- "gen_ai.conversation.id": "caller-root",
- "vercel.session_id": "remote-session",
- "agent.run.id": "remote-session",
- });
- });
- });
- const input = {
- rootSessionId: "root-session",
- sessionId: "session-1",
- traceSessionId: "root-session",
- };
- it("uses the explicit trace session when active context belongs to another session", () => {
- vi.stubEnv("VERCEL_ENV", "preview");
- const context = new ContextContainer();
- context.set(TraceRootKey, { kind: "inherited", sessionId: "unrelated-session" });
- contextStorage.run(context, () => {
- expect(
- agentTraceIdentityAttributes({
- rootSessionId: "caller-root",
- sessionId: "remote",
- traceSessionId: "remote",
- })["vercel.session_id"],
- ).toBe("remote");
- });
- });
-
- it("uses the GenAI conversation identifier outside Vercel", () => {
- vi.stubEnv("VERCEL_ENV", undefined);
-
- expect(agentTraceIdentityAttributes(input)).toEqual({
- "agent.run.id": "session-1",
- "agent.trace.schema.version": 4,
- "gen_ai.conversation.id": "root-session",
- });
- });
-
- it("adds the Vercel session identifier on Vercel", () => {
- vi.stubEnv("VERCEL_ENV", "preview");
- const context = new ContextContainer();
- context.set(ConversationIdKey, "caller-conversation");
-
- contextStorage.run(context, () => {
- expect(agentTraceIdentityAttributes(input)).toEqual({
- "agent.run.id": "session-1",
- "agent.trace.schema.version": 4,
- "gen_ai.conversation.id": "caller-conversation",
- "vercel.session_id": "root-session",
- });
- });
- });
-});
diff --git a/packages/eve/src/tracing/agent-otel-attributes.ts b/packages/eve/src/tracing/agent-otel-attributes.ts
deleted file mode 100644
index dfa77f366..000000000
--- a/packages/eve/src/tracing/agent-otel-attributes.ts
+++ /dev/null
@@ -1,27 +0,0 @@
-import { operationConversationId } from "#tracing/eve/conversation-context.js";
-import { AGENT_TRACE_SCHEMA_VERSION } from "#tracing/agent-span-contract.js";
-
-export function traceSessionIdOf(scope: {
- readonly traceSessionId?: string;
- readonly rootSessionId?: string;
- readonly sessionId: string;
-}): string {
- return scope.traceSessionId ?? scope.rootSessionId ?? scope.sessionId;
-}
-
-export function agentTraceIdentityAttributes(input: {
- readonly rootSessionId: string;
- readonly sessionId: string;
- readonly traceSessionId: string;
-}): Record<string, string | number> {
- const conversationId = operationConversationId(input);
- const attributes: Record<string, string | number> = {
- "agent.run.id": input.sessionId,
- "agent.trace.schema.version": AGENT_TRACE_SCHEMA_VERSION,
- "gen_ai.conversation.id": conversationId,
- };
- if (process.env.VERCEL_ENV !== undefined) {
- attributes["vercel.session_id"] = input.traceSessionId;
- }
- return attributes;
-}
diff --git a/packages/eve/src/tracing/agent-otel-content.test.ts b/packages/eve/src/tracing/agent-otel-content.test.ts
deleted file mode 100644
index 74dd467a0..000000000
--- a/packages/eve/src/tracing/agent-otel-content.test.ts
+++ /dev/null
@@ -1,297 +0,0 @@
-import { describe, expect, it, vi } from "vitest";
-
-import type { FrameworkMessageKind } from "#harness/messages.js";
-import {
- CONTENT_ATTRIBUTE_LIMIT,
- genAiInputMessagesAttribute,
- genAiOutputMessagesAttribute,
- genAiSystemInstructionsAttribute,
- genAiToolDefinitionsAttribute,
- toolResultsContentAttribute,
-} from "#tracing/agent-otel-content.js";
-
-const FRAMEWORK_MESSAGE_KINDS = [
- "context.instruction",
- "context.state",
- "context.compaction",
- "memory.load",
- "execution.continuation",
- "execution.retry",
-] as const satisfies readonly FrameworkMessageKind[];
-
-describe("GenAI message attributes", () => {
- it.each([
- ["Buffer", () => Buffer.alloc(64, 97)],
- ["Uint8Array", () => new Uint8Array(64).fill(97)],
- ["base64", () => "a".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)],
- ["data URL", () => `data:image/png;base64,${"a".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)}`],
- ] as const)("omits %s attachment payloads", (_, createData) => {
- const data = createData();
- const entries = Object.entries;
- let binaryEnumerations = 0;
- const spy = vi.spyOn(Object, "entries").mockImplementation((value) => {
- if (value === data) binaryEnumerations += 1;
- return entries(value);
- });
-
- let attribute: string | undefined;
- try {
- attribute = genAiInputMessagesAttribute([
- {
- role: "user",
- content: [
- { type: "text", text: "Summarize file." },
- { type: "file", mediaType: "application/octet-stream", filename: "test.bin", data },
- ],
- },
- ]);
- } finally {
- spy.mockRestore();
- }
-
- expect(binaryEnumerations).toBe(0);
- expect(attribute).toBe(
- '[{"parts":[{"content":"Summarize file.","type":"text"},{"type":"file"}],"role":"user"}]',
- );
- });
-
- it("preserves every user-message kind in the GenAI input attribute", () => {
- expect(
- genAiInputMessagesAttribute([
- { content: "A real user message.", kind: "user", role: "user" },
- {
- content: "Continue the interrupted turn.",
- kind: "execution.continuation",
- role: "user",
- },
- ]),
- ).toBe(
- '[{"kind":"user","parts":[{"content":"A real user message.","type":"text"}],"role":"user"},{"kind":"execution.continuation","parts":[{"content":"Continue the interrupted turn.","type":"text"}],"role":"user"}]',
- );
- });
-
- it.each(FRAMEWORK_MESSAGE_KINDS)("preserves %s in the GenAI input attribute", (kind) => {
- const attribute = genAiInputMessagesAttribute([
- { content: "Framework message.", kind, role: "user" },
- ]);
-
- expect(JSON.parse(attribute!)).toEqual([
- {
- kind,
- parts: [{ content: "Framework message.", type: "text" }],
- role: "user",
- },
- ]);
- });
-
- it.each([
- ["only", [{ content: "x".repeat(CONTENT_ATTRIBUTE_LIMIT + 1), role: "user" }], undefined],
- [
- "newest",
- [
- { content: "older message", role: "assistant" },
- {
- content: "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2),
- kind: "context.state",
- role: "user",
- },
- ],
- "context.state",
- ],
- ] as const)("keeps a truncated %s message when it alone exceeds the cap", (_, messages, kind) => {
- const attribute = genAiInputMessagesAttribute(messages);
-
- expect(attribute).toBeDefined();
- expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
- const parsed = JSON.parse(attribute!) as Array<Record<string, unknown>>;
- expect(parsed).toHaveLength(1);
- expect(parsed[0]).toMatchObject({
- parts: [{ content: expect.stringMatching(/… \[truncated\]$/u), type: "text" }],
- role: "user",
- });
- expect(parsed[0]?.kind).toBe(kind);
- expect(attribute).not.toContain("older message");
- });
-
- it("formats model input, output, and system instructions for inspectors", () => {
- expect(
- genAiInputMessagesAttribute([
- { content: "hello", kind: "user", role: "user" },
- {
- content: [
- {
- input: { message: "echo" },
- toolCallId: "call-1",
- toolName: "delegate",
- type: "tool-call",
- },
- ],
- role: "assistant",
- },
- ]),
- ).toBe(
- '[{"kind":"user","parts":[{"content":"hello","type":"text"}],"role":"user"},{"parts":[{"arguments":{"message":"echo"},"id":"call-1","name":"delegate","type":"tool_call"}],"role":"assistant"}]',
- );
- expect(genAiSystemInstructionsAttribute("Be concise.")).toBe(
- '[{"content":"Be concise.","type":"text"}]',
- );
- expect(
- genAiOutputMessagesAttribute(
- [
- { text: "Working.", type: "text" },
- {
- callId: "call-1",
- input: { message: "echo" },
- toolName: "delegate",
- type: "tool-call",
- },
- ],
- "tool-calls",
- ),
- ).toBe(
- '[{"finish_reason":"tool_call","parts":[{"content":"Working.","type":"text"},{"arguments":{"message":"echo"},"id":"call-1","name":"delegate","type":"tool_call"}],"role":"assistant"}]',
- );
- });
-
- it.each([
- ["near the limit", "line of instructions\n".repeat(1500)],
- ["over the limit", "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)],
- ["escape-heavy", "\\\n".repeat(9000)],
- ])("keeps %s system instructions as valid capped JSON", (_, instructions) => {
- const attribute = genAiSystemInstructionsAttribute(instructions);
-
- expect(attribute).toBeDefined();
- expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
- const parsed = JSON.parse(attribute!) as Array<Record<string, unknown>>;
- expect(parsed).toHaveLength(1);
- expect(parsed[0]).toMatchObject({ type: "text" });
- expect(parsed[0]?.content).toEqual(expect.stringContaining("… [truncated]"));
- });
-
- it("marks a message truncated when oversized non-text parts are omitted", () => {
- const attribute = genAiInputMessagesAttribute([
- {
- content: [
- { text: "partial assistant answer", type: "text" },
- {
- input: { payload: "x".repeat(CONTENT_ATTRIBUTE_LIMIT) },
- toolCallId: "call-1",
- toolName: "delegate",
- type: "tool-call",
- },
- ],
- role: "assistant",
- },
- ]);
-
- expect(attribute).toBeDefined();
- expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
- const parsed = JSON.parse(attribute!) as Array<Record<string, unknown>>;
- expect(parsed).toEqual([
- {
- parts: [{ content: "partial assistant answer… [truncated]", type: "text" }],
- role: "assistant",
- },
- ]);
- });
-});
-
-describe("genAiToolDefinitionsAttribute", () => {
- it("maps AI SDK input schemas to GenAI parameters", () => {
- expect(
- genAiToolDefinitionsAttribute([
- {
- description: "Get the current weather.",
- inputSchema: {
- properties: { city: { type: "string" } },
- required: ["city"],
- type: "object",
- },
- name: "get_weather",
- },
- ]),
- ).toBe(
- '[{"name":"get_weather","description":"Get the current weather.","parameters":{"properties":{"city":{"type":"string"}},"required":["city"],"type":"object"}}]',
- );
- });
-
- it("drops trailing definitions to keep oversized content valid JSON", () => {
- const attribute = genAiToolDefinitionsAttribute([
- { inputSchema: { type: "object" }, name: "first" },
- {
- description: "x".repeat(CONTENT_ATTRIBUTE_LIMIT),
- inputSchema: { type: "object" },
- name: "oversized",
- },
- { inputSchema: { type: "object" }, name: "last" },
- ]);
-
- expect(attribute).toBeDefined();
- expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
- expect(JSON.parse(attribute!)).toEqual([{ name: "first", parameters: { type: "object" } }]);
- });
-
- it("returns undefined when no tool definitions are available", () => {
- expect(genAiToolDefinitionsAttribute([])).toBeUndefined();
- expect(genAiToolDefinitionsAttribute(undefined)).toBeUndefined();
- });
-});
-
-describe("toolResultsContentAttribute", () => {
- it("returns the full payload when it fits", () => {
- const json = toolResultsContentAttribute([
- { input: { query: "weather" }, output: { results: ["sunny"] }, toolName: "web_search" },
- ]);
- expect(json).toBe(
- '[{"input":{"query":"weather"},"output":{"results":["sunny"]},"toolName":"web_search"}]',
- );
- });
-
- it("returns undefined for no results", () => {
- expect(toolResultsContentAttribute([])).toBeUndefined();
- });
-
- it("keeps oversized payloads valid JSON by capping entry text", () => {
- const json = toolResultsContentAttribute([
- {
- input: { query: "weather" },
- output: { excerpts: "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2) },
- toolName: "web_search",
- },
- ]);
- expect(json).toBeDefined();
- expect(json!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
- const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;
- expect(parsed[0]!.toolName).toBe("web_search");
- expect(parsed[0]!.input).toBe('{"query":"weather"}');
- expect(parsed[0]!.output).toContain("… [truncated]");
- });
-
- it("preserves the error key through truncation", () => {
- const json = toolResultsContentAttribute([
- {
- error: `quota exceeded ${"y".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)}`,
- input: { query: "again" },
- toolName: "web_search",
- },
- ]);
- const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;
- expect(parsed[0]!.error).toContain("quota exceeded");
- expect(parsed[0]!.error).toContain("… [truncated]");
- expect(parsed[0]!.output).toBeUndefined();
- });
-
- it("splits the budget across many oversized entries", () => {
- const entries = Array.from({ length: 8 }, (_, index) => ({
- input: { index },
- output: "z".repeat(CONTENT_ATTRIBUTE_LIMIT),
- toolName: `tool_${index}`,
- }));
- const json = toolResultsContentAttribute(entries);
- expect(json).toBeDefined();
- expect(json!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
- const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;
- expect(parsed).toHaveLength(8);
- expect(parsed[7]!.toolName).toBe("tool_7");
- });
-});
diff --git a/packages/eve/src/tracing/agent-otel-content.ts b/packages/eve/src/tracing/agent-otel-content.ts
deleted file mode 100644
index 4810380c1..000000000
--- a/packages/eve/src/tracing/agent-otel-content.ts
+++ /dev/null
@@ -1,282 +0,0 @@
-import { isUserMessageKind } from "#harness/messages.js";
-
-/**
- * Serializes model and tool payloads into span content attributes for the
- * local trace viewer: prompt messages, the system prompt, responses, and
- * tool arguments/results. Everything is capped so a giant payload cannot
- * bloat a span segment past the reader's per-file limit, and message
- * payloads stay valid JSON through every truncation path.
- */
-
-/** Content attributes are capped so a giant payload cannot bloat a span segment. */
-export const CONTENT_ATTRIBUTE_LIMIT = 32 * 1024;
-
-/** Serializes one payload value as capped JSON. */
-export function contentAttribute(value: unknown): string | undefined {
- if (value === undefined) return undefined;
- let json: string | undefined;
- try {
- json = JSON.stringify(value);
- } catch {
- return undefined;
- }
- if (json === undefined) return undefined;
- return textContentAttribute(json);
-}
-
-/** Serializes model messages using the OpenTelemetry GenAI message schema. */
-export function genAiInputMessagesAttribute(messages: unknown): string | undefined {
- if (!Array.isArray(messages)) return undefined;
- const formatted = messages.flatMap((message) => {
- if (!isRecord(message) || message.role === "system" || typeof message.role !== "string") {
- return [];
- }
- const parts = semanticParts(message.content);
- if (parts.length === 0) return [];
- const kind =
- message.role === "user" && isUserMessageKind(message.kind) ? message.kind : undefined;
- return [
- kind === undefined ? { parts, role: message.role } : { kind, parts, role: message.role },
- ];
- });
- for (let start = 0; start < formatted.length; start += 1) {
- const json = semanticJsonAttribute(formatted.slice(start));
- if (json !== undefined) return json;
- }
- return truncateSingleSemanticMessage(formatted) ?? semanticJsonAttribute([]);
-}
-
-/** Serializes the system prompt using the OpenTelemetry GenAI instruction schema. */
-export function genAiSystemInstructionsAttribute(instructions: unknown): string | undefined {
- const text = systemPromptText(instructions);
- if (text === undefined) return undefined;
- return fitSemanticText(text, (content) => [{ content, type: "text" }]);
-}
-
-/** Serializes tool definitions, dropping trailing entries to keep the JSON whole. */
-export function genAiToolDefinitionsAttribute(tools: unknown): string | undefined {
- if (!Array.isArray(tools)) return undefined;
- const definitions = tools.flatMap((tool): Record<string, unknown>[] => {
- if (!isRecord(tool) || typeof tool.name !== "string") return [];
- const definition: Record<string, unknown> = { name: tool.name };
- if (typeof tool.description === "string") definition.description = tool.description;
- if (tool.inputSchema !== undefined) definition.parameters = tool.inputSchema;
- return [definition];
- });
- if (definitions.length === 0) return undefined;
-
- const serialized: string[] = [];
- let length = 2;
- for (const definition of definitions) {
- const json = stringifyContent(definition);
- if (json === undefined) break;
- const nextLength = length + (serialized.length === 0 ? 0 : 1) + json.length;
- if (nextLength > CONTENT_ATTRIBUTE_LIMIT) break;
- serialized.push(json);
- length = nextLength;
- }
- return `[${serialized.join(",")}]`;
-}
-
-/** Serializes one model response using the OpenTelemetry GenAI message schema. */
-export function genAiOutputMessagesAttribute(
- content: readonly unknown[],
- finishReason: string,
-): string | undefined {
- const parts = semanticParts(content);
- return semanticJsonAttribute([
- {
- finish_reason: finishReason === "tool-calls" ? "tool_call" : finishReason,
- parts,
- role: "assistant",
- },
- ]);
-}
-
-/** Serializes memory records using the OpenTelemetry GenAI memory-records schema. */
-export function genAiMemoryRecordsAttribute(records: readonly unknown[]): string | undefined {
- return semanticJsonAttribute(records);
-}
-
-/**
- * Serializes provider-executed tool results, keeping the attribute valid
- * JSON under the cap: when the full payload is too big, each entry's input
- * and output collapse to capped text at progressively smaller budgets
- * instead of cutting the JSON mid-string.
- */
-export function toolResultsContentAttribute(
- results: readonly Record<string, unknown>[],
-): string | undefined {
- if (results.length === 0) return undefined;
- const full = stringifyContent(results);
- if (full !== undefined && full.length <= CONTENT_ATTRIBUTE_LIMIT) return full;
- for (let cap = CONTENT_ATTRIBUTE_LIMIT; cap >= 0; cap = cap >= 256 ? Math.floor(cap / 2) : -1) {
- const json = stringifyContent(results.map((entry) => cappedToolResult(entry, cap)));
- if (json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT) return json;
- }
- return undefined;
-}
-
-function cappedToolResult(entry: Record<string, unknown>, cap: number): Record<string, unknown> {
- const out: Record<string, unknown> = { toolName: entry.toolName };
- for (const key of ["input", "output", "error"]) {
- if (!(key in entry)) continue;
- const value = entry[key];
- const text = typeof value === "string" ? value : (stringifyContent(value) ?? "");
- out[key] = text.length <= cap ? text : `${text.slice(0, cap)}… [truncated]`;
- }
- return out;
-}
-
-/** Normalizes the AI SDK's `instructions` prompt to plain text. */
-function systemPromptText(instructions: unknown): string | undefined {
- if (typeof instructions === "string") return instructions.length === 0 ? undefined : instructions;
- if (!isRecord(instructions) && !Array.isArray(instructions)) return undefined;
- const messages = Array.isArray(instructions) ? instructions : [instructions];
- const texts: string[] = [];
- for (const message of messages) {
- if (!isRecord(message)) continue;
- if (typeof message.content === "string") texts.push(message.content);
- else if (Array.isArray(message.content))
- for (const part of message.content)
- if (isRecord(part) && part.type === "text" && typeof part.text === "string")
- texts.push(part.text);
- }
- const joined = texts.join("\n\n").trim();
- return joined.length === 0 ? undefined : joined;
-}
-
-/** Caps plain text, marking the cut. */
-export function textContentAttribute(text: string): string | undefined {
- if (text.length === 0) return undefined;
- return text.length <= CONTENT_ATTRIBUTE_LIMIT
- ? text
- : `${text.slice(0, CONTENT_ATTRIBUTE_LIMIT)}… [truncated]`;
-}
-
-function stringifyContent(value: unknown): string | undefined {
- try {
- return JSON.stringify(value);
- } catch {
- return undefined;
- }
-}
-
-function semanticJsonAttribute(value: unknown): string | undefined {
- const json = stringifyContent(value);
- return json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT ? json : undefined;
-}
-
-function truncateSingleSemanticMessage(
- messages: readonly Record<string, unknown>[],
-): string | undefined {
- const message = messages.at(-1);
- if (message === undefined || typeof message.role !== "string") return undefined;
- const text = semanticMessageText(message.parts);
- const base =
- typeof message.kind === "string"
- ? { kind: message.kind, role: message.role }
- : { role: message.role };
- return fitSemanticText(
- text,
- (content) => [{ ...base, parts: [{ content, type: "text" }] }],
- true,
- );
-}
-
-function fitSemanticText(
- text: string,
- wrap: (content: string) => unknown,
- alwaysMark = false,
-): string | undefined {
- if (!alwaysMark && text.length <= CONTENT_ATTRIBUTE_LIMIT) {
- const full = semanticJsonAttribute(wrap(text));
- if (full !== undefined) return full;
- }
- for (let length = Math.min(text.length, CONTENT_ATTRIBUTE_LIMIT); length > 0; length -= 256) {
- const json = semanticJsonAttribute(wrap(`${text.slice(0, length)}… [truncated]`));
- if (json !== undefined) return json;
- }
- return semanticJsonAttribute(wrap("… [truncated]"));
-}
-
-function semanticMessageText(parts: unknown): string {
- if (!Array.isArray(parts)) return "";
- let text = "";
- for (const part of parts) {
- if (!isRecord(part) || part.type !== "text" || typeof part.content !== "string") continue;
- const separator = text.length === 0 ? "" : "\n";
- const remaining = CONTENT_ATTRIBUTE_LIMIT + 1 - text.length - separator.length;
- if (remaining <= 0) break;
- text += separator + part.content.slice(0, remaining);
- }
- return text;
-}
-
-function semanticParts(content: unknown): Record<string, unknown>[] {
- if (typeof content === "string") return [{ content, type: "text" }];
- if (!Array.isArray(content)) return [];
- const parts: Record<string, unknown>[] = [];
- for (const part of content) {
- const formatted = semanticPart(part);
- if (formatted !== undefined) parts.push(formatted);
- }
- return parts;
-}
-
-function semanticPart(value: unknown): Record<string, unknown> | undefined {
- if (!isRecord(value) || typeof value.type !== "string") return undefined;
- switch (value.type) {
- case "text":
- case "reasoning": {
- const content = typeof value.text === "string" ? value.text : value.content;
- return typeof content === "string" ? { content, type: value.type } : undefined;
- }
- case "tool-call":
- return {
- arguments: value.input,
- id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,
- name: value.toolName,
- type: "tool_call",
- };
- case "tool-result":
- return {
- id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,
- response: toolResponse(value.output),
- type: "tool_call_response",
- };
- case "tool-error":
- return {
- id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,
- response: value.error instanceof Error ? value.error.message : value.error,
- type: "tool_call_response",
- };
- default:
- return { type: value.type };
- }
-}
-
-function toolResponse(output: unknown): unknown {
- if (!isRecord(output)) return output;
- if (
- (output.type === "text" ||
- output.type === "error-text" ||
- output.type === "json" ||
- output.type === "error-json") &&
- "value" in output
- ) {
- return output.value;
- }
- if (output.type === "execution-denied") {
- return { denied: true, reason: output.reason };
- }
- return output;
-}
-
-function stringValue(value: unknown): string | undefined {
- return typeof value === "string" ? value : undefined;
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
- return typeof value === "object" && value !== null && !Array.isArray(value);
-}
diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.ts
deleted file mode 100644
index 98ee7c459..000000000
--- a/packages/eve/src/tracing/agent-otel-provider.ts
+++ /dev/null
@@ -1,706 +0,0 @@
-import {
- ROOT_CONTEXT,
- SpanKind,
- context,
- type Context,
- type Link,
- type Span,
- type SpanContext,
- type Tracer,
- trace,
-} from "#compiled/@opentelemetry/api/index.js";
-
-import { contextStorage } from "#context/container.js";
-import { SessionTraceSeedKey } from "#context/keys.js";
-import { withoutInstrumentationContent } from "#instrumentation/content.js";
-import { instrumentationEventForTraceDecision } from "#instrumentation/content-policy.js";
-import type { AgentTraceStateStore, AgentTurnTraceState } from "#tracing/agent-trace-state.js";
-import {
- contentAttribute,
- genAiInputMessagesAttribute,
- genAiOutputMessagesAttribute,
- genAiSystemInstructionsAttribute,
- genAiToolDefinitionsAttribute,
- textContentAttribute,
- toolResultsContentAttribute,
-} from "#tracing/agent-otel-content.js";
-import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
-import { createAgentApprovalInstrumentation } from "#tracing/agent-approval-instrumentation.js";
-import { createAgentChannelDeliveryInstrumentation } from "#tracing/eve/agent-channel-delivery-instrumentation.js";
-import { createAgentToolInstrumentation } from "#tracing/agent-tool-instrumentation.js";
-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";
-import { markAgentTraceContext } from "#tracing/eve/agent-trace-context.js";
-import { agentTraceIdentityAttributes, traceSessionIdOf } from "#tracing/agent-otel-attributes.js";
-import * as runtimeAttributes from "#tracing/agent-otel-runtime-context.js";
-import { createAgentMemoryInstrumentation } from "#tracing/agent-memory-instrumentation.js";
-import {
- readGatewayCost,
- setAgentInvocationUsage,
- setAgentUsage,
- setGenAiUsage,
-} from "#tracing/agent-otel-usage.js";
-import { createAgentOtelSessionContext } from "#tracing/agent-otel-session-context.js";
-import {
- rememberTurnInputMessages,
- rememberTurnOutputMessages,
-} from "#tracing/agent-otel-turn-state.js";
-import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";
-import { isSampledTrace, resolveTracePolicyDecision } from "#tracing/sampled-trace.js";
-import {
- applyLiveDeliveryAudienceCeiling,
- resolveForwardedTraceSeed,
-} from "#shared/forwarded-trace-policy.js";
-import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";
-import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";
-import { suppressTracing } from "#tracing/suppress-tracing.js";
-import { normalizeChannelAudience, type ChannelAudience } from "#shared/channel-audience.js";
-import type {
- InstrumentationEvent,
- InstrumentationStepAttemptMetadataEvent,
- InstrumentationAttemptScope,
- InstrumentationStepAttemptStartedEvent,
- InstrumentationStepAttemptTerminalEvent,
- InstrumentationContextRunner,
- InstrumentationModelCallTerminalEvent,
- InstrumentationModelCallStartedEvent,
- InstrumentationProviderDefinition,
- InstrumentationSessionStartedEvent,
- InstrumentationTraceSeed,
- InstrumentationSessionTransitionEvent,
- InstrumentationTurnStartedEvent,
- InstrumentationTurnTerminalEvent,
-} from "#instrumentation/lifecycle.js";
-import { attemptIdempotencyKey } from "#instrumentation/lifecycle.js";
-import {
- AGENT_SPAN_NAMES,
- agentInvocationSpanName,
- modelSpanName,
- type AgentSamplingOperation,
-} from "#tracing/agent-span-contract.js";
-import { withErrorContent } from "#tracing/error-content-context.js";
-import { withAgentToolContentPolicy } from "#tracing/agent-tool-span-context.js";
-import { withOperationConversation } from "#tracing/eve/conversation-context.js";
-import { recordAgentSpanError as recordError } from "#tracing/agent-span-error.js";
-import { resolveInstrumentationEnvironment } from "#internal/application/dev-environment.js";
-import type { ConversationEnvironment } from "#shared/conversation-context.js";
-
-type SpanState = { readonly context: Context; readonly span: Span };
-
-export interface AgentOtelInstrumentationInput {
- readonly environment?: ConversationEnvironment;
- /** Whether any destination records model and tool inputs. */
- readonly recordInputs?: boolean;
- /** Whether any destination records model and tool outputs. */
- readonly recordOutputs?: boolean;
- readonly frameworkVersion: string;
- readonly idGenerator: AgentSpanIdGenerator;
- readonly samplesTrace?: (traceId: string, operation?: AgentSamplingOperation) => boolean;
- readonly stateStore: AgentTraceStateStore;
- readonly tracer: Tracer;
- readonly tracePolicy?: TraceCapturePolicy;
-}
-
-/** OTel event definition and its trusted framework context runner. */
-interface AgentOtelInstrumentation {
- readonly hook: InstrumentationProviderDefinition;
- readonly prepareSessionTrace: (
- event: InstrumentationSessionStartedEvent,
- ) => Promise<InstrumentationTraceSeed>;
- readonly prepareTurnTrace: (
- event: InstrumentationTurnStartedEvent,
- ) => Promise<InstrumentationTraceSeed>;
- readonly runInContext: InstrumentationContextRunner;
-}
-
-/** Creates OTel instrumentation for eve's structural and GenAI spans. */
-export function createAgentOtelInstrumentation(
- input: AgentOtelInstrumentationInput,
-): AgentOtelInstrumentation {
- const environment = input.environment ?? resolveInstrumentationEnvironment();
- const recordInputs = input.recordInputs ?? false;
- const recordOutputs = input.recordOutputs ?? false;
- const executionContexts = new WeakMap<InstrumentationAttemptScope, Map<string, Context>>();
- const attemptScopes = new Map<string, InstrumentationAttemptScope>();
- // A lost serverless worker retries the whole turn step from entry.
- const steps = new WeakMap<InstrumentationAttemptScope, SpanState>();
- const modelSpans = new WeakMap<InstrumentationAttemptScope, Map<string, SpanState>>();
- const tools = createAgentToolInstrumentation({
- frameworkVersion: input.frameworkVersion,
- idGenerator: input.idGenerator,
- recordInputs,
- recordOutputs,
- resolveTraceContext: async (event) => {
- const turn = await input.stateStore.getTurn(event.scope.sessionId, event.scope.turnId);
- return turn?.context;
- },
- stateStore: input.stateStore,
- resolveFallback: (event) => {
- const scope = attemptScopes.get(event.scope.attemptId) ?? event.scope;
- const step = steps.get(scope);
- return step === undefined
- ? undefined
- : { context: step.context, spanContext: step.span.spanContext() };
- },
- tracer: input.tracer,
- });
- const approvals = createAgentApprovalInstrumentation({
- actionContextFor: tools.dispatchContextFor,
- frameworkVersion: input.frameworkVersion,
- idGenerator: input.idGenerator,
- tracer: input.tracer,
- });
- const memory = createAgentMemoryInstrumentation({ ...input, environment });
- const { prepareSessionTrace, prepareTurnTrace } = createAgentOtelSessionContext({
- ...input,
- environment,
- });
-
- const projectEvent = async (event: InstrumentationEvent): Promise<InstrumentationEvent> => {
- const session = await input.stateStore.getSession(sessionIdForEvent(event));
- const audience = audienceForEvent(event, session?.channelAudience);
- const eventSeed = "traceSeed" in event ? event.traceSeed : undefined;
- const contextSeed = contextStorage.getStore()?.get(SessionTraceSeedKey);
- const contextTraceState = resolveForwardedTraceSeed(contextSeed);
- const eventTraceState = resolveForwardedTraceSeed(
- eventSeed,
- contextTraceState?.forwardedTracePolicy,
- );
- const decisionForTrace = (trace: { readonly traceFlags: number } | undefined) =>
- trace === undefined
- ? undefined
- : resolveTracePolicyDecision(isSampledTrace(trace), { audience, environment });
- const decision =
- eventTraceState?.decision ??
- contextTraceState?.decision ??
- readInstrumentationDecision(session?.decision) ??
- decisionForTrace(eventSeed) ??
- decisionForTrace(contextSeed) ??
- decisionForTrace(session?.context);
- if (decision === undefined) return withoutInstrumentationContent(event);
- const normalizedEvent =
- eventTraceState === undefined || !("traceSeed" in event) || event.traceSeed === undefined
- ? event
- : {
- ...event,
- traceSeed: {
- ...event.traceSeed,
- decision: eventTraceState.decision,
- traceFlags: eventTraceState.traceFlags,
- },
- };
- return instrumentationEventForTraceDecision(
- normalizedEvent,
- applyLiveDeliveryAudienceCeiling(
- decision.action === "drop"
- ? decision
- : {
- action: "record",
- recordInputs: recordInputs && decision.recordInputs,
- recordOutputs: recordOutputs && decision.recordOutputs,
- },
- audience,
- eventTraceState?.forwardedTracePolicy ?? contextTraceState?.forwardedTracePolicy,
- environment,
- ),
- { audience, environment },
- { applyAudienceCeiling: false },
- );
- };
-
- const onSessionStarted = async (event: InstrumentationSessionStartedEvent): Promise<void> => {
- await prepareSessionTrace(event);
- };
-
- const onTurnStarted = async (event: InstrumentationTurnStartedEvent): Promise<void> => {
- await prepareTurnTrace(event);
- };
-
- const onStepStarted = async (event: InstrumentationStepAttemptStartedEvent): Promise<void> => {
- const turn = await input.stateStore.getTurn(event.scope.sessionId, event.scope.turnId);
- if (turn === undefined || !isSampledTrace(turn.context)) return;
- const session = await input.stateStore.getSession(event.scope.sessionId);
- const turnContext = withChannelAudience(
- contextFromSpanContext(turn.context),
- event.scope.channelAudience,
- );
- const activeSpanContext = trace.getSpan(context.active())?.spanContext();
- const stepSpan = input.idGenerator.withSpanId(
- input.idGenerator.deriveSpanId(attemptIdempotencyKey(event.scope)),
- () =>
- input.tracer.startSpan(
- AGENT_SPAN_NAMES.step,
- {
- attributes: runtimeAttributes.agentStepAttributes({
- event,
- frameworkVersion: input.frameworkVersion,
- session,
- turn,
- }),
- links:
- activeSpanContext === undefined || activeSpanContext.traceId === turn.context.traceId
- ? undefined
- : [
- {
- attributes: { "eve.link.type": "workflow.delivery" },
- context: activeSpanContext,
- },
- ],
- },
- turnContext,
- ),
- );
- stepSpan.addEvent("step.started");
- const stepContext = trace.setSpan(turnContext, stepSpan);
- steps.set(event.scope, { context: stepContext, span: stepSpan });
- attemptScopes.set(event.scope.attemptId, event.scope);
- };
-
- const onStepTerminal = async (event: InstrumentationStepAttemptTerminalEvent): Promise<void> => {
- const scope = attemptScopes.get(event.scope.attemptId) ?? event.scope;
- executionContexts.delete(scope);
- drainOpenSpans({ ...event, scope });
- tools.drain(
- event.scope.attemptId,
- event.type === "step.attempt.failed" ? { error: event.error } : undefined,
- );
- if (event.type === "step.attempt.failed") {
- await tools.failForAttempt(scope, event.error);
- }
- attemptScopes.delete(event.scope.attemptId);
- const attempt = steps.get(scope);
- if (attempt === undefined) return;
- attempt.span.addEvent(
- event.type === "step.attempt.completed" ? "step.completed" : "step.failed",
- );
- if (event.type === "step.attempt.failed") {
- recordError(attempt.span, event.error);
- }
- attempt.span.end();
- steps.delete(scope);
- };
-
- const onTurnTerminal = async (event: InstrumentationTurnTerminalEvent): Promise<void> => {
- await input.stateStore.updateTurn(event.sessionId, event.turnId, (turn) => ({
- ...turn,
- terminal:
- event.type === "turn.failed"
- ? { error: event.error, type: event.type }
- : { type: event.type },
- }));
- };
-
- const onSessionTransition = async (
- event: InstrumentationSessionTransitionEvent,
- ): Promise<void> => {
- if (event.type === "session.failed" && event.turnId !== undefined) {
- await input.stateStore.updateTurn(event.sessionId, event.turnId, (turn) => ({
- ...turn,
- terminal: turn.terminal ?? { error: event.error, type: "turn.failed" },
- }));
- }
- if (event.turnId !== undefined) {
- const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);
- if (turn !== undefined) {
- const session = await input.stateStore.getSession(event.sessionId);
- if (isSampledTrace(turn.context)) {
- const agentName = session?.agentName ?? turn.subagentName;
- const parentContext = withChannelAudience(
- turn.caller === undefined || turn.caller.isRemote === true
- ? ROOT_CONTEXT
- : contextFromSpanContext(turn.caller),
- session?.channelAudience,
- );
- const startSpan = () =>
- input.tracer.startSpan(
- agentInvocationSpanName(agentName),
- {
- attributes: runtimeAttributes.agentActivationAttributes({
- agentName,
- frameworkVersion: input.frameworkVersion,
- session,
- sessionId: event.sessionId,
- turnId: event.turnId!,
- turn,
- }),
- kind: SpanKind.INTERNAL,
- links: agentActivationLinks(turn),
- root: turn.caller === undefined || turn.caller.isRemote === true,
- startTime: turn.startTimeMs,
- },
- parentContext,
- );
- const span = input.idGenerator.withSpanId(turn.context.spanId, () =>
- input.idGenerator.withTraceId(turn.context.traceId, startSpan),
- );
- setAgentInvocationUsage(span, turn.modelUsage);
- span.addEvent("turn.started", undefined, turn.startTimeMs);
- if (turn.terminal !== undefined) {
- span.setAttribute(
- "agent.turn.outcome",
- turn.terminal.type === "turn.completed"
- ? "completed"
- : turn.terminal.type === "turn.cancelled"
- ? "cancelled"
- : "failed",
- );
- span.addEvent(turn.terminal.type);
- if (turn.terminal.type === "turn.failed") {
- recordError(span, turn.terminal.error);
- }
- }
- span.end();
- }
- await input.stateStore.deleteTurn(event.sessionId, event.turnId);
- }
- }
- // `session.waiting` is not terminal — the session may resume with a new
- // turn that still needs its metadata — so only release session-scoped
- // state on terminal transitions.
- if (event.type === "session.completed" || event.type === "session.failed") {
- await tools.deleteForSession(event.sessionId);
- await input.stateStore.deleteSession(event.sessionId);
- }
- };
-
- const onModelCallStarted = (
- event: InstrumentationModelCallStartedEvent,
- ): void | PromiseLike<void> => {
- const attempt = steps.get(event.scope);
- if (attempt === undefined) return;
- attempt.span.setAttribute("agent.model.id", event.model.modelId);
- const span = input.tracer.startSpan(
- modelSpanName(event.model.modelId),
- {
- attributes: {
- "gen_ai.agent.name": event.scope.functionId,
- "gen_ai.operation.name": "chat",
- "gen_ai.provider.name": event.model.provider,
- "gen_ai.request.model": event.model.modelId,
- ...agentSpanNamingAttributes(modelSpanName(event.model.modelId), "chat"),
- ...agentTraceIdentityAttributes({
- rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,
- traceSessionId: traceSessionIdOf(event.scope),
- sessionId: event.scope.sessionId,
- }),
- ...runtimeAttributes.runtimeContextAttributes(event.runtimeContext),
- },
- kind: SpanKind.CLIENT,
- },
- attempt.context,
- );
- let inputMessagesAttribute: string | undefined;
- if (recordInputs && event.input !== undefined) {
- inputMessagesAttribute = genAiInputMessagesAttribute(event.input.messages);
- if (inputMessagesAttribute !== undefined) {
- span.setAttribute("gen_ai.input.messages", inputMessagesAttribute);
- }
- const genAiSystem = genAiSystemInstructionsAttribute(event.input.instructions);
- if (genAiSystem !== undefined) {
- span.setAttribute("gen_ai.system_instructions", genAiSystem);
- }
- const toolDefinitions = genAiToolDefinitionsAttribute(event.input.tools);
- if (toolDefinitions !== undefined) {
- span.setAttribute("gen_ai.tool.definitions", toolDefinitions);
- }
- }
- const state = { context: trace.setSpan(attempt.context, span), span };
- getExecutionContexts(event.scope).set(event.idempotencyKey, state.context);
- getSpanStates(modelSpans, event.scope).set(event.idempotencyKey, state);
- return rememberTurnInputMessages(input.stateStore, event.scope, inputMessagesAttribute);
- };
-
- const onModelCallTerminal = async (
- event: InstrumentationModelCallTerminalEvent,
- ): Promise<void> => {
- executionContexts.get(event.scope)?.delete(event.idempotencyKey);
- const state = takeSpanState(modelSpans, event.scope, event.idempotencyKey);
- if (state === undefined) return;
- if (event.type === "model.call.failed") {
- recordError(state.span, event.error);
- } else {
- await recordTurnUsage(event);
- setGenAiUsage(state.span, event.usage);
- if (event.responseId !== undefined) {
- state.span.setAttribute("gen_ai.response.id", event.responseId);
- }
- if (event.responseModelId !== undefined) {
- state.span.setAttribute("gen_ai.response.model", event.responseModelId);
- }
- state.span.setAttribute("gen_ai.response.finish_reasons", [event.finishReason]);
- const attempt = steps.get(event.scope);
- if (attempt !== undefined) setAgentUsage(attempt.span, event.usage);
- if (recordOutputs) {
- state.span.setAttribute("ai.response.finish_reason", event.finishReason);
- const content = event.content;
- if (content === undefined) {
- state.span.end();
- return;
- }
- const outputMessages = genAiOutputMessagesAttribute(content, event.finishReason);
- if (outputMessages !== undefined) {
- state.span.setAttribute("gen_ai.output.messages", outputMessages);
- await rememberTurnOutputMessages(input.stateStore, event.scope, outputMessages);
- }
- const reasoning = textContentAttribute(
- content
- .filter((part) => part.type === "reasoning")
- .map((part) => part.text)
- .filter((part) => part.trim().length > 0)
- .join("\n"),
- );
- if (reasoning !== undefined) state.span.setAttribute("ai.response.reasoning", reasoning);
- const text = textContentAttribute(
- content
- .filter((part) => part.type === "text")
- .map((part) => part.text)
- .join(""),
- );
- if (text !== undefined) state.span.setAttribute("ai.response.text", text);
- const toolCalls = content
- .filter((part) => part.type === "tool-call")
- .map((part) => ({ callId: part.callId, input: part.input, toolName: part.toolName }));
- if (toolCalls.length > 0) {
- const json = contentAttribute(toolCalls);
- if (json !== undefined) state.span.setAttribute("ai.response.tool_calls", json);
- }
- // Provider-executed tools (e.g. web_search) run inside the model call,
- // never reach eve's tool loop, and so never get an execute_tool span.
- // Their results only exist as content parts on the model response.
- const toolResults = content
- .filter((part) => part.type === "tool-result" || part.type === "tool-error")
- .map((part) =>
- part.type === "tool-result"
- ? {
- callId: part.callId,
- input: part.input,
- output: part.output,
- toolName: part.toolName,
- }
- : {
- callId: part.callId,
- error: errorText(part.error),
- input: part.input,
- toolName: part.toolName,
- },
- );
- if (toolResults.length > 0) {
- const json = toolResultsContentAttribute(toolResults);
- if (json !== undefined) state.span.setAttribute("ai.response.tool_results", json);
- }
- }
- }
- state.span.end();
- };
-
- const recordTurnUsage = async (
- event: Extract<
- InstrumentationModelCallTerminalEvent,
- { readonly type: "model.call.completed" }
- >,
- ): Promise<void> => {
- if (event.usage.inputTokens === undefined && event.usage.outputTokens === undefined) return;
- // The bridge publishes at most one completion per physical execution.
- // Workflow retries restart from pre-step state, so abandoned additions are
- // not merged; distinct completed retries consumed tokens and count here.
- await input.stateStore.updateTurn(event.scope.sessionId, event.scope.turnId, (turn) => ({
- ...turn,
- modelUsage: {
- inputTokens:
- event.usage.inputTokens === undefined
- ? turn.modelUsage?.inputTokens
- : (turn.modelUsage?.inputTokens ?? 0) + event.usage.inputTokens,
- outputTokens:
- event.usage.outputTokens === undefined
- ? turn.modelUsage?.outputTokens
- : (turn.modelUsage?.outputTokens ?? 0) + event.usage.outputTokens,
- },
- }));
- };
-
- const channelDeliveries = createAgentChannelDeliveryInstrumentation({
- recordInputs,
- stateStore: input.stateStore,
- });
-
- const onStepMetadata = (event: InstrumentationStepAttemptMetadataEvent): void => {
- const attempt = steps.get(event.scope);
- if (attempt === undefined) return;
- // Vercel AI Gateway reports per-call cost in providerMetadata.gateway;
- // attributes exist only when it was actually the gateway serving the call.
- const costAttributes = readGatewayCost(event.providerMetadata);
- if (costAttributes === undefined) return;
- // The vendored OTel Span surface only has singular setAttribute.
- for (const [key, value] of Object.entries(costAttributes)) {
- attempt.span.setAttribute(key, value);
- }
- };
-
- return {
- hook: {
- events: {
- ...channelDeliveries,
- ...approvals,
- ...memory.events,
- "step.attempt.completed": onStepTerminal,
- "step.attempt.failed": onStepTerminal,
- "step.attempt.metadata": onStepMetadata,
- "step.attempt.started": onStepStarted,
- "model.call.completed": onModelCallTerminal,
- "model.call.failed": onModelCallTerminal,
- "model.call.started": onModelCallStarted,
- "session.completed": onSessionTransition,
- "session.failed": onSessionTransition,
- "session.started": onSessionStarted,
- "session.waiting": onSessionTransition,
- ...tools.events,
- "turn.cancelled": onTurnTerminal,
- "turn.completed": onTurnTerminal,
- "turn.failed": onTurnTerminal,
- "turn.started": onTurnStarted,
- },
- name: "eve.otel",
- projectEvent,
- tracePolicy: () => ({ emit: true, recordInputs, recordOutputs }),
- },
- prepareSessionTrace,
- prepareTurnTrace,
- async runInContext(operation, execute) {
- if (operation.type === "memory.operation") return memory.runInContext(operation, execute);
- const scope = attemptScopes.get(operation.scope.attemptId) ?? operation.scope;
- const contexts = executionContexts.get(scope);
- let parent =
- operation.type === "model.call"
- ? contexts?.get(operation.idempotencyKey)
- : tools.contextFor(operation.scope.attemptId, operation.idempotencyKey);
- if (parent === undefined) {
- const turn = await input.stateStore.getTurn(
- operation.scope.sessionId,
- operation.scope.turnId,
- );
- if (turn !== undefined) {
- parent = withChannelAudience(
- contextFromSpanContext(turn.context),
- operation.scope.channelAudience,
- );
- if (!isSampledTrace(turn.context)) parent = suppressTracing(parent);
- }
- }
- const session = await input.stateStore.getSession(operation.scope.sessionId);
- const seed = resolveForwardedTraceSeed(contextStorage.getStore()?.get(SessionTraceSeedKey));
- const decision = seed?.decision ?? session?.decision;
- const effective =
- decision === undefined
- ? undefined
- : applyLiveDeliveryAudienceCeiling(
- decision,
- normalizeChannelAudience(operation.scope.channelAudience),
- seed?.forwardedTracePolicy,
- environment,
- );
- const toolContentPolicy = {
- recordInputs: recordInputs && effective?.action === "record" && effective.recordInputs,
- recordOutputs: recordOutputs && effective?.action === "record" && effective.recordOutputs,
- };
- const run = (executionParent: Context | undefined) => {
- const activeParent = executionParent ?? parent;
- if (activeParent === undefined) return execute();
- const withErrorPolicy = withErrorContent(activeParent, toolContentPolicy.recordOutputs);
- const operationContext =
- operation.type === "tool.call"
- ? withAgentToolContentPolicy(withErrorPolicy, toolContentPolicy)
- : withErrorPolicy;
- return context.with(
- markAgentTraceContext(withOperationConversation(operationContext, operation.scope)),
- execute,
- );
- };
- return operation.type === "tool.call"
- ? tools.runInContext(
- { ...operation, input: toolContentPolicy.recordInputs ? operation.input : undefined },
- run,
- toolContentPolicy.recordOutputs,
- )
- : run(undefined);
- },
- };
-
- function getExecutionContexts(scope: InstrumentationAttemptScope): Map<string, Context> {
- let state = executionContexts.get(scope);
- if (state === undefined) {
- state = new Map();
- executionContexts.set(scope, state);
- }
- return state;
- }
-
- function drainOpenSpans(event: InstrumentationStepAttemptTerminalEvent): void {
- for (const state of modelSpans.get(event.scope)?.values() ?? []) {
- if (event.type === "step.attempt.failed") recordError(state.span, event.error);
- state.span.end();
- }
- modelSpans.delete(event.scope);
- }
-}
-
-function agentActivationLinks(turn: AgentTurnTraceState): Link[] | undefined {
- const links: Link[] = [];
- if (turn.caller?.isRemote === true) {
- links.push({ context: turn.caller, attributes: { "eve.link.type": "agent.dispatch" } });
- }
- if (turn.channelDelivery?.requestTraceContext !== undefined) {
- links.push({
- context: turn.channelDelivery.requestTraceContext,
- attributes: { "eve.link.type": "channel.request" },
- });
- }
- return links.length === 0 ? undefined : links;
-}
-
-function sessionIdForEvent(event: InstrumentationEvent): string {
- return "scope" in event ? event.scope.sessionId : event.sessionId;
-}
-
-function audienceForEvent(
- event: InstrumentationEvent,
- sessionAudience: ChannelAudience | undefined,
-): ChannelAudience {
- if ("delivery" in event) return normalizeChannelAudience(event.delivery.channelAudience);
- if ("scope" in event && event.scope.channelAudience !== undefined) {
- return normalizeChannelAudience(event.scope.channelAudience);
- }
- if (event.type === "session.started") {
- return normalizeChannelAudience(event.channelAudience);
- }
- return normalizeChannelAudience(sessionAudience);
-}
-
-function getSpanStates<T>(
- spans: WeakMap<InstrumentationAttemptScope, Map<string, T>>,
- scope: InstrumentationAttemptScope,
-): Map<string, T> {
- let scoped = spans.get(scope);
- if (scoped === undefined) {
- scoped = new Map();
- spans.set(scope, scoped);
- }
- return scoped;
-}
-
-function takeSpanState<T>(
- spans: WeakMap<InstrumentationAttemptScope, Map<string, T>>,
- scope: InstrumentationAttemptScope,
- id: string,
-): T | undefined {
- const scoped = spans.get(scope);
- const state = scoped?.get(id);
- if (scoped === undefined) return undefined;
- scoped.delete(id);
- if (scoped.size === 0) spans.delete(scope);
- return state;
-}
-
-function contextFromSpanContext(spanContext: SpanContext): Context {
- return trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(spanContext));
-}
-
-const errorText = (error: unknown): unknown => (error instanceof Error ? error.message : error);
diff --git a/packages/eve/src/tracing/agent-otel-runtime-context.test.ts b/packages/eve/src/tracing/agent-otel-runtime-context.test.ts
deleted file mode 100644
index 6c6a93b3b..000000000
--- a/packages/eve/src/tracing/agent-otel-runtime-context.test.ts
+++ /dev/null
@@ -1,187 +0,0 @@
-import { describe, expect, it } from "vitest";
-
-import { agentActivationAttributes } from "#tracing/agent-otel-runtime-context.js";
-import type { AgentSessionTraceState, AgentTurnTraceState } from "#tracing/agent-trace-state.js";
-
-const context = {
- spanId: "1".repeat(16),
- traceFlags: 1,
- traceId: "2".repeat(32),
-};
-
-function turn(overrides: Partial<AgentTurnTraceState> = {}): AgentTurnTraceState {
- return {
- context,
- rootSessionId: "root-session",
- traceSessionId: "root-session",
- sequence: 0,
- startTimeMs: 1,
- ...overrides,
- };
-}
-
-function session(overrides: Partial<AgentSessionTraceState> = {}): AgentSessionTraceState {
- return {
- agentName: "general",
- channelAudience: "public",
- channelKind: "slack",
- context,
- decision: { action: "record", recordInputs: true, recordOutputs: true },
- rootSessionId: "root-session",
- traceSessionId: "root-session",
- title: "Call the general agent",
- ...overrides,
- };
-}
-
-describe("agentActivationAttributes", () => {
- it.each([
- {
- name: "root",
- sessionId: "root-session",
- traceSessionId: "root-session",
- parent: undefined,
- ownsMetadata: true,
- },
- {
- name: "remote",
- sessionId: "remote-session",
- traceSessionId: "remote-session",
- parent: "caller",
- ownsMetadata: true,
- },
- {
- name: "remote local child",
- sessionId: "child",
- traceSessionId: "remote-session",
- parent: "remote-session",
- ownsMetadata: false,
- },
- {
- name: "local child",
- sessionId: "child",
- traceSessionId: "root-session",
- parent: "root-session",
- ownsMetadata: false,
- },
- ])(
- "derives $name metadata ownership from explicit session identity",
- ({ sessionId, traceSessionId, parent, ownsMetadata }) => {
- const attributes = agentActivationAttributes({
- frameworkVersion: "test",
- session: session({ traceSessionId, channelKind: "http", title: "Task" }),
- sessionId,
- turnId: "turn_0",
- turn: turn({
- traceSessionId,
- parentLineage:
- parent === undefined
- ? undefined
- : { sessionId: parent, callId: "call", turnId: "turn_0" },
- }),
- });
- expect(attributes["agent.channel.kind"]).toBe(ownsMetadata ? "http" : undefined);
- expect(attributes["agent.session.origin"]).toBe(ownsMetadata ? "channel" : undefined);
- expect(attributes["agent.session.title"]).toBe(ownsMetadata ? "Task" : undefined);
- expect(attributes["agent.run.type"]).toBe(parent === undefined ? "session" : "subagent");
- expect(attributes["agent.parent_run.id"]).toBe(parent);
- expect(attributes["agent.parent_call.id"]).toBe(parent === undefined ? undefined : "call");
- },
- );
- it("emits root-session inventory and trace-policy metadata", () => {
- expect(
- agentActivationAttributes({
- agentName: "general",
- frameworkVersion: "test",
- session: session(),
- sessionId: "root-session",
- turn: turn(),
- turnId: "turn_0",
- }),
- ).toMatchObject({
- "agent.channel.audience": "public",
- "agent.channel.kind": "slack",
- "agent.run.type": "session",
- "agent.session.origin": "channel",
- "agent.session.title": "Call the general agent",
- "agent.trace.content.input": true,
- "agent.trace.content.output": true,
- });
- });
-
- it("keeps root-only provenance and title off delegated activations", () => {
- const attributes = agentActivationAttributes({
- agentName: "general",
- frameworkVersion: "test",
- session: session({
- channelAudience: "private",
- decision: { action: "record", recordInputs: false, recordOutputs: true },
- scheduleId: "daily-report",
- }),
- sessionId: "child-session",
- turn: turn({
- parentLineage: {
- callId: "call-1",
- sessionId: "parent-session",
- subagentName: "general",
- turnId: "turn_0",
- },
- }),
- turnId: "turn_0",
- });
-
- expect(attributes).toMatchObject({
- "agent.channel.audience": "private",
- "agent.run.type": "subagent",
- "agent.trace.content.input": false,
- "agent.trace.content.output": true,
- });
- expect(attributes["agent.channel.kind"]).toBeUndefined();
- expect(attributes["agent.schedule.id"]).toBeUndefined();
- expect(attributes["agent.session.origin"]).toBeUndefined();
- expect(attributes["agent.session.title"]).toBeUndefined();
- });
-
- it("normalizes a raw channel type when the normalized kind is absent", () => {
- const attributes = agentActivationAttributes({
- agentName: "general",
- frameworkVersion: "test",
- session: session({ channelKind: undefined, channelType: "slack" }),
- sessionId: "root-session",
- turn: turn(),
- turnId: "turn_0",
- });
-
- expect(attributes["agent.channel.kind"]).toBe("unknown");
- });
-
- it("keeps actual channel delivery metadata on delegated activations", () => {
- const attributes = agentActivationAttributes({
- agentName: "general",
- frameworkVersion: "test",
- session: session(),
- sessionId: "child-session",
- turn: turn({
- channelDelivery: {
- channelKind: "channel:slack",
- channelName: "slack",
- deliveryId: "delivery-1",
- },
- parentLineage: {
- callId: "call-1",
- sessionId: "parent-session",
- subagentName: "general",
- turnId: "turn_0",
- },
- }),
- turnId: "turn_1",
- });
-
- expect(attributes).toMatchObject({
- "agent.channel.delivery.id": "delivery-1",
- "agent.channel.kind": "channel:slack",
- "agent.channel.name": "slack",
- "agent.run.type": "subagent",
- });
- });
-});
diff --git a/packages/eve/src/tracing/agent-otel-runtime-context.ts b/packages/eve/src/tracing/agent-otel-runtime-context.ts
deleted file mode 100644
index 40055a224..000000000
--- a/packages/eve/src/tracing/agent-otel-runtime-context.ts
+++ /dev/null
@@ -1,183 +0,0 @@
-import type { AgentSessionTraceState, AgentTurnTraceState } from "#tracing/agent-trace-state.js";
-import type { InstrumentationStepAttemptStartedEvent } from "#instrumentation/lifecycle.js";
-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";
-import { agentInvocationSpanName } from "#tracing/agent-span-contract.js";
-import { agentTraceIdentityAttributes, traceSessionIdOf } from "#tracing/agent-otel-attributes.js";
-import { normalizeInstrumentationChannelKind } from "#internal/instrumentation.js";
-
-type SpanAttributePrimitive = string | number | boolean;
-type SpanAttributeValue = SpanAttributePrimitive | SpanAttributePrimitive[];
-
-export function agentActivationAttributes(input: {
- readonly agentName?: string;
- readonly frameworkVersion: string;
- readonly session?: AgentSessionTraceState;
- readonly sessionId: string;
- readonly turnId: string;
- readonly turn: AgentTurnTraceState;
-}): Record<string, string | number | boolean | undefined> {
- const recordsTrace = input.session?.decision?.action === "record";
- const recordsInputs = recordsTrace && input.session?.decision?.recordInputs === true;
- const recordsOutputs = recordsTrace && input.session?.decision?.recordOutputs === true;
- const parentLineage = input.turn.parentLineage ?? input.session?.parentLineage;
- const isSubagent = parentLineage !== undefined;
- const ownsSessionMetadata = !isSubagent || input.turn.traceSessionId === input.sessionId;
- const channelClassification = agentChannelClassificationAttributes(
- input.session,
- input.turn,
- input.sessionId,
- );
- const scheduleId = isSubagent ? undefined : input.session?.scheduleId;
- return {
- "agent.framework.name": "eve",
- "agent.framework.version": input.frameworkVersion,
- "agent.name": input.agentName,
- "agent.channel.audience": input.session?.channelAudience,
- ...agentPrincipalAttributes(input.turn),
- "agent.channel.delivery.id": input.turn.channelDelivery?.deliveryId,
- "agent.channel.delivery.input": input.turn.channelDelivery?.inputAttribute,
- ...channelClassification,
- "agent.channel.name": input.turn.channelDelivery?.channelName,
- "agent.channel.request.id": input.turn.channelDelivery?.requestId,
- "agent.parent_call.id": parentLineage?.callId,
- "agent.parent_run.id": parentLineage?.sessionId,
- "agent.run.type": isSubagent ? "subagent" : "session",
- "agent.schedule.id": scheduleId,
- "agent.session.title": ownsSessionMetadata && recordsInputs ? input.session?.title : undefined,
- "agent.subagent.name": input.turn.subagentName,
- "agent.trace.content.input": recordsInputs,
- "agent.trace.content.output": recordsOutputs,
- "agent.turn.id": input.turnId,
- "agent.turn.sequence": input.turn.sequence,
- "gen_ai.agent.name": input.agentName,
- "gen_ai.input.messages": input.turn.inputMessagesAttribute,
- "gen_ai.operation.name": "invoke_agent",
- "gen_ai.output.messages": input.turn.outputMessagesAttribute,
- ...agentSpanNamingAttributes(agentInvocationSpanName(input.agentName), "invoke_agent"),
- ...agentTraceIdentityAttributes({
- rootSessionId: input.turn.rootSessionId,
- traceSessionId: input.turn.traceSessionId,
- sessionId: input.sessionId,
- }),
- };
-}
-
-export function agentChannelClassificationAttributes(
- session: AgentSessionTraceState | undefined,
- turn: AgentTurnTraceState,
- sessionId: string,
-): {
- readonly "agent.channel.kind": string | undefined;
- readonly "agent.session.origin": string | undefined;
-} {
- const isSubagent = (turn.parentLineage ?? session?.parentLineage) !== undefined;
- const ownsSessionMetadata = !isSubagent || turn.traceSessionId === sessionId;
- const channelKind =
- turn.channelDelivery?.channelKind ??
- (!ownsSessionMetadata
- ? undefined
- : (session?.channelKind ??
- (session?.channelType === undefined
- ? undefined
- : normalizeInstrumentationChannelKind(session.channelType))));
- const origin =
- !ownsSessionMetadata || channelKind === undefined
- ? undefined
- : session?.scheduleId !== undefined
- ? "schedule"
- : "channel";
- return {
- "agent.channel.kind": channelKind,
- "agent.session.origin": origin,
- };
-}
-
-export function agentStepAttributes(input: {
- readonly event: InstrumentationStepAttemptStartedEvent;
- readonly frameworkVersion: string;
- readonly session?: AgentSessionTraceState;
- readonly turn: AgentTurnTraceState;
-}) {
- const { event, session, turn } = input;
- const channelClassification = agentChannelClassificationAttributes(
- session,
- turn,
- event.scope.sessionId,
- );
- return {
- "agent.framework.name": "eve",
- "agent.framework.version": input.frameworkVersion,
- "agent.step.attempt": event.scope.attemptIndex,
- "agent.step.index": event.scope.stepIndex,
- "agent.turn.id": event.scope.turnId,
- "agent.name": event.scope.functionId,
- // Leave an unresolved kind open for a later delivery to classify.
- ...(channelClassification["agent.channel.kind"] === "unknown"
- ? undefined
- : channelClassification),
- "gen_ai.operation.name": "workflow",
- ...agentSpanNamingAttributes("agent.step", "workflow"),
- ...agentTraceIdentityAttributes({
- rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,
- traceSessionId: traceSessionIdOf(event.scope),
- sessionId: event.scope.sessionId,
- }),
- ...runtimeContextAttributes(event.runtimeContext),
- };
-}
-
-export function agentPrincipalAttributes(turn: AgentTurnTraceState): Record<string, string> {
- const attributes: Record<string, string> = {};
- setOptionalAttribute(attributes, "agent.principal.current.id", turn.currentPrincipal?.id);
- setOptionalAttribute(attributes, "agent.principal.current.type", turn.currentPrincipal?.type);
- setOptionalAttribute(attributes, "agent.principal.initiator.id", turn.initiatorPrincipal?.id);
- setOptionalAttribute(attributes, "agent.principal.initiator.type", turn.initiatorPrincipal?.type);
- return attributes;
-}
-
-/** Flattens merged runtime context into AI SDK-compatible span attributes. */
-export function runtimeContextAttributes(
- runtimeContext: Readonly<Record<string, unknown>> | undefined,
-): Record<string, SpanAttributeValue> {
- const attributes: Record<string, SpanAttributeValue> = {};
- if (runtimeContext === undefined) return attributes;
- for (const [key, value] of Object.entries(runtimeContext)) {
- flattenContextAttribute(attributes, `ai.settings.context.${key}`, value);
- }
- return attributes;
-}
-
-function flattenContextAttribute(
- attributes: Record<string, SpanAttributeValue>,
- key: string,
- value: unknown,
-): void {
- if (value == null) return;
- if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
- attributes[key] = value;
- return;
- }
- if (Array.isArray(value)) {
- const primitives = value.filter(
- (entry): entry is SpanAttributePrimitive =>
- typeof entry === "string" || typeof entry === "number" || typeof entry === "boolean",
- );
- if (primitives.length !== value.length) return;
- if (new Set(primitives.map((entry) => typeof entry)).size !== 1) return;
- attributes[key] = primitives;
- return;
- }
- if (typeof value === "object") {
- for (const [nestedKey, nestedValue] of Object.entries(value)) {
- flattenContextAttribute(attributes, `${key}.${nestedKey}`, nestedValue);
- }
- }
-}
-
-function setOptionalAttribute(
- attributes: Record<string, string>,
- key: string,
- value: string | undefined,
-): void {
- if (value !== undefined) attributes[key] = value;
-}
diff --git a/packages/eve/src/tracing/agent-otel-session-context.ts b/packages/eve/src/tracing/agent-otel-session-context.ts
deleted file mode 100644
index 8a360a78c..000000000
--- a/packages/eve/src/tracing/agent-otel-session-context.ts
+++ /dev/null
@@ -1,245 +0,0 @@
-import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";
-
-import {
- type InstrumentationSessionStartedEvent,
- type InstrumentationTraceContext,
- type InstrumentationTraceSeed,
- type InstrumentationTurnStartedEvent,
-} from "#instrumentation/lifecycle.js";
-import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
-import { normalizeChannelAudience } from "#shared/channel-audience.js";
-import type { ChannelAudience } from "#shared/channel-audience.js";
-import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";
-import {
- isSampledTrace,
- resolveTracePolicy,
- resolveTracePolicyDecision,
-} from "#tracing/sampled-trace.js";
-import type { AgentSessionTraceState, AgentTraceStateStore } from "#tracing/agent-trace-state.js";
-import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";
-import {
- agentInvocationSpanName,
- type AgentSamplingOperation,
-} from "#tracing/agent-span-contract.js";
-import { agentActivationAttributes } from "#tracing/agent-otel-runtime-context.js";
-import type { AgentTurnTraceState } from "#tracing/agent-trace-state.js";
-import { applyPrincipalTraceDecision } from "#instrumentation/principal-summary.js";
-import { normalizeInstrumentationChannelKind } from "#internal/instrumentation.js";
-import type { ConversationEnvironment } from "#shared/conversation-context.js";
-import { traceSessionIdOf } from "#tracing/agent-otel-attributes.js";
-
-interface AgentOtelSessionContextInput {
- readonly environment: ConversationEnvironment;
- readonly frameworkVersion: string;
- readonly idGenerator: AgentSpanIdGenerator;
- readonly samplesTrace?: (traceId: string, operation?: AgentSamplingOperation) => boolean;
- readonly stateStore: AgentTraceStateStore;
- readonly tracePolicy?: TraceCapturePolicy;
-}
-
-type SessionMetadata = Omit<InstrumentationSessionStartedEvent, "idempotencyKey" | "type">;
-
-interface AgentOtelSessionContext {
- readonly ensureSessionContext: (event: SessionMetadata) => Promise<AgentSessionTraceState>;
- readonly prepareSessionTrace: (
- event: InstrumentationSessionStartedEvent,
- ) => Promise<InstrumentationTraceSeed>;
- readonly prepareTurnTrace: (
- event: InstrumentationTurnStartedEvent,
- ) => Promise<InstrumentationTraceSeed>;
-}
-
-export function createAgentOtelSessionContext(
- input: AgentOtelSessionContextInput,
-): AgentOtelSessionContext {
- const ensureSessionContext = async (event: SessionMetadata): Promise<AgentSessionTraceState> => {
- let state = await input.stateStore.getSession(event.sessionId);
- if (state === undefined) {
- const channelAudience = normalizeChannelAudience(event.channelAudience);
- const decision = resolveSessionTraceDecision(
- event,
- channelAudience,
- input.environment,
- input.tracePolicy,
- );
- state = {
- agentName: event.agentName,
- channelAudience,
- channelKind: event.channelKind,
- channelType: event.channelType,
- decision,
- context: initialSessionContext(input, event, decision),
- parentLineage: event.parentLineage,
- rootSessionId: event.rootSessionId,
- traceSessionId: traceSessionIdOf(event),
- scheduleId: event.scheduleId,
- title: event.title,
- };
- await input.stateStore.setSession(event.sessionId, state);
- }
- return state;
- };
-
- const prepareSessionTrace = async (
- event: InstrumentationSessionStartedEvent,
- ): Promise<InstrumentationTraceSeed> => {
- const session = await ensureSessionContext(event);
- return portableSpanContext(session.context, session.decision);
- };
-
- const prepareTurnTrace = async (
- event: InstrumentationTurnStartedEvent,
- ): Promise<InstrumentationTraceSeed> => {
- const prepared = await input.stateStore.getTurn(event.sessionId, event.turnId);
- if (prepared !== undefined) {
- const session = await input.stateStore.getSession(event.sessionId);
- return portableSpanContext(prepared.context, session?.decision);
- }
-
- const session = await ensureSessionContext(event);
- const useInitialContext = event.sequence === 0;
- const caller = useInitialContext ? event.parentTraceContext : undefined;
- let turnContext = useInitialContext
- ? { ...session.context, isRemote: false }
- : freshTurnContext(input, event.idempotencyKey, session.decision);
- if (caller !== undefined && !("isRemote" in caller && caller.isRemote === true))
- turnContext = { ...turnContext, traceId: caller.traceId };
- const turn: AgentTurnTraceState = {
- caller: caller === undefined ? undefined : adoptedSpanContext(caller),
- context: turnContext,
- currentPrincipal: applyPrincipalTraceDecision(event.currentPrincipal, session.decision),
- initiatorPrincipal: applyPrincipalTraceDecision(event.initiatorPrincipal, session.decision),
- parentLineage: event.parentLineage ?? session.parentLineage,
- rootSessionId: event.rootSessionId,
- traceSessionId: traceSessionIdOf(event),
- sequence: event.sequence,
- startTimeMs: Date.now(),
- subagentName: (event.parentLineage ?? session.parentLineage)?.subagentName,
- };
- if (
- isSampledTrace(turn.context) &&
- (caller === undefined || ("isRemote" in caller && caller.isRemote === true))
- ) {
- const agentName = session.agentName ?? turn.subagentName;
- const sampled =
- input.samplesTrace?.(turn.context.traceId, {
- name: agentInvocationSpanName(agentName),
- attributes: agentActivationAttributes({
- agentName,
- frameworkVersion: input.frameworkVersion,
- session,
- sessionId: event.sessionId,
- turnId: event.turnId,
- turn,
- }),
- }) ?? true;
- turnContext = { ...turnContext, traceFlags: sampled ? 1 : 0 };
- }
- await input.stateStore.setTurn(event.sessionId, event.turnId, {
- ...turn,
- context: turnContext,
- });
- return portableSpanContext(turnContext, session.decision);
- };
-
- return { ensureSessionContext, prepareSessionTrace, prepareTurnTrace };
-}
-
-function portableSpanContext(
- spanContext: SpanContext,
- decision?: InstrumentationTraceSeed["decision"],
-): InstrumentationTraceSeed {
- return {
- decision,
- spanId: spanContext.spanId,
- traceFlags: spanContext.traceFlags,
- traceId: spanContext.traceId,
- };
-}
-
-function adoptedSpanContext(handed: InstrumentationTraceContext): SpanContext {
- return {
- isRemote: "isRemote" in handed && handed.isRemote === true,
- spanId: handed.spanId,
- traceFlags: handed.traceFlags,
- traceId: handed.traceId,
- };
-}
-
-function initialSessionContext(
- input: AgentOtelSessionContextInput,
- event: SessionMetadata,
- decision: ReturnType<typeof resolveTracePolicy>,
-): SpanContext {
- const handed = event.traceSeed;
- if (handed !== undefined) {
- return {
- ...adoptedSpanContext(handed),
- traceFlags: decision.action === "drop" ? 0 : handed.traceFlags,
- };
- }
- const traceId =
- (event.parentTraceContext !== undefined &&
- "isRemote" in event.parentTraceContext &&
- event.parentTraceContext.isRemote === true
- ? undefined
- : event.parentTraceContext?.traceId) ??
- input.idGenerator.deriveTraceId(`session:${event.sessionId}`);
- const sampled = decision.action === "record";
- return {
- isRemote: false,
- spanId: input.idGenerator.deriveSpanId(`session:${event.sessionId}`),
- traceFlags: sampled ? 1 : 0,
- traceId,
- };
-}
-
-function freshTurnContext(
- input: AgentOtelSessionContextInput,
- idempotencyKey: string,
- decision: AgentSessionTraceState["decision"],
-): SpanContext {
- const traceId = input.idGenerator.deriveTraceId(`turn:${idempotencyKey}`);
- const sampled = decision?.action === "record";
- return {
- isRemote: false,
- spanId: input.idGenerator.deriveSpanId(`turn:${idempotencyKey}`),
- traceFlags: sampled ? 1 : 0,
- traceId,
- };
-}
-
-function resolveSessionTraceDecision(
- event: SessionMetadata,
- audience: ChannelAudience,
- environment: ConversationEnvironment,
- policy: TraceCapturePolicy | undefined,
-): ReturnType<typeof resolveTracePolicy> {
- const content = { audience, environment };
- if (event.parentTraceContext !== undefined && !isSampledTrace(event.parentTraceContext)) {
- return { action: "drop" };
- }
- if (event.traceSeed?.decision !== undefined) {
- return readInstrumentationDecision(event.traceSeed.decision) ?? { action: "drop" };
- }
- if (event.traceSeed !== undefined) {
- return resolveTracePolicyDecision(isSampledTrace(event.traceSeed), content);
- }
- if (event.parentTraceContext !== undefined) {
- return resolveTracePolicyDecision(isSampledTrace(event.parentTraceContext), content);
- }
- if (event.agentName === undefined) {
- return policy === undefined ? resolveTracePolicyDecision(true, content) : { action: "drop" };
- }
- // The tool loop can evaluate the same policy before this first-session
- // preparation path; the persisted decision removes that window on replay.
- return resolveTracePolicy(policy, {
- agentName: event.agentName,
- audience,
- channel: {
- kind: normalizeInstrumentationChannelKind(event.channelKind ?? event.channelType),
- },
- environment,
- principalType: "unknown",
- });
-}
diff --git a/packages/eve/src/tracing/agent-otel-turn-state.test.ts b/packages/eve/src/tracing/agent-otel-turn-state.test.ts
deleted file mode 100644
index b293d33bd..000000000
--- a/packages/eve/src/tracing/agent-otel-turn-state.test.ts
+++ /dev/null
@@ -1,46 +0,0 @@
-import { describe, expect, it } from "vitest";
-
-import {
- rememberTurnInputMessages,
- rememberTurnOutputMessages,
-} from "#tracing/agent-otel-turn-state.js";
-import { InMemoryAgentTraceStateStore } from "#internal/testing/in-memory-agent-trace-state-store.js";
-
-describe("agent OTel turn state", () => {
- it("keeps the first model input and latest model output across a multi-call turn", async () => {
- const store = new InMemoryAgentTraceStateStore();
- store.setTurn("session-1", "turn-1", {
- context: spanContext(),
- rootSessionId: "session-1",
- traceSessionId: "session-1",
- sequence: 0,
- startTimeMs: 1,
- });
- const scope = {
- attemptId: "attempt-1",
- attemptIndex: 0,
- functionId: "agent",
- sessionId: "session-1",
- stepIndex: 0,
- turnId: "turn-1",
- };
-
- await rememberTurnInputMessages(store, scope, "first input");
- await rememberTurnInputMessages(store, scope, "second input");
- await rememberTurnOutputMessages(store, scope, "first output");
- await rememberTurnOutputMessages(store, scope, "final output");
-
- expect(store.getTurn("session-1", "turn-1")).toMatchObject({
- inputMessagesAttribute: "first input",
- outputMessagesAttribute: "final output",
- });
- });
-});
-
-function spanContext() {
- return {
- spanId: "2222222222222222",
- traceFlags: 1,
- traceId: "11111111111111111111111111111111",
- };
-}
diff --git a/packages/eve/src/tracing/agent-otel-turn-state.ts b/packages/eve/src/tracing/agent-otel-turn-state.ts
deleted file mode 100644
index e799fea1c..000000000
--- a/packages/eve/src/tracing/agent-otel-turn-state.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-import type { InstrumentationAttemptScope } from "#instrumentation/lifecycle.js";
-import type { AgentTraceStateStore } from "#tracing/agent-trace-state.js";
-
-/** Keeps the first model input so the turn-level invoke_agent span represents its task. */
-export function rememberTurnInputMessages(
- store: AgentTraceStateStore,
- scope: InstrumentationAttemptScope,
- inputMessagesAttribute: string | undefined,
-): void | PromiseLike<void> {
- if (inputMessagesAttribute === undefined) return;
- return store.updateTurn(scope.sessionId, scope.turnId, (turn) => ({
- ...turn,
- inputMessagesAttribute: turn.inputMessagesAttribute ?? inputMessagesAttribute,
- }));
-}
-
-/** Keeps the latest model output so the completed invoke_agent span represents its result. */
-export function rememberTurnOutputMessages(
- store: AgentTraceStateStore,
- scope: InstrumentationAttemptScope,
- outputMessagesAttribute: string | undefined,
-): void | PromiseLike<void> {
- if (outputMessagesAttribute === undefined) return;
- return store.updateTurn(scope.sessionId, scope.turnId, (turn) => ({
- ...turn,
- outputMessagesAttribute,
- }));
-}
diff --git a/packages/eve/src/tracing/agent-otel-usage.ts b/packages/eve/src/tracing/agent-otel-usage.ts
deleted file mode 100644
index 1c136dacc..000000000
--- a/packages/eve/src/tracing/agent-otel-usage.ts
+++ /dev/null
@@ -1,84 +0,0 @@
-import type { Span } from "#compiled/@opentelemetry/api/index.js";
-
-import type { InstrumentationUsage } from "#instrumentation/lifecycle.js";
-import type { AgentTurnTraceState } from "#tracing/agent-trace-state.js";
-import { AGENT_USAGE_ATTRIBUTES } from "#tracing/agent-span-contract.js";
-
-/** Applies eve's structural token usage attributes to an agent span. */
-export function setAgentUsage(span: Span, usage: InstrumentationUsage): void {
- if (usage.costUsd !== undefined) {
- span.setAttribute(AGENT_USAGE_ATTRIBUTES.costUsd, usage.costUsd);
- }
- if (usage.inputTokens !== undefined) {
- span.setAttribute(AGENT_USAGE_ATTRIBUTES.inputTokens, usage.inputTokens);
- }
- if (usage.outputTokens !== undefined) {
- span.setAttribute(AGENT_USAGE_ATTRIBUTES.outputTokens, usage.outputTokens);
- }
- const details = usage.inputTokenDetails;
- if (details?.cacheReadTokens !== undefined) {
- span.setAttribute(AGENT_USAGE_ATTRIBUTES.cacheReadTokens, details.cacheReadTokens);
- }
- if (details?.cacheWriteTokens !== undefined) {
- span.setAttribute(AGENT_USAGE_ATTRIBUTES.cacheWriteTokens, details.cacheWriteTokens);
- }
-}
-
-/** Applies standard GenAI token usage and retains eve's cost attribute. */
-export function setGenAiUsage(span: Span, usage: InstrumentationUsage): void {
- if (usage.costUsd !== undefined) {
- span.setAttribute(AGENT_USAGE_ATTRIBUTES.costUsd, usage.costUsd);
- }
- if (usage.inputTokens !== undefined) {
- span.setAttribute("gen_ai.usage.input_tokens", usage.inputTokens);
- }
- if (usage.outputTokens !== undefined) {
- span.setAttribute("gen_ai.usage.output_tokens", usage.outputTokens);
- }
- const details = usage.inputTokenDetails;
- if (details?.cacheReadTokens !== undefined) {
- span.setAttribute("gen_ai.usage.cache_read.input_tokens", details.cacheReadTokens);
- }
- if (details?.cacheWriteTokens !== undefined) {
- span.setAttribute("gen_ai.usage.cache_write.input_tokens", details.cacheWriteTokens);
- }
-}
-
-export function setAgentInvocationUsage(
- span: Span,
- modelUsage: AgentTurnTraceState["modelUsage"],
-): void {
- if (modelUsage === undefined) return;
- setGenAiUsage(span, modelUsage);
-}
-
-/** Projects Vercel AI Gateway cost metadata onto GenAI span attributes. */
-export function readGatewayCost(
- providerMetadata: Readonly<Record<string, unknown>>,
-): Record<string, string | number> | undefined {
- const gateway = providerMetadata.gateway;
- if (!isRecord(gateway)) return undefined;
- const attributes: Record<string, string | number> = {};
- const cost = readUsd(gateway.cost);
- if (cost !== undefined) attributes["gen_ai.usage.cost"] = cost;
- const gatewayCost = readUsd(gateway.gatewayCost);
- if (gatewayCost !== undefined) attributes["gen_ai.usage.gateway_cost"] = gatewayCost;
- const inputCost = readUsd(gateway.inputInferenceCost);
- if (inputCost !== undefined) attributes["gen_ai.usage.input_cost"] = inputCost;
- const outputCost = readUsd(gateway.outputInferenceCost);
- if (outputCost !== undefined) attributes["gen_ai.usage.output_cost"] = outputCost;
- if (typeof gateway.generationId === "string" && gateway.generationId.length > 0) {
- attributes["gen_ai.generation.id"] = gateway.generationId;
- }
- return Object.keys(attributes).length === 0 ? undefined : attributes;
-}
-
-function readUsd(value: unknown): number | undefined {
- if (typeof value !== "string") return undefined;
- const parsed = Number(value);
- return Number.isFinite(parsed) ? parsed : undefined;
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
- return typeof value === "object" && value !== null && !Array.isArray(value);
-}
diff --git a/packages/eve/src/tracing/agent-span-contract.ts b/packages/eve/src/tracing/agent-span-contract.ts
deleted file mode 100644
index 917aec7e0..000000000
--- a/packages/eve/src/tracing/agent-span-contract.ts
+++ /dev/null
@@ -1,62 +0,0 @@
-export const AGENT_TRACE_SCHEMA_VERSION = 4;
-
-export const AGENT_SPAN_NAMES = {
- approval: "agent.approval",
- channelRequest: "agent.channel.request",
- step: "agent.step",
-} as const;
-
-export const AGENT_USAGE_ATTRIBUTES = {
- costUsd: "agent.usage.cost_usd",
- inputTokens: "agent.usage.input_tokens",
- outputTokens: "agent.usage.output_tokens",
- cacheReadTokens: "agent.usage.cache_read_tokens",
- cacheWriteTokens: "agent.usage.cache_write_tokens",
-} as const;
-
-export function agentInvocationSpanName(agentName: string | undefined): string {
- return agentName === undefined ? "invoke_agent" : `invoke_agent ${agentName}`;
-}
-
-export function modelSpanName(modelId: string): string {
- return `chat ${modelId}`;
-}
-
-export interface AgentSamplingOperation {
- readonly name: string;
- readonly attributes?: Readonly<Record<string, string | number | boolean | undefined>>;
-}
-
-interface AgentSpanRecord {
- readonly name: string;
- readonly attributes: Readonly<Record<string, unknown>>;
-}
-
-/** Marks the `execute_tool` span of a tool call made outside any conversation. */
-export const DIRECT_TOOL_CALL_ATTRIBUTE = "eve.tool.invocation";
-export const DIRECT_TOOL_CALL_VALUE = "direct";
-
-/**
- * A direct tool call has no turn, so its own span is what starts and ends
- * its claim on the trace, the way an activation span does for a turn.
- */
-export function isDirectToolCallSpan(span: Pick<AgentSpanRecord, "attributes">): boolean {
- return span.attributes[DIRECT_TOOL_CALL_ATTRIBUTE] === DIRECT_TOOL_CALL_VALUE;
-}
-
-export function isAgentActivationSpan(span: AgentSpanRecord): boolean {
- return (
- span.name === "agent.turn" ||
- (span.attributes["gen_ai.operation.name"] === "invoke_agent" &&
- span.attributes["agent.invocation.role"] !== "caller" &&
- typeof span.attributes["agent.turn.id"] === "string")
- );
-}
-
-export function agentTurnIdentity(span: AgentSpanRecord): string | undefined {
- const sessionId = span.attributes["agent.run.id"];
- const turnId = span.attributes["agent.turn.id"];
- return typeof turnId === "string"
- ? `${typeof sessionId === "string" ? sessionId : ""}\0${turnId}`
- : undefined;
-}
diff --git a/packages/eve/src/tracing/agent-span-error.ts b/packages/eve/src/tracing/agent-span-error.ts
deleted file mode 100644
index e2b4dcf8a..000000000
--- a/packages/eve/src/tracing/agent-span-error.ts
+++ /dev/null
@@ -1,21 +0,0 @@
-import { SpanStatusCode, type Span } from "#compiled/@opentelemetry/api/index.js";
-import { boundedTraceError } from "#tracing/bounded-error.js";
-import { truncateTelemetryText } from "#tracing/telemetry-budget.js";
-
-/** Call with policy-projected errors; undefined retains failure status without content. */
-export function recordAgentSpanError(span: Span, error: unknown, errorType?: string): void {
- span.setAttribute(
- "error.type",
- truncateTelemetryText(
- errorType ?? (error instanceof Error ? error.name || "Error" : "_OTHER"),
- 128,
- ),
- );
- if (error instanceof Error) {
- const bounded = boundedTraceError(error);
- span.recordException(bounded);
- span.setStatus({ code: SpanStatusCode.ERROR, message: bounded.message });
- } else {
- span.setStatus({ code: SpanStatusCode.ERROR });
- }
-}
diff --git a/packages/eve/src/tracing/agent-span-id-generator.ts b/packages/eve/src/tracing/agent-span-id-generator.ts
deleted file mode 100644
index ca569cb8b..000000000
--- a/packages/eve/src/tracing/agent-span-id-generator.ts
+++ /dev/null
@@ -1,82 +0,0 @@
-import { createHash } from "node:crypto";
-
-/**
- * Id generator shared by `registerOTel` and the agent OTel provider. A span
- * whose lifetime crosses durable worker boundaries (`invoke_agent`) is emitted
- * at its terminal; priming the next span id lets that span carry the
- * pre-allocated id its descendants already parented to.
- */
-export class AgentSpanIdGenerator {
- #primedSpanId: string | undefined;
- #primedTraceId: string | undefined;
-
- /** Reserves a span id for a span emitted later via {@link withSpanId}. */
- allocateSpanId(): string {
- return randomHexId(16);
- }
-
- /** Derives one span id for a replay-stable instrumentation event. */
- deriveSpanId(key: string): string {
- return deriveAgentSpanId(key);
- }
-
- deriveTraceId(key: string): string {
- const traceId = createHash("sha256").update(key).digest("hex").slice(0, 32);
- return /^0+$/u.test(traceId) ? "00000000000000000000000000000001" : traceId;
- }
-
- generateSpanId(): string {
- const primed = this.#primedSpanId;
- if (primed !== undefined) {
- this.#primedSpanId = undefined;
- return primed;
- }
- return randomHexId(16);
- }
-
- generateTraceId(): string {
- const primed = this.#primedTraceId;
- if (primed !== undefined) {
- this.#primedTraceId = undefined;
- return primed;
- }
- return randomHexId(32);
- }
-
- /** Runs `startSpan` so the next span started carries `spanId`. */
- withSpanId<T>(spanId: string, startSpan: () => T): T {
- this.#primedSpanId = spanId;
- try {
- return startSpan();
- } finally {
- this.#primedSpanId = undefined;
- }
- }
-
- withTraceId<T>(traceId: string, startSpan: () => T): T {
- this.#primedTraceId = traceId;
- try {
- return startSpan();
- } finally {
- this.#primedTraceId = undefined;
- }
- }
-}
-
-function deriveAgentSpanId(key: string): string {
- const spanId = createHash("sha256").update(key).digest("hex").slice(0, 16);
- return /^0+$/u.test(spanId) ? "0000000000000001" : spanId;
-}
-
-const HEX_DIGITS = "0123456789abcdef";
-
-function randomHexId(length: number): string {
- let id: string;
- do {
- id = "";
- for (let index = 0; index < length; index += 1) {
- id += HEX_DIGITS[Math.trunc(Math.random() * 16)];
- }
- } while (!/[1-9a-f]/u.test(id));
- return id;
-}
diff --git a/packages/eve/src/tracing/agent-span-naming.ts b/packages/eve/src/tracing/agent-span-naming.ts
deleted file mode 100644
index b268155ae..000000000
--- a/packages/eve/src/tracing/agent-span-naming.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-// Datadog maps operation and resource separately from the OTel span name.
-export function agentSpanNamingAttributes(
- name: string,
- operation: string = name,
-): Record<string, string> {
- return { "operation.name": operation, "resource.name": name };
-}
diff --git a/packages/eve/src/tracing/agent-tool-instrumentation.ts b/packages/eve/src/tracing/agent-tool-instrumentation.ts
deleted file mode 100644
index 256d34dba..000000000
--- a/packages/eve/src/tracing/agent-tool-instrumentation.ts
+++ /dev/null
@@ -1,563 +0,0 @@
-import {
- ROOT_CONTEXT,
- SpanKind,
- type Context,
- type Attributes,
- type Span,
- type SpanContext,
- type Tracer,
- trace,
-} from "#compiled/@opentelemetry/api/index.js";
-
-import type {
- InstrumentationAttemptScope,
- InstrumentationExecutionOperation,
- InstrumentationToolCallStartedEvent,
- InstrumentationToolCallTerminalEvent,
-} from "#instrumentation/lifecycle.js";
-import { actionIdempotencyKey, attemptIdempotencyKey } from "#instrumentation/lifecycle.js";
-import { contentAttribute, textContentAttribute } from "#tracing/agent-otel-content.js";
-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";
-import { agentTraceIdentityAttributes, traceSessionIdOf } from "#tracing/agent-otel-attributes.js";
-import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";
-import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
-import type { AgentActionTraceState, AgentTraceStateStore } from "#tracing/agent-trace-state.js";
-import { normalizeChannelAudience } from "#shared/channel-audience.js";
-import { isSampledTrace } from "#tracing/sampled-trace.js";
-import { setAgentUsage } from "#tracing/agent-otel-usage.js";
-import { recordAgentSpanError as recordError } from "#tracing/agent-span-error.js";
-import { withAgentToolSpanContext } from "#tracing/agent-tool-span-context.js";
-
-interface ToolSpanState {
- readonly actionKey: string;
- readonly attemptId: string;
- context: Context;
- readonly additionalAttributes: Attributes;
- readonly event: InstrumentationToolCallStartedEvent;
- readonly fallbackParent: Context;
- readonly idempotencyKey: string;
- readonly spanId: string;
- readonly startTimeMs: number;
- finished?: true;
- span?: Span;
- terminal?: InstrumentationToolCallTerminalEvent;
- pendingError?: { readonly error: unknown; readonly errorType?: string };
- correlated?: true;
-}
-
-export interface AgentToolContext {
- readonly context: Context;
- readonly spanContext: SpanContext;
-}
-
-interface AgentToolInstrumentation {
- deleteForSession(sessionId: string): Promise<void>;
- failForAttempt(scope: InstrumentationAttemptScope, error: unknown): Promise<void>;
- dispatchContextFor(
- sessionId: string,
- turnId: string,
- callId: string,
- ): Promise<AgentToolContext | undefined>;
- contextFor(attemptId: string, idempotencyKey: string): Context | undefined;
- drain(attemptId: string, failure?: { readonly error: unknown }): void;
- readonly events: {
- readonly "tool.call.completed": (event: InstrumentationToolCallTerminalEvent) => Promise<void>;
- readonly "tool.call.failed": (event: InstrumentationToolCallTerminalEvent) => Promise<void>;
- readonly "tool.call.started": (event: InstrumentationToolCallStartedEvent) => Promise<void>;
- };
- runInContext<T>(
- operation: Extract<InstrumentationExecutionOperation, { type: "tool.call" }>,
- execute: (parent: Context | undefined) => PromiseLike<T>,
- recordOutputs: boolean,
- ): Promise<T>;
-}
-
-/** Enriches durable tool calls, exporting SDK-only calls under their step. */
-export function createAgentToolInstrumentation(input: {
- readonly frameworkVersion: string;
- readonly resolveTraceContext: (
- event: InstrumentationToolCallStartedEvent,
- ) => SpanContext | undefined | PromiseLike<SpanContext | undefined>;
- readonly idGenerator: AgentSpanIdGenerator;
- readonly recordInputs: boolean;
- readonly recordOutputs: boolean;
- readonly resolveFallback: (
- event: InstrumentationToolCallStartedEvent,
- ) => { readonly context: Context; readonly spanContext: SpanContext } | undefined;
- readonly tracer: Tracer;
- readonly stateStore: AgentTraceStateStore;
-}): AgentToolInstrumentation {
- const byAction = new Map<string, ToolSpanState>();
- const byAttempt = new Map<string, Map<string, ToolSpanState>>();
- const dispatchesByAttempt = new Map<string, Set<string>>();
-
- async function dispatchContextFor(
- sessionId: string,
- turnId: string,
- callId: string,
- ): Promise<AgentToolContext | undefined> {
- const state =
- (await input.stateStore.getAction(actionIdempotencyKey(sessionId, turnId, callId))) ??
- (await input.stateStore.findAction(sessionId, callId));
- if (state === undefined) return undefined;
- const spanContext = {
- isRemote: false,
- spanId: state.spanId,
- traceFlags: state.parent.traceFlags,
- traceId: state.parent.traceId,
- };
- return {
- context: withChannelAudience(contextFromSpanContext(spanContext), state.channelAudience),
- spanContext,
- };
- }
-
- async function onDispatchStarted(event: InstrumentationToolCallStartedEvent): Promise<void> {
- const traceContext = await input.resolveTraceContext(event);
- if (traceContext === undefined || !isSampledTrace(traceContext)) return;
- const existing = await input.stateStore.getAction(event.idempotencyKey);
- const state: AgentActionTraceState = existing ?? {
- attemptIndex: event.scope.attemptIndex,
- callId: event.callId,
- channelAudience: normalizeChannelAudience(event.scope.channelAudience),
- inputAttribute: input.recordInputs ? contentAttribute(event.input) : undefined,
- kind: event.kind ?? "tool-call",
- name: event.toolName,
- toolAttributes: { "agent.tool.is_framework": event.frameworkTool === true },
- parent: {
- spanId: input.idGenerator.deriveSpanId(attemptIdempotencyKey(event.scope)),
- traceFlags: traceContext.traceFlags,
- traceId: traceContext.traceId,
- },
- rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,
- traceSessionId: traceSessionIdOf(event.scope),
- sessionId: event.scope.sessionId,
- spanId: input.idGenerator.deriveSpanId(`action:${event.idempotencyKey}`),
- startTimeMs: Math.min(
- event.startedAtMs ?? Date.now(),
- byAction.get(event.idempotencyKey)?.startTimeMs ?? Infinity,
- ),
- stepIndex: event.scope.stepIndex,
- turnId: event.scope.turnId,
- };
- await input.stateStore.setAction(event.idempotencyKey, state);
- if (event.isWorkflowTool === true)
- await input.stateStore.setActionAnchor(event.idempotencyKey, state);
- const keys = dispatchesByAttempt.get(event.scope.attemptId) ?? new Set<string>();
- keys.add(event.idempotencyKey);
- dispatchesByAttempt.set(event.scope.attemptId, keys);
- const execution = byAction.get(event.idempotencyKey);
- if (execution !== undefined && execution.finished !== true) {
- await correlate(execution);
- finishIfReady(execution);
- }
- }
-
- async function onDispatchTerminal(event: InstrumentationToolCallTerminalEvent): Promise<void> {
- const state = await input.stateStore.getAction(event.idempotencyKey);
- if (state === undefined) return;
- try {
- const span = startDispatchSpan(state);
- const outcome = event.outcome ?? (event.type === "tool.call.failed" ? "failed" : "completed");
- span.setAttribute(
- "agent.action.outcome",
- state.toolFailed === true && outcome === "completed" ? "failed" : outcome,
- );
- if (event.usage !== undefined) setAgentUsage(span, event.usage);
- if (event.type === "tool.call.failed") {
- recordToolError(span, event.error, event.errorCode);
- } else if (event.output.type === "error") {
- recordToolError(span, event.output.error);
- } else if (state.toolFailed === true) {
- recordToolError(
- span,
- state.toolErrorAttribute,
- typeof state.toolAttributes?.["error.type"] === "string"
- ? state.toolAttributes["error.type"]
- : undefined,
- );
- } else if (input.recordOutputs) {
- const result = contentAttribute(event.output.output);
- if (result !== undefined) span.setAttribute("gen_ai.tool.call.result", result);
- }
- span.end(event.acceptedAtMs ?? state.toolEndTimeMs);
- } finally {
- await input.stateStore.deleteAction(event.idempotencyKey);
- for (const [attemptId, keys] of dispatchesByAttempt) {
- keys.delete(event.idempotencyKey);
- if (keys.size === 0) dispatchesByAttempt.delete(attemptId);
- }
- }
- }
-
- function startDispatchSpan(state: AgentActionTraceState): Span {
- const invocation = state.kind === "subagent-call" || state.kind === "remote-agent-call";
- const span = input.idGenerator.withSpanId(state.spanId, () =>
- input.tracer.startSpan(
- `execute_tool ${state.name}`,
- {
- attributes: {
- "agent.action.kind": state.kind,
- "agent.action.name": state.name,
- "gen_ai.operation.name": "execute_tool",
- "gen_ai.tool.call.id": state.callId,
- "gen_ai.tool.name": state.name,
- "gen_ai.tool.type": "function",
- "agent.framework.name": "eve",
- "agent.framework.version": input.frameworkVersion,
- "agent.step.attempt": state.attemptIndex,
- "agent.step.index": state.stepIndex,
- "agent.turn.id": state.turnId,
- ...agentSpanNamingAttributes(`execute_tool ${state.name}`, "execute_tool"),
- ...agentTraceIdentityAttributes({
- rootSessionId: state.rootSessionId,
- traceSessionId: state.traceSessionId,
- sessionId: state.sessionId,
- }),
- ...(invocation
- ? { "gen_ai.agent.name": state.name, "agent.invocation.role": "caller" }
- : undefined),
- ...state.toolAttributes,
- },
- kind: state.kind === "remote-agent-call" ? SpanKind.CLIENT : SpanKind.INTERNAL,
- startTime: state.startTimeMs,
- },
- withChannelAudience(
- contextFromSpanContext({ ...state.parent, isRemote: false }),
- state.channelAudience,
- ),
- ),
- );
- if (state.inputAttribute !== undefined)
- span.setAttribute("gen_ai.tool.call.arguments", state.inputAttribute);
- return span;
- }
-
- const onStarted = async (event: InstrumentationToolCallStartedEvent): Promise<void> => {
- if (byAttempt.get(event.scope.attemptId)?.has(event.idempotencyKey)) return;
- const actionKey = actionIdempotencyKey(event.scope.sessionId, event.scope.turnId, event.callId);
- const fallback = input.resolveFallback(event);
- let state = fallback === undefined ? undefined : reserve(event, actionKey, fallback);
- const actionParent = await dispatchContextFor(
- event.scope.sessionId,
- event.scope.turnId,
- event.callId,
- );
- if (state === undefined) {
- if (actionParent === undefined) return;
- state = reserve(event, actionKey, actionParent);
- }
- if (actionParent !== undefined) await correlate(state);
- };
-
- const onTerminal = async (event: InstrumentationToolCallTerminalEvent): Promise<void> => {
- const state = byAttempt.get(event.scope.attemptId)?.get(event.idempotencyKey);
- if (state === undefined) return;
- state.terminal = event;
-
- if (state.span === undefined) {
- const actionParent = await dispatchContextFor(
- state.event.scope.sessionId,
- state.event.scope.turnId,
- state.event.callId,
- );
- if (actionParent !== undefined) await correlate(state);
- }
- finishIfReady(state);
- };
-
- return {
- dispatchContextFor,
- async deleteForSession(sessionId) {
- await input.stateStore.deleteActions(sessionId);
- await input.stateStore.deleteActionAnchors(sessionId);
- },
- async failForAttempt(scope, error) {
- const keys = dispatchesByAttempt.get(scope.attemptId);
- dispatchesByAttempt.delete(scope.attemptId);
- for (const key of keys ?? []) {
- const state = await input.stateStore.getAction(key);
- if (state === undefined) continue;
- const span = startDispatchSpan(state);
- recordError(span, error);
- span.end();
- await input.stateStore.deleteAction(key);
- }
- },
- contextFor: (attemptId, idempotencyKey) =>
- byAttempt.get(attemptId)?.get(idempotencyKey)?.context,
- drain(attemptId, failure) {
- const states = byAttempt.get(attemptId);
- if (states === undefined) return;
- for (const state of states.values()) {
- if (state.finished === true) continue;
- if (state.span === undefined && state.correlated !== true)
- startSpan(state, state.fallbackParent);
- finish(state, failure);
- }
- byAttempt.delete(attemptId);
- },
- events: {
- "tool.call.completed": onDispatchTerminal,
- "tool.call.failed": onDispatchTerminal,
- "tool.call.started": onDispatchStarted,
- },
- async runInContext(operation, execute, recordOutputs) {
- if (operation.callId === undefined || operation.toolName === undefined) {
- return execute(
- byAttempt.get(operation.scope.attemptId)?.get(operation.idempotencyKey)?.context,
- );
- }
- await onStarted({
- ...operation,
- callId: operation.callId,
- toolName: operation.toolName,
- input: operation.input,
- type: "tool.call.started",
- });
- const state = byAttempt.get(operation.scope.attemptId)?.get(operation.idempotencyKey);
- const startedAtMs = operation.startedAtMs ?? Date.now();
- try {
- const result = await execute(state?.context);
- await onTerminal({
- type: operation.failed ? "tool.call.failed" : "tool.call.completed",
- idempotencyKey: operation.idempotencyKey,
- scope: operation.scope,
- completedAtMs: operation.completedAtMs ?? Date.now(),
- durationMs: (operation.completedAtMs ?? Date.now()) - startedAtMs,
- output: { type: "result", output: recordOutputs ? result : undefined },
- });
- return result;
- } catch (error) {
- await onTerminal({
- type: "tool.call.failed",
- idempotencyKey: operation.idempotencyKey,
- scope: operation.scope,
- completedAtMs: Date.now(),
- error: recordOutputs ? error : undefined,
- });
- throw error;
- }
- },
- };
-
- function getAttemptStates(attemptId: string): Map<string, ToolSpanState> {
- let states = byAttempt.get(attemptId);
- if (states === undefined) {
- states = new Map();
- byAttempt.set(attemptId, states);
- }
- return states;
- }
-
- function reserve(
- event: InstrumentationToolCallStartedEvent,
- actionKey: string,
- parent: { readonly context: Context; readonly spanContext: SpanContext },
- ): ToolSpanState {
- const spanId = input.idGenerator.deriveSpanId(`action:${actionKey}`);
- const state: ToolSpanState = {
- actionKey,
- attemptId: event.scope.attemptId,
- additionalAttributes: {},
- context: withChannelAudience(
- contextFromSpanContext({
- isRemote: false,
- spanId,
- traceFlags: parent.spanContext.traceFlags,
- traceId: parent.spanContext.traceId,
- }),
- event.scope.channelAudience,
- ),
- event,
- fallbackParent: parent.context,
- idempotencyKey: event.idempotencyKey,
- spanId,
- startTimeMs: event.startedAtMs ?? Date.now(),
- };
- state.context = withAgentToolSpanContext(state.context, {
- recordInputs: input.recordInputs,
- recordOutputs: input.recordOutputs,
- setAttributes(attributes) {
- Object.assign(state.additionalAttributes, attributes);
- if (state.span === undefined) return;
- for (const [name, value] of Object.entries(attributes)) {
- if (value !== undefined) state.span.setAttribute(name, value);
- }
- },
- recordError(error, errorType) {
- state.pendingError = { error, errorType };
- if (state.span !== undefined) recordError(state.span, error, errorType);
- },
- });
- getAttemptStates(event.scope.attemptId).set(event.idempotencyKey, state);
- byAction.set(actionKey, state);
- return state;
- }
-
- function startSpan(state: ToolSpanState, parent: Context): void {
- if (state.span !== undefined || state.finished === true) return;
- state.span = input.idGenerator.withSpanId(state.spanId, () =>
- input.tracer.startSpan(
- `execute_tool ${state.event.toolName}`,
- {
- attributes: { ...toolAttributes(state.event), ...state.additionalAttributes },
- kind: SpanKind.INTERNAL,
- startTime: state.startTimeMs,
- },
- parent,
- ),
- );
- if (state.pendingError !== undefined) {
- recordError(state.span, state.pendingError.error, state.pendingError.errorType);
- }
- if (input.recordInputs) {
- const args = contentAttribute(state.event.input);
- if (args !== undefined) state.span.setAttribute("gen_ai.tool.call.arguments", args);
- }
- }
-
- function finishIfReady(state: ToolSpanState): void {
- if ((state.span === undefined && state.correlated !== true) || state.terminal === undefined)
- return;
- finish(state);
- }
-
- async function correlate(state: ToolSpanState): Promise<void> {
- const action =
- (await input.stateStore.getAction(state.actionKey)) ??
- (await input.stateStore.findAction(state.event.scope.sessionId, state.event.callId));
- if (action === undefined) return;
- state.correlated = true;
- state.context = trace.setSpan(
- state.context,
- trace.wrapSpanContext({
- isRemote: false,
- spanId: action.spanId,
- traceFlags: action.parent.traceFlags,
- traceId: action.parent.traceId,
- }),
- );
- const terminal = state.terminal;
- const toolFailed =
- state.pendingError !== undefined ||
- terminal?.type === "tool.call.failed" ||
- terminal?.output.type === "error";
- const error =
- state.pendingError?.error ??
- (terminal?.type === "tool.call.failed"
- ? terminal.error
- : terminal?.output.type === "error"
- ? terminal.output.error
- : undefined);
- const attributes: Attributes = {
- ...action.toolAttributes,
- ...(state.event.scope.functionId === undefined ||
- action.kind === "subagent-call" ||
- action.kind === "remote-agent-call"
- ? undefined
- : { "gen_ai.agent.name": state.event.scope.functionId }),
- "agent.tool.is_framework": state.event.frameworkTool === true,
- ...state.additionalAttributes,
- ...(state.pendingError?.errorType === undefined
- ? undefined
- : { "error.type": state.pendingError.errorType }),
- ...(terminal?.type === "tool.call.completed" && terminal.durationMs !== undefined
- ? { "gen_ai.execute_tool.duration": terminal.durationMs / 1000 }
- : undefined),
- };
- await input.stateStore.setAction(
- actionIdempotencyKey(action.sessionId, action.turnId, action.callId),
- {
- ...action,
- startTimeMs: Math.min(action.startTimeMs, state.startTimeMs),
- toolAttributes: attributes,
- toolEndTimeMs: terminal?.completedAtMs ?? action.toolEndTimeMs,
- toolFailed: toolFailed || action.toolFailed,
- toolErrorAttribute:
- input.recordOutputs && error !== undefined
- ? contentAttribute(error instanceof Error ? error.message : error)
- : action.toolErrorAttribute,
- },
- );
- }
-
- function finish(state: ToolSpanState, failure?: { readonly error: unknown }): void {
- const span = state.span;
- if (state.finished === true) return;
- if (state.correlated === true) {
- state.finished = true;
- forget(state);
- return;
- }
- if (span === undefined) return;
- state.finished = true;
- const terminal = state.terminal;
- // As `@ai-sdk/otel` records it: execute's own run time, in seconds, success or error.
- if (terminal?.type === "tool.call.completed" && terminal.durationMs !== undefined) {
- span.setAttribute("gen_ai.execute_tool.duration", terminal.durationMs / 1000);
- }
- if (failure !== undefined) {
- recordError(span, failure.error);
- } else if (terminal?.type === "tool.call.failed") {
- recordError(span, terminal.error);
- } else if (terminal?.output.type === "error") {
- recordError(span, terminal.output.error);
- } else if (terminal !== undefined && input.recordOutputs) {
- const result = contentAttribute(terminal.output.output);
- if (result !== undefined) span.setAttribute("gen_ai.tool.call.result", result);
- }
- span.end(terminal?.completedAtMs);
- forget(state);
- }
-
- function forget(state: ToolSpanState): void {
- byAction.delete(state.actionKey);
- const states = byAttempt.get(state.attemptId);
- states?.delete(state.idempotencyKey);
- if (states?.size === 0) byAttempt.delete(state.attemptId);
- }
-}
-
-function toolAttributes(event: InstrumentationToolCallStartedEvent): Attributes {
- return {
- "agent.tool.is_framework": event.frameworkTool === true,
- ...(event.scope.functionId === undefined
- ? {}
- : { "gen_ai.agent.name": event.scope.functionId }),
- "gen_ai.operation.name": "execute_tool",
- "gen_ai.tool.call.id": event.callId,
- "gen_ai.tool.name": event.toolName,
- "gen_ai.tool.type": "function",
- ...agentSpanNamingAttributes(`execute_tool ${event.toolName}`, "execute_tool"),
- ...agentTraceIdentityAttributes({
- rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,
- traceSessionId: traceSessionIdOf(event.scope),
- sessionId: event.scope.sessionId,
- }),
- };
-}
-
-function contextFromSpanContext(spanContext: SpanContext): Context {
- return trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(spanContext));
-}
-
-function recordToolError(span: Span, error: unknown, errorType?: string): void {
- if (error instanceof Error || error === undefined) {
- recordError(span, error, errorType);
- return;
- }
- const serialized =
- typeof error === "string" ? textContentAttribute(error) : contentAttribute(error);
- const message =
- typeof error === "object" && error !== null && !Array.isArray(error)
- ? Reflect.get(error, "message")
- : undefined;
- const detail =
- typeof message === "string"
- ? textContentAttribute(serialized === undefined ? message : `${message}\n${serialized}`)
- : serialized;
- const normalized = detail === undefined ? undefined : new Error(detail);
- if (normalized !== undefined && errorType !== undefined) normalized.name = errorType;
- recordError(span, normalized, errorType);
-}
diff --git a/packages/eve/src/tracing/agent-tool-span-context.test.ts b/packages/eve/src/tracing/agent-tool-span-context.test.ts
deleted file mode 100644
index b9227c111..000000000
--- a/packages/eve/src/tracing/agent-tool-span-context.test.ts
+++ /dev/null
@@ -1,37 +0,0 @@
-import { ROOT_CONTEXT } from "#compiled/@opentelemetry/api/index.js";
-import { describe, expect, it, vi } from "vitest";
-
-import {
- agentToolContentPolicy,
- annotateAgentToolSpan,
- recordAgentToolSpanError,
- withAgentToolContentPolicy,
- withAgentToolSpanContext,
-} from "#tracing/agent-tool-span-context.js";
-
-describe("agent tool span context", () => {
- it("narrows the capture policy without dropping the span writer", () => {
- const setAttributes = vi.fn();
- const recordError = vi.fn();
- const context = withAgentToolSpanContext(ROOT_CONTEXT, {
- recordError,
- recordInputs: true,
- recordOutputs: true,
- setAttributes,
- });
- const narrowed = withAgentToolContentPolicy(context, {
- recordInputs: false,
- recordOutputs: false,
- });
-
- expect(agentToolContentPolicy(narrowed)).toEqual({
- recordInputs: false,
- recordOutputs: false,
- });
- expect(annotateAgentToolSpan({ "mcp.method.name": "tools/call" }, narrowed)).toBe(true);
- recordAgentToolSpanError(new Error("private result"), "tool_error", narrowed);
-
- expect(setAttributes).toHaveBeenCalledWith({ "mcp.method.name": "tools/call" });
- expect(recordError).toHaveBeenCalledWith(undefined, "tool_error");
- });
-});
diff --git a/packages/eve/src/tracing/agent-tool-span-context.ts b/packages/eve/src/tracing/agent-tool-span-context.ts
deleted file mode 100644
index f08b7e27b..000000000
--- a/packages/eve/src/tracing/agent-tool-span-context.ts
+++ /dev/null
@@ -1,73 +0,0 @@
-import {
- context as otelContext,
- createContextKey,
- type Attributes,
- type Context,
-} from "#compiled/@opentelemetry/api/index.js";
-
-interface AgentToolContentPolicy {
- readonly recordInputs: boolean;
- readonly recordOutputs: boolean;
-}
-
-interface AgentToolSpanContext extends AgentToolContentPolicy {
- readonly recordError?: (error: unknown, errorType?: string) => void;
- readonly setAttributes?: (attributes: Attributes) => void;
-}
-
-const AGENT_TOOL_SPAN_CONTEXT_KEY = createContextKey("eve.agent.tool-span-context");
-const NO_CONTENT_POLICY: AgentToolContentPolicy = { recordInputs: false, recordOutputs: false };
-
-export function withAgentToolSpanContext(context: Context, value: AgentToolSpanContext): Context {
- return context.setValue(AGENT_TOOL_SPAN_CONTEXT_KEY, value);
-}
-
-export function withAgentToolContentPolicy(
- context: Context,
- policy: AgentToolContentPolicy,
-): Context {
- const current = agentToolSpanContext(context);
- return context.setValue(AGENT_TOOL_SPAN_CONTEXT_KEY, {
- ...current,
- ...policy,
- } satisfies AgentToolSpanContext);
-}
-
-export function agentToolSpanContext(
- context: Context = otelContext.active(),
-): AgentToolSpanContext | undefined {
- return context.getValue(AGENT_TOOL_SPAN_CONTEXT_KEY) as AgentToolSpanContext | undefined;
-}
-
-export function agentToolContentPolicy(
- context: Context = otelContext.active(),
-): AgentToolContentPolicy {
- const spanContext = agentToolSpanContext(context);
- return spanContext === undefined
- ? NO_CONTENT_POLICY
- : {
- recordInputs: spanContext.recordInputs,
- recordOutputs: spanContext.recordOutputs,
- };
-}
-
-export function annotateAgentToolSpan(
- attributes: Attributes,
- context: Context = otelContext.active(),
-): boolean {
- const setAttributes = agentToolSpanContext(context)?.setAttributes;
- if (setAttributes === undefined) return false;
- setAttributes(attributes);
- return true;
-}
-
-export function recordAgentToolSpanError(
- error: unknown,
- errorType?: string,
- context: Context = otelContext.active(),
-): boolean {
- const spanContext = agentToolSpanContext(context);
- if (spanContext?.recordError === undefined) return false;
- spanContext.recordError(spanContext.recordOutputs ? error : undefined, errorType);
- return true;
-}
diff --git a/packages/eve/src/tracing/agent-trace-context-codec.ts b/packages/eve/src/tracing/agent-trace-context-codec.ts
deleted file mode 100644
index 3f823b836..000000000
--- a/packages/eve/src/tracing/agent-trace-context-codec.ts
+++ /dev/null
@@ -1,318 +0,0 @@
-import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";
-
-import type {
- AgentActionTraceState,
- AgentSessionTraceState,
- AgentTurnTraceState,
-} from "#tracing/agent-trace-state.js";
-import { normalizeChannelAudience } from "#shared/channel-audience.js";
-import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";
-import { boundedTraceError } from "#tracing/bounded-error.js";
-import { CONTENT_ATTRIBUTE_LIMIT } from "#tracing/agent-otel-content.js";
-import { boundedPrincipalId } from "#tracing/telemetry-budget.js";
-import { isInstrumentationPrincipalType } from "#instrumentation/lifecycle.js";
-
-export const AGENT_TRACE_CONTEXT_KEY = "eve.harness.agentTrace";
-
-export function decodeTraceSessionId(state: {
- readonly traceSessionId?: unknown;
- readonly rootSessionId?: unknown;
-}): string {
- // Persisted state from before trace-session identity uses the lineage root.
- return typeof state.traceSessionId === "string"
- ? state.traceSessionId
- : typeof state.rootSessionId === "string"
- ? state.rootSessionId
- : "";
-}
-
-export interface AgentTraceContextState {
- readonly actionAnchors: Readonly<Record<string, AgentActionTraceState>>;
- readonly actions: Readonly<Record<string, AgentActionTraceState>>;
- readonly sessions: Readonly<Record<string, AgentSessionTraceState>>;
- readonly turns: Readonly<Record<string, AgentTurnTraceState>>;
-}
-
-export function emptyAgentTraceContextState(): AgentTraceContextState {
- return { actionAnchors: {}, actions: {}, sessions: {}, turns: {} };
-}
-
-export function serializeAgentTraceContextState(state: AgentTraceContextState): unknown {
- return {
- actionAnchors: state.actionAnchors,
- actions: state.actions,
- sessions: Object.fromEntries(
- Object.entries(state.sessions).map(([id, value]) => [
- id,
- { ...value, context: serializeSpanContext(value.context) },
- ]),
- ),
- turns: Object.fromEntries(
- Object.entries(state.turns).map(([id, value]) => [
- id,
- {
- ...value,
- caller: value.caller === undefined ? undefined : serializeSpanContext(value.caller),
- channelDelivery:
- value.channelDelivery === undefined
- ? undefined
- : {
- ...value.channelDelivery,
- requestTraceContext:
- value.channelDelivery.requestTraceContext === undefined
- ? undefined
- : serializeSpanContext(value.channelDelivery.requestTraceContext),
- },
- context: serializeSpanContext(value.context),
- terminal:
- value.terminal === undefined
- ? undefined
- : value.terminal.type === "turn.failed"
- ? { error: serializeError(value.terminal.error), type: value.terminal.type }
- : { type: value.terminal.type },
- },
- ]),
- ),
- };
-}
-
-export function deserializeAgentTraceContextState(data: unknown): AgentTraceContextState {
- if (!isRecord(data)) return emptyAgentTraceContextState();
- return {
- actionAnchors: deserializeRecord(data.actionAnchors, deserializeAction),
- actions: deserializeRecord(data.actions, deserializeAction),
- sessions: deserializeRecord(data.sessions, deserializeSession),
- turns: deserializeRecord(data.turns, deserializeTurn),
- };
-}
-
-function deserializeSession(value: unknown): AgentSessionTraceState | undefined {
- if (!isRecord(value) || !isSpanContext(value.context)) return undefined;
- return {
- agentName: typeof value.agentName === "string" ? value.agentName : undefined,
- channelAudience: normalizeChannelAudience(value.channelAudience),
- channelKind: typeof value.channelKind === "string" ? value.channelKind : undefined,
- channelType: typeof value.channelType === "string" ? value.channelType : undefined,
- context: value.context,
- decision: readInstrumentationDecision(value.decision),
- parentLineage: deserializeParentLineage(value.parentLineage),
- rootSessionId: typeof value.rootSessionId === "string" ? value.rootSessionId : "",
- traceSessionId: decodeTraceSessionId(value),
- scheduleId: typeof value.scheduleId === "string" ? value.scheduleId : undefined,
- title: typeof value.title === "string" ? value.title : undefined,
- };
-}
-
-function deserializeTurn(value: unknown): AgentTurnTraceState | undefined {
- if (!isRecord(value) || !isSpanContext(value.context) || typeof value.startTimeMs !== "number") {
- return undefined;
- }
- return {
- caller: isSpanContext(value.caller) ? value.caller : undefined,
- channelDelivery: deserializeTurnChannelDelivery(value.channelDelivery),
- context: value.context,
- currentPrincipal: deserializePrincipalSummary(value.currentPrincipal),
- initiatorPrincipal: deserializePrincipalSummary(value.initiatorPrincipal),
- inputMessagesAttribute: deserializeContentAttribute(value.inputMessagesAttribute),
- modelUsage: deserializeModelUsage(value.modelUsage),
- outputMessagesAttribute: deserializeContentAttribute(value.outputMessagesAttribute),
- parentLineage: deserializeParentLineage(value.parentLineage),
- rootSessionId: typeof value.rootSessionId === "string" ? value.rootSessionId : "",
- traceSessionId: decodeTraceSessionId(value),
- sequence: typeof value.sequence === "number" ? value.sequence : 0,
- startTimeMs: value.startTimeMs,
- subagentName: typeof value.subagentName === "string" ? value.subagentName : undefined,
- terminal: deserializeTurnTerminal(value.terminal),
- };
-}
-
-function deserializeTurnChannelDelivery(value: unknown): AgentTurnTraceState["channelDelivery"] {
- if (
- !isRecord(value) ||
- typeof value.channelKind !== "string" ||
- typeof value.channelName !== "string" ||
- typeof value.deliveryId !== "string"
- ) {
- return undefined;
- }
- return {
- channelKind: value.channelKind,
- channelName: value.channelName,
- deliveryId: value.deliveryId,
- inputAttribute: typeof value.inputAttribute === "string" ? value.inputAttribute : undefined,
- requestId: typeof value.requestId === "string" ? value.requestId : undefined,
- requestTraceContext: isSpanContext(value.requestTraceContext)
- ? value.requestTraceContext
- : undefined,
- };
-}
-
-function deserializePrincipalSummary(
- value: unknown,
-): AgentTurnTraceState["currentPrincipal"] | undefined {
- if (!isRecord(value) || !isInstrumentationPrincipalType(value.type)) return undefined;
- const id = value.type === "none" ? undefined : boundedPrincipalId(value.id);
- return id === undefined ? { type: value.type } : { id, type: value.type };
-}
-
-function deserializeAction(value: unknown): AgentActionTraceState | undefined {
- if (
- !isRecord(value) ||
- typeof value.attemptIndex !== "number" ||
- typeof value.callId !== "string" ||
- !isActionKind(value.kind) ||
- typeof value.name !== "string" ||
- !isSpanContext(value.parent) ||
- typeof value.rootSessionId !== "string" ||
- typeof value.sessionId !== "string" ||
- typeof value.spanId !== "string" ||
- typeof value.startTimeMs !== "number" ||
- typeof value.stepIndex !== "number" ||
- typeof value.turnId !== "string"
- ) {
- return undefined;
- }
- return {
- attemptIndex: value.attemptIndex,
- callId: value.callId,
- channelAudience: normalizeChannelAudience(value.channelAudience),
- inputAttribute: typeof value.inputAttribute === "string" ? value.inputAttribute : undefined,
- kind: value.kind,
- name: value.name,
- parent: value.parent,
- rootSessionId: value.rootSessionId,
- traceSessionId: decodeTraceSessionId(value),
- sessionId: value.sessionId,
- spanId: value.spanId,
- startTimeMs: value.startTimeMs,
- stepIndex: value.stepIndex,
- turnId: value.turnId,
- toolEndTimeMs: typeof value.toolEndTimeMs === "number" ? value.toolEndTimeMs : undefined,
- toolFailed: value.toolFailed === true,
- toolErrorAttribute:
- typeof value.toolErrorAttribute === "string" ? value.toolErrorAttribute : undefined,
- toolAttributes: isRecord(value.toolAttributes)
- ? Object.fromEntries(
- Object.entries(value.toolAttributes).filter(
- (
- entry,
- ): entry is [string, NonNullable<AgentActionTraceState["toolAttributes"]>[string]] => {
- const attribute = entry[1];
- return (
- typeof attribute === "string" ||
- typeof attribute === "number" ||
- typeof attribute === "boolean" ||
- (Array.isArray(attribute) &&
- attribute.every((entry) =>
- ["string", "number", "boolean"].includes(typeof entry),
- ))
- );
- },
- ),
- )
- : undefined,
- };
-}
-
-function deserializeContentAttribute(value: unknown): string | undefined {
- return typeof value === "string" && value.length <= CONTENT_ATTRIBUTE_LIMIT ? value : undefined;
-}
-
-function deserializeModelUsage(
- value: unknown,
-): NonNullable<AgentTurnTraceState["modelUsage"]> | undefined {
- if (!isRecord(value)) return undefined;
- const inputTokens = typeof value.inputTokens === "number" ? value.inputTokens : undefined;
- const outputTokens = typeof value.outputTokens === "number" ? value.outputTokens : undefined;
- return inputTokens === undefined && outputTokens === undefined
- ? undefined
- : { inputTokens, outputTokens };
-}
-
-function deserializeParentLineage(value: unknown): AgentSessionTraceState["parentLineage"] {
- if (
- !isRecord(value) ||
- typeof value.callId !== "string" ||
- typeof value.sessionId !== "string" ||
- typeof value.turnId !== "string"
- ) {
- return undefined;
- }
- return {
- callId: value.callId,
- sessionId: value.sessionId,
- subagentName: typeof value.subagentName === "string" ? value.subagentName : undefined,
- turnId: value.turnId,
- };
-}
-
-function deserializeTurnTerminal(value: unknown): AgentTurnTraceState["terminal"] {
- if (!isRecord(value) || !isTurnTerminalType(value.type)) return undefined;
- return value.type === "turn.failed"
- ? { error: deserializeError(value.error), type: value.type }
- : { type: value.type };
-}
-
-function deserializeRecord<T>(
- value: unknown,
- deserialize: (entry: unknown) => T | undefined,
-): Record<string, T> {
- if (!isRecord(value)) return {};
- const result: Record<string, T> = {};
- for (const [key, entry] of Object.entries(value)) {
- const parsed = deserialize(entry);
- if (parsed !== undefined) result[key] = parsed;
- }
- return result;
-}
-
-function serializeSpanContext(context: SpanContext): Record<string, unknown> {
- return {
- isRemote: context.isRemote,
- spanId: context.spanId,
- traceFlags: context.traceFlags,
- traceId: context.traceId,
- };
-}
-
-function serializeError(error: unknown): unknown {
- if (!(error instanceof Error)) return undefined;
- const bounded = boundedTraceError(error);
- return { message: bounded.message, name: bounded.name, stack: bounded.stack };
-}
-
-function deserializeError(value: unknown): Error | undefined {
- if (!isRecord(value) || typeof value.message !== "string") return undefined;
- const error = new Error(value.message);
- if (typeof value.name === "string") error.name = value.name;
- if (typeof value.stack === "string") error.stack = value.stack;
- return error;
-}
-
-function isActionKind(value: unknown): value is AgentActionTraceState["kind"] {
- return (
- value === "load-skill" ||
- value === "remote-agent-call" ||
- value === "subagent-call" ||
- value === "tool-call"
- );
-}
-
-function isTurnTerminalType(
- value: unknown,
-): value is "turn.cancelled" | "turn.completed" | "turn.failed" {
- return value === "turn.cancelled" || value === "turn.completed" || value === "turn.failed";
-}
-
-function isSpanContext(value: unknown): value is SpanContext {
- return (
- isRecord(value) &&
- typeof value.spanId === "string" &&
- typeof value.traceFlags === "number" &&
- typeof value.traceId === "string"
- );
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
- return typeof value === "object" && value !== null && !Array.isArray(value);
-}
diff --git a/packages/eve/src/tracing/agent-trace-context-store.ts b/packages/eve/src/tracing/agent-trace-context-store.ts
deleted file mode 100644
index b23f0267a..000000000
--- a/packages/eve/src/tracing/agent-trace-context-store.ts
+++ /dev/null
@@ -1,291 +0,0 @@
-import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";
-
-import type { SessionTraceContext } from "#channel/types.js";
-import { contextStorage, loadContext } from "#context/container.js";
-import { ContextKey } from "#context/key.js";
-import type { ContextAccessor } from "#context/key.js";
-import { SessionTraceSeedKey, type SessionTraceSeed } from "#context/keys.js";
-import type {
- AgentActionTraceState,
- AgentSessionTraceState,
- AgentTraceStateStore,
- AgentTurnTraceState,
-} from "#tracing/agent-trace-state.js";
-import { actionIdempotencyKey } from "#instrumentation/lifecycle.js";
-import type { SessionStateMap } from "#harness/types.js";
-import { getBlockingWorkflowToolRuns } from "#harness/workflow-tool-runs.js";
-
-import { createLogger } from "#internal/logging.js";
-import type { InstrumentationDecision } from "#shared/instrumentation-decision.js";
-import {
- decisionToTraceContentCeiling,
- resolveForwardedTraceSeed,
-} from "#shared/forwarded-trace-policy.js";
-import {
- deserializeAgentTraceContextState,
- AGENT_TRACE_CONTEXT_KEY,
- emptyAgentTraceContextState,
- serializeAgentTraceContextState,
- type AgentTraceContextState,
-} from "#tracing/agent-trace-context-codec.js";
-
-const AgentTraceContextKey = new ContextKey<AgentTraceContextState>(AGENT_TRACE_CONTEXT_KEY, {
- codec: {
- deserialize: deserializeAgentTraceContextState,
- serialize: serializeAgentTraceContextState,
- },
-});
-
-/** Run after task-provider commits, so a pending workflow tool call keeps its anchor. */
-export function pruneAgentTraceState(
- context: ContextAccessor,
- sessionId: string,
- sessionState: SessionStateMap | undefined,
-): void {
- try {
- pruneTraceOwnership(context, sessionId, sessionState);
- } catch (error) {
- createLogger("tracing.retention").warn(
- "could not reconcile trace ownership; preserving trace state",
- { error },
- );
- }
-}
-
-function pruneTraceOwnership(
- context: ContextAccessor,
- sessionId: string,
- sessionState: SessionStateMap | undefined,
-): void {
- const state = context.get(AgentTraceContextKey);
- if (state === undefined) return;
- const calls = new Set(getBlockingWorkflowToolRuns(sessionState).map((run) => run.callId));
- const actionAnchors = Object.fromEntries(
- Object.entries(state.actionAnchors).filter(
- ([key, action]) =>
- action.sessionId !== sessionId ||
- state.actions[key] !== undefined ||
- calls.has(action.callId),
- ),
- );
- context.set(AgentTraceContextKey, { ...state, actionAnchors });
-}
-
-export function readSessionTraceDecision(
- context: ContextAccessor,
- sessionId: string,
-): InstrumentationDecision | undefined {
- return context.get(AgentTraceContextKey)?.sessions[sessionId]?.decision;
-}
-
-/** Keeps only framework trace state from an interrupted step's context changes. */
-export function preserveSerializedAgentTraceState(
- original: Record<string, unknown>,
- interrupted: Record<string, unknown>,
-): Record<string, unknown> {
- const traceState = interrupted[AgentTraceContextKey.name];
- return traceState === undefined
- ? original
- : { ...original, [AgentTraceContextKey.name]: traceState };
-}
-
-/** Reads the active turn context straight out of a serialized Workflow context. */
-export function readTurnTraceContext(
- serializedContext: Readonly<Record<string, unknown>>,
- sessionId: string,
- turnId: string,
-): SessionTraceContext | undefined {
- const raw = serializedContext[AgentTraceContextKey.name];
- if (raw === undefined) return undefined;
- const state = deserializeAgentTraceContextState(raw);
- const turn = state.turns[turnKey(sessionId, turnId)];
- return turn === undefined
- ? undefined
- : withTraceDecision(serializedContext, turn.context, state.sessions[sessionId]?.decision);
-}
-
-/** Reads the durable action span that should parent a dispatched child agent. */
-export function readActionTraceContext(
- serializedContext: Readonly<Record<string, unknown>>,
- sessionId: string,
- turnId: string,
- callId: string,
-): SessionTraceContext | undefined {
- const raw = serializedContext[AgentTraceContextKey.name];
- if (raw === undefined) return undefined;
- const state = deserializeAgentTraceContextState(raw);
- const key = actionIdempotencyKey(sessionId, turnId, callId);
- const action = state.actions[key] ?? state.actionAnchors[key];
- if (action === undefined) return undefined;
- return withTraceDecision(
- serializedContext,
- {
- isRemote: false,
- spanId: action.spanId,
- traceFlags: action.parent.traceFlags,
- traceId: action.parent.traceId,
- },
- state.sessions[action.sessionId]?.decision,
- );
-}
-
-function withTraceDecision(
- serializedContext: Readonly<Record<string, unknown>>,
- context: SpanContext,
- storedDecision?: InstrumentationDecision,
-): SessionTraceContext {
- const seed = serializedContext[SessionTraceSeedKey.name] as SessionTraceSeed | undefined;
- const traceState = resolveForwardedTraceSeed({
- decision: storedDecision ?? seed?.decision,
- forwardedTracePolicy: seed?.forwardedTracePolicy,
- traceFlags: context.traceFlags,
- })!;
- const decision = traceState.decision;
- const forwardedTracePolicy = traceState.forwardedTracePolicy;
- const ceiling = decisionToTraceContentCeiling(decision);
- const resolvedContext = { ...context, traceFlags: traceState.traceFlags };
- if (forwardedTracePolicy === undefined) {
- return decision === undefined ? resolvedContext : { ...resolvedContext, decision };
- }
- const narrowedForwardedTracePolicy =
- ceiling === undefined ? forwardedTracePolicy : { ...forwardedTracePolicy, ceiling };
- if (decision === undefined) {
- return { ...resolvedContext, forwardedTracePolicy: narrowedForwardedTracePolicy };
- }
- return {
- ...resolvedContext,
- decision,
- forwardedTracePolicy: narrowedForwardedTracePolicy,
- };
-}
-
-/** Durable trace state backed by eve's serialized Workflow context. */
-export class ContextAgentTraceStateStore implements AgentTraceStateStore {
- deleteAction(idempotencyKey: string): void {
- updateState((state) => {
- const actions = { ...state.actions };
- delete actions[idempotencyKey];
- return { ...state, actions };
- });
- }
-
- deleteActionAnchors(sessionId: string): void {
- updateState((state) => ({
- ...state,
- actionAnchors: Object.fromEntries(
- Object.entries(state.actionAnchors).filter(([, anchor]) => anchor.sessionId !== sessionId),
- ),
- }));
- }
-
- deleteActions(sessionId: string, turnId?: string): void {
- updateState((state) => {
- const actions = { ...state.actions };
- for (const [key, action] of Object.entries(actions)) {
- if (action.sessionId === sessionId && (turnId === undefined || action.turnId === turnId)) {
- delete actions[key];
- }
- }
- return { ...state, actions };
- });
- }
-
- deleteSession(sessionId: string): void {
- updateState((state) => {
- const sessions = { ...state.sessions };
- delete sessions[sessionId];
- return { ...state, sessions };
- });
- }
-
- deleteTurn(sessionId: string, turnId: string): void {
- updateState((state) => {
- const turns = { ...state.turns };
- delete turns[turnKey(sessionId, turnId)];
- return { ...state, turns };
- });
- }
-
- findAction(sessionId: string, callId: string): AgentActionTraceState | undefined {
- return Object.values(contextStorage.getStore()?.get(AgentTraceContextKey)?.actions ?? {}).find(
- (state) => state.sessionId === sessionId && state.callId === callId,
- );
- }
-
- findActionAnchor(
- sessionId: string,
- turnId: string,
- callId: string,
- ): AgentActionTraceState | undefined {
- return Object.values(
- contextStorage.getStore()?.get(AgentTraceContextKey)?.actionAnchors ?? {},
- ).find(
- (anchor) =>
- anchor.sessionId === sessionId && anchor.turnId === turnId && anchor.callId === callId,
- );
- }
-
- getAction(idempotencyKey: string): AgentActionTraceState | undefined {
- return contextStorage.getStore()?.get(AgentTraceContextKey)?.actions[idempotencyKey];
- }
-
- getSession(sessionId: string): AgentSessionTraceState | undefined {
- return contextStorage.getStore()?.get(AgentTraceContextKey)?.sessions[sessionId];
- }
-
- getTurn(sessionId: string, turnId: string): AgentTurnTraceState | undefined {
- return contextStorage.getStore()?.get(AgentTraceContextKey)?.turns[turnKey(sessionId, turnId)];
- }
-
- setAction(idempotencyKey: string, value: AgentActionTraceState): void {
- updateState((state) => ({
- ...state,
- actions: { ...state.actions, [idempotencyKey]: value },
- }));
- }
-
- setActionAnchor(idempotencyKey: string, value: AgentActionTraceState): void {
- updateState((state) => ({
- ...state,
- actionAnchors: { ...state.actionAnchors, [idempotencyKey]: value },
- }));
- }
-
- setSession(sessionId: string, value: AgentSessionTraceState): void {
- updateState((state) => ({
- ...state,
- sessions: { ...state.sessions, [sessionId]: value },
- }));
- }
-
- setTurn(sessionId: string, turnId: string, value: AgentTurnTraceState): void {
- updateState((state) => ({
- ...state,
- turns: { ...state.turns, [turnKey(sessionId, turnId)]: value },
- }));
- }
-
- updateTurn(
- sessionId: string,
- turnId: string,
- update: (state: AgentTurnTraceState) => AgentTurnTraceState,
- ): void {
- updateState((state) => {
- const key = turnKey(sessionId, turnId);
- const current = state.turns[key];
- return current === undefined
- ? state
- : { ...state, turns: { ...state.turns, [key]: update(current) } };
- });
- }
-}
-
-function updateState(update: (state: AgentTraceContextState) => AgentTraceContextState): void {
- loadContext().set(AgentTraceContextKey, (state) =>
- update(state ?? emptyAgentTraceContextState()),
- );
-}
-
-function turnKey(sessionId: string, turnId: string): string {
- return `${sessionId}\0${turnId}`;
-}
diff --git a/packages/eve/src/tracing/agent-trace-state.ts b/packages/eve/src/tracing/agent-trace-state.ts
deleted file mode 100644
index 74278d1a3..000000000
--- a/packages/eve/src/tracing/agent-trace-state.ts
+++ /dev/null
@@ -1,116 +0,0 @@
-import type { Attributes, SpanContext } from "#compiled/@opentelemetry/api/index.js";
-
-import type {
- InstrumentationToolCallKind,
- InstrumentationParentLineage,
- InstrumentationPrincipalSummary,
- InstrumentationTraceContext,
- InstrumentationTurnFailedEvent,
- InstrumentationTurnSettledEvent,
-} from "#instrumentation/lifecycle.js";
-import type { ChannelAudience } from "#shared/channel-audience.js";
-import type { InstrumentationDecision } from "#shared/instrumentation-decision.js";
-
-export interface AgentSessionTraceState {
- readonly traceSessionId: string;
- readonly channelAudience?: ChannelAudience;
- readonly agentName?: string;
- readonly channelKind?: string;
- readonly channelType?: string;
- readonly context: SpanContext;
- readonly decision?: InstrumentationDecision;
- readonly parentLineage?: InstrumentationParentLineage;
- readonly rootSessionId: string;
- readonly scheduleId?: string;
- readonly title?: string;
-}
-
-export interface AgentTurnTraceState {
- readonly traceSessionId: string;
- readonly caller?: SpanContext;
- readonly channelDelivery?: AgentTurnChannelDeliveryTraceState;
- readonly context: SpanContext;
- readonly currentPrincipal?: InstrumentationPrincipalSummary;
- readonly initiatorPrincipal?: InstrumentationPrincipalSummary;
- readonly parentLineage?: InstrumentationParentLineage;
- /** First model input projected onto the turn-level invoke_agent span. */
- readonly inputMessagesAttribute?: string;
- readonly modelUsage?: { readonly inputTokens?: number; readonly outputTokens?: number };
- /** Latest model output projected onto the turn-level invoke_agent span. */
- readonly outputMessagesAttribute?: string;
- readonly rootSessionId: string;
- readonly sequence: number;
- readonly startTimeMs: number;
- readonly subagentName?: string;
- readonly terminal?:
- | { readonly error: unknown; readonly type: InstrumentationTurnFailedEvent["type"] }
- | { readonly type: InstrumentationTurnSettledEvent["type"] };
-}
-
-export interface AgentTurnChannelDeliveryTraceState {
- readonly channelKind: string;
- readonly channelName: string;
- readonly deliveryId: string;
- readonly inputAttribute?: string;
- readonly requestId?: string;
- readonly requestTraceContext?: SpanContext;
-}
-
-export interface AgentActionTraceState {
- readonly traceSessionId: string;
- readonly attemptIndex: number;
- readonly callId: string;
- readonly channelAudience?: ChannelAudience;
- readonly inputAttribute?: string;
- readonly kind: InstrumentationToolCallKind;
- readonly name: string;
- readonly parent: InstrumentationTraceContext;
- readonly rootSessionId: string;
- readonly sessionId: string;
- readonly spanId: string;
- readonly startTimeMs: number;
- readonly stepIndex: number;
- readonly turnId: string;
- readonly toolAttributes?: Attributes;
- readonly toolEndTimeMs?: number;
- readonly toolFailed?: boolean;
- readonly toolErrorAttribute?: string;
-}
-
-/** Provider-owned serializable storage for durable agent trace state. */
-export interface AgentTraceStateStore {
- deleteAction(idempotencyKey: string): void | PromiseLike<void>;
- deleteActionAnchors(sessionId: string): void | PromiseLike<void>;
- deleteActions(sessionId: string, turnId?: string): void | PromiseLike<void>;
- deleteSession(sessionId: string): void | PromiseLike<void>;
- deleteTurn(sessionId: string, turnId: string): void | PromiseLike<void>;
- findAction(
- sessionId: string,
- callId: string,
- ): AgentActionTraceState | undefined | PromiseLike<AgentActionTraceState | undefined>;
- findActionAnchor(
- sessionId: string,
- turnId: string,
- callId: string,
- ): AgentActionTraceState | undefined | PromiseLike<AgentActionTraceState | undefined>;
- getAction(
- idempotencyKey: string,
- ): AgentActionTraceState | undefined | PromiseLike<AgentActionTraceState | undefined>;
- getSession(
- sessionId: string,
- ): AgentSessionTraceState | undefined | PromiseLike<AgentSessionTraceState | undefined>;
- getTurn(
- sessionId: string,
- turnId: string,
- ): AgentTurnTraceState | undefined | PromiseLike<AgentTurnTraceState | undefined>;
- setAction(idempotencyKey: string, state: AgentActionTraceState): void | PromiseLike<void>;
- setActionAnchor(idempotencyKey: string, state: AgentActionTraceState): void | PromiseLike<void>;
- setSession(sessionId: string, state: AgentSessionTraceState): void | PromiseLike<void>;
- setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void | PromiseLike<void>;
- /** Atomically updates an existing turn and does nothing after that turn is deleted. */
- updateTurn(
- sessionId: string,
- turnId: string,
- update: (state: AgentTurnTraceState) => AgentTurnTraceState,
- ): void | PromiseLike<void>;
-}
diff --git a/packages/eve/src/tracing/bounded-error.ts b/packages/eve/src/tracing/bounded-error.ts
deleted file mode 100644
index c457c476b..000000000
--- a/packages/eve/src/tracing/bounded-error.ts
+++ /dev/null
@@ -1,8 +0,0 @@
-import { truncateTelemetryText } from "#tracing/telemetry-budget.js";
-
-export function boundedTraceError(value: Error): Error {
- const error = new Error(truncateTelemetryText(value.message, 4096));
- error.name = truncateTelemetryText(value.name, 128);
- error.stack = value.stack === undefined ? undefined : truncateTelemetryText(value.stack, 8192);
- return error;
-}
diff --git a/packages/eve/src/tracing/content-attributes.test.ts b/packages/eve/src/tracing/content-attributes.test.ts
deleted file mode 100644
index f60c580ea..000000000
--- a/packages/eve/src/tracing/content-attributes.test.ts
+++ /dev/null
@@ -1,125 +0,0 @@
-import { describe, expect, it } from "vitest";
-
-import { withoutDeclinedContent } from "#tracing/content-attributes.js";
-
-const ATTRIBUTES = {
- "agent.channel.delivery.input": '{"message":"private"}',
- "agent.session.title": "Private title",
- "agent.trace.content.input": true,
- "agent.trace.content.output": true,
- "gen_ai.input.messages": "what the user said",
- "gen_ai.system_instructions": '[{"content":"private instructions","type":"text"}]',
- "ai.response.finish_reason": "stop",
- "ai.response.text": "what the model said",
- "gen_ai.request.model": "test-model",
- "gen_ai.tool.call.arguments": "{}",
- "gen_ai.tool.call.result": "42",
- "gen_ai.tool.name": "weather",
-};
-
-describe("withoutDeclinedContent", () => {
- it("keeps everything when the destination declined nothing", () => {
- expect(
- withoutDeclinedContent(ATTRIBUTES, { recordInputs: true, recordOutputs: true }),
- ).toBeUndefined();
- });
-
- it("keeps everything when the span carries none of what was declined", () => {
- expect(
- withoutDeclinedContent(
- { "gen_ai.request.model": "test-model" },
- { recordInputs: false, recordOutputs: false },
- ),
- ).toBeUndefined();
- });
-
- it("drops inputs alone", () => {
- expect(
- withoutDeclinedContent(ATTRIBUTES, { recordInputs: false, recordOutputs: true }),
- ).toEqual({
- "agent.trace.content.input": false,
- "agent.trace.content.output": true,
- "ai.response.finish_reason": "stop",
- "ai.response.text": "what the model said",
- "gen_ai.request.model": "test-model",
- "gen_ai.tool.call.result": "42",
- "gen_ai.tool.name": "weather",
- });
- });
-
- it("drops outputs alone", () => {
- expect(
- withoutDeclinedContent(ATTRIBUTES, { recordInputs: true, recordOutputs: false }),
- ).toEqual({
- "agent.channel.delivery.input": '{"message":"private"}',
- "agent.session.title": "Private title",
- "agent.trace.content.input": true,
- "agent.trace.content.output": false,
- "gen_ai.input.messages": "what the user said",
- "gen_ai.system_instructions": '[{"content":"private instructions","type":"text"}]',
- "ai.response.finish_reason": "stop",
- "gen_ai.request.model": "test-model",
- "gen_ai.tool.call.arguments": "{}",
- "gen_ai.tool.name": "weather",
- });
- });
-
- it("drops system instructions when the destination declined inputs", () => {
- expect(
- withoutDeclinedContent(
- { "gen_ai.system_instructions": '[{"content":"private","type":"text"}]' },
- { recordInputs: false, recordOutputs: true },
- ),
- ).toEqual({});
- });
-
- // The prefixes are shared: `ai.response.finish_reason` and `gen_ai.tool.name`
- // say what happened rather than what was said, so declining content cannot
- // cost a destination the ability to read its own traces.
- it("keeps metadata that shares a prefix with content", () => {
- expect(
- withoutDeclinedContent(ATTRIBUTES, { recordInputs: false, recordOutputs: false }),
- ).toEqual({
- "agent.trace.content.input": false,
- "agent.trace.content.output": false,
- "ai.response.finish_reason": "stop",
- "gen_ai.request.model": "test-model",
- "gen_ai.tool.name": "weather",
- });
- });
-
- it("leaves the attributes it was handed alone", () => {
- const attributes = { ...ATTRIBUTES };
- withoutDeclinedContent(attributes, { recordInputs: false, recordOutputs: false });
- expect(attributes).toEqual(ATTRIBUTES);
- });
-
- it("narrows content policy attributes even when no content is present", () => {
- expect(
- withoutDeclinedContent(
- {
- "agent.trace.content.input": true,
- "agent.trace.content.output": true,
- },
- { recordInputs: false, recordOutputs: true },
- ),
- ).toEqual({
- "agent.trace.content.input": false,
- "agent.trace.content.output": true,
- });
- });
-
- it("redacts recalled memory records as input content", () => {
- const searched = {
- "gen_ai.memory.records": '[{"content":"Private preference"}]',
- "gen_ai.operation.name": "search_memory",
- };
-
- expect(withoutDeclinedContent(searched, { recordInputs: false, recordOutputs: true })).toEqual({
- "gen_ai.operation.name": "search_memory",
- });
- expect(
- withoutDeclinedContent(searched, { recordInputs: true, recordOutputs: false }),
- ).toBeUndefined();
- });
-});
diff --git a/packages/eve/src/tracing/content-attributes.ts b/packages/eve/src/tracing/content-attributes.ts
deleted file mode 100644
index 2409ab0cf..000000000
--- a/packages/eve/src/tracing/content-attributes.ts
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Which span attributes carry conversation content, and in which direction.
- *
- * Two vocabularies land on the same spans: the ones eve sets on its own
- * `agent.*` spans, and the ones the AI SDK's OpenTelemetry integration sets on
- * the model-call spans beneath them. A destination that declined content has to
- * be rid of both, so both are listed here rather than in the module that writes
- * each.
- *
- * Listed by name rather than by prefix because the prefixes are shared with
- * metadata that must survive: `ai.response.finish_reason` and
- * `gen_ai.tool.name` say what happened, not what was said. The cost is that a
- * new content attribute in a dependency is not covered until it is added here.
- */
-
-/** Prompts, instructions, tool arguments — what went in. */
-const INPUT_CONTENT_ATTRIBUTES: ReadonlySet<string> = new Set([
- "agent.channel.delivery.input",
- "agent.session.title",
- "agent.approval.request",
- "ai.documents",
- "ai.prompt",
- "ai.prompt.toolChoice",
- "ai.prompt.tools",
- "ai.value",
- "ai.values",
- "gen_ai.input.messages",
- "gen_ai.memory.records",
- "gen_ai.system_instructions",
- "gen_ai.tool.call.arguments",
- "gen_ai.tool.definitions",
-]);
-
-/**
- * Responses, reasoning, tool results — what came out.
- *
- * `ai.toolCall.args` is here rather than above because the AI SDK gates it on
- * `recordOutputs`; matching that is what keeps one destination's view coherent.
- */
-const OUTPUT_CONTENT_ATTRIBUTES: ReadonlySet<string> = new Set([
- "agent.approval.response",
- "ai.embedding",
- "ai.embeddings",
- "ai.ranking",
- "ai.response.files",
- "ai.response.object",
- "ai.response.reasoning",
- "ai.response.text",
- "ai.response.toolCalls",
- "ai.response.tool_calls",
- "ai.response.tool_results",
- "ai.toolCall.args",
- "ai.toolCall.result",
- "gen_ai.output.messages",
- "gen_ai.tool.call.result",
-]);
-
-const INPUT_POLICY_ATTRIBUTE = "agent.trace.content.input";
-const OUTPUT_POLICY_ATTRIBUTE = "agent.trace.content.output";
-
-/** What one destination is willing to receive. */
-export interface ResolvedContentOptions {
- readonly recordInputs: boolean;
- readonly recordOutputs: boolean;
-}
-
-function isDeclined(key: string, content: ResolvedContentOptions): boolean {
- if (!content.recordInputs && INPUT_CONTENT_ATTRIBUTES.has(key)) return true;
- return !content.recordOutputs && OUTPUT_CONTENT_ATTRIBUTES.has(key);
-}
-
-function narrowsContentPolicy(
- attributes: Readonly<Record<string, unknown>>,
- content: ResolvedContentOptions,
-): boolean {
- return (
- (!content.recordInputs && attributes[INPUT_POLICY_ATTRIBUTE] === true) ||
- (!content.recordOutputs && attributes[OUTPUT_POLICY_ATTRIBUTE] === true)
- );
-}
-
-function exportedAttributeValue(
- key: string,
- value: unknown,
- content: ResolvedContentOptions,
-): unknown {
- if (key === INPUT_POLICY_ATTRIBUTE && value === true && !content.recordInputs) return false;
- if (key === OUTPUT_POLICY_ATTRIBUTE && value === true && !content.recordOutputs) return false;
- return value;
-}
-
-/**
- * The attributes with the declined content removed, or `undefined` when there
- * was none to remove.
- *
- * The `undefined` return is what lets the caller forward the original span
- * untouched in the common case, so a destination that declined a direction the
- * span never carried costs one pass over its keys and no allocation.
- */
-export function withoutDeclinedContent(
- attributes: Readonly<Record<string, unknown>>,
- content: ResolvedContentOptions,
-): Record<string, unknown> | undefined {
- const keys = Object.keys(attributes);
- if (!keys.some((key) => isDeclined(key, content)) && !narrowsContentPolicy(attributes, content)) {
- return undefined;
- }
-
- const kept: Record<string, unknown> = {};
- for (const key of keys) {
- if (!isDeclined(key, content)) {
- kept[key] = exportedAttributeValue(key, attributes[key], content);
- }
- }
- return kept;
-}
diff --git a/packages/eve/src/tracing/error-content-context.ts b/packages/eve/src/tracing/error-content-context.ts
deleted file mode 100644
index d32415030..000000000
--- a/packages/eve/src/tracing/error-content-context.ts
+++ /dev/null
@@ -1,11 +0,0 @@
-import { context, createContextKey, type Context } from "#compiled/@opentelemetry/api/index.js";
-
-const ERROR_CONTENT_KEY = createContextKey("eve.trace.error-content");
-
-export function withErrorContent(context: Context, allowed: boolean): Context {
- return context.setValue(ERROR_CONTENT_KEY, allowed);
-}
-
-export function capturesErrorContent(): boolean {
- return context.active().getValue(ERROR_CONTENT_KEY) !== false;
-}
diff --git a/packages/eve/src/tracing/eve/agent-approval-instrumentation.ts b/packages/eve/src/tracing/eve/agent-approval-instrumentation.ts
new file mode 100644
index 000000000..b641ed8bc
--- /dev/null
+++ b/packages/eve/src/tracing/eve/agent-approval-instrumentation.ts
@@ -0,0 +1,63 @@
+import type {
+ InstrumentationHandlerContext,
+ InstrumentationInputRequestedEvent,
+ InstrumentationInputResolvedEvent,
+ InstrumentationProviderDefinition,
+} from "#instrumentation/lifecycle.js";
+import { resumeTool } from "#tracing/eve/agent-tool-instrumentation.js";
+import type { AgentActionTraceState } from "#tracing/eve/agent-trace-state.js";
+import type { AgentTracing } from "#compiled/@eve/tracing/index.js";
+
+/** Approval spans live under their tool call; hook state keeps only the locator. */
+export function createAgentApprovalInstrumentation(input: {
+ readonly tracing: AgentTracing;
+ readonly actionStateFor: (
+ sessionId: string,
+ turnId: string,
+ callId: string,
+ ) => Promise<AgentActionTraceState | undefined>;
+}): Pick<
+ NonNullable<InstrumentationProviderDefinition["events"]>,
+ "input.requested" | "input.resolved"
+> {
+ async function toolFor(sessionId: string, turnId: string, callId: string) {
+ const state = await input.actionStateFor(sessionId, turnId, callId);
+ return state === undefined ? undefined : resumeTool(input.tracing, state);
+ }
+
+ async function onRequested(
+ event: InstrumentationInputRequestedEvent,
+ ctx: InstrumentationHandlerContext,
+ ): Promise<void> {
+ if (event.kind !== "tool-approval" || ctx.state.get() !== undefined) return;
+ const tool = await toolFor(event.scope.sessionId, event.scope.turnId, event.action.callId);
+ const approval = await tool?.approval({ requestId: event.requestId, request: event.request });
+ if (approval === undefined) return;
+ ctx.state.set({
+ callId: event.action.callId,
+ sessionId: event.scope.sessionId,
+ turnId: event.scope.turnId,
+ });
+ }
+
+ async function onResolved(
+ event: InstrumentationInputResolvedEvent,
+ ctx: InstrumentationHandlerContext,
+ ): Promise<void> {
+ const stored = ctx.state.get();
+ if (stored === null || typeof stored !== "object" || Array.isArray(stored)) return;
+ const { callId, sessionId, turnId } = stored as Record<string, unknown>;
+ if (typeof callId !== "string" || typeof sessionId !== "string" || typeof turnId !== "string")
+ return;
+ const approval = (await toolFor(sessionId, turnId, callId))?.findApproval(event.requestId);
+ if (approval === undefined) return;
+ if (event.outcome === "failed") await approval.fail(event.error);
+ else
+ await approval.complete({
+ outcome: event.outcome as "approved" | "denied" | "cancelled" | "ignored" | "invalid",
+ response: event.response,
+ });
+ }
+
+ return { "input.requested": onRequested, "input.resolved": onResolved };
+}
diff --git a/packages/eve/src/tracing/eve/agent-channel-delivery-instrumentation.ts b/packages/eve/src/tracing/eve/agent-channel-delivery-instrumentation.ts
index 599afe3af..66e09242a 100644
--- a/packages/eve/src/tracing/eve/agent-channel-delivery-instrumentation.ts
+++ b/packages/eve/src/tracing/eve/agent-channel-delivery-instrumentation.ts
@@ -9,9 +9,9 @@ import type {
import { contextStorage } from "#context/container.js";
import { ActiveChannelDeliveriesKey } from "#context/keys.js";
import type { JsonValue } from "#shared/json.js";
-import { contentAttribute } from "#tracing/agent-otel-content.js";
-import type { AgentTraceStateStore } from "#tracing/agent-trace-state.js";
-import { isSampledTrace } from "#tracing/sampled-trace.js";
+import { contentAttribute } from "#compiled/@eve/tracing/otel.js";
+import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";
+import { isSampledTrace } from "#shared/trace-policy.js";
interface ChannelDeliveryState {
readonly inputAttribute?: string;
@@ -33,11 +33,11 @@ export function createAgentChannelDeliveryInstrumentation(input: {
event: InstrumentationChannelDeliveryStartedEvent,
ctx: InstrumentationHandlerContext,
): Promise<void> => {
- const session = await input.stateStore.getSession(event.sessionId);
+ const session = await input.stateStore.get("session", event.sessionId);
const turn =
event.turnId === undefined || event.sequence === undefined
? undefined
- : await input.stateStore.getTurn(event.sessionId, event.turnId);
+ : await input.stateStore.get("turn", JSON.stringify([event.sessionId, event.turnId]));
const traceContext = turn?.context ?? session?.context;
if (traceContext === undefined || !isSampledTrace(traceContext)) return;
const inputAttribute = input.recordInputs ? contentAttribute(event.input) : undefined;
@@ -59,7 +59,10 @@ export function createAgentChannelDeliveryInstrumentation(input: {
): Promise<void> => {
const state = readState(ctx.state.get());
if (state === undefined || event.turnId === undefined) return;
- const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);
+ const turn = await input.stateStore.get(
+ "turn",
+ JSON.stringify([event.sessionId, event.turnId]),
+ );
if (
turn === undefined ||
!isSampledTrace(turn.context) ||
@@ -67,23 +70,27 @@ export function createAgentChannelDeliveryInstrumentation(input: {
) {
return;
}
- await input.stateStore.updateTurn(event.sessionId, event.turnId, (current) => ({
- ...current,
- channelDelivery: {
- channelKind: event.delivery.channelKind,
- channelName: event.delivery.channelName,
- deliveryId: event.delivery.deliveryId,
- ...(state.inputAttribute === undefined
- ? undefined
- : { inputAttribute: state.inputAttribute }),
- ...(event.delivery.requestId === undefined
- ? undefined
- : { requestId: event.delivery.requestId }),
- ...(state.requestTraceContext === undefined
- ? undefined
- : { requestTraceContext: state.requestTraceContext }),
- },
- }));
+ await input.stateStore.update(
+ "turn",
+ JSON.stringify([event.sessionId, event.turnId]),
+ (current) => ({
+ ...current,
+ channelDelivery: {
+ channelKind: event.delivery.channelKind,
+ channelName: event.delivery.channelName,
+ deliveryId: event.delivery.deliveryId,
+ ...(state.inputAttribute === undefined
+ ? undefined
+ : { inputAttribute: state.inputAttribute }),
+ ...(event.delivery.requestId === undefined
+ ? undefined
+ : { requestId: event.delivery.requestId }),
+ ...(state.requestTraceContext === undefined
+ ? undefined
+ : { requestTraceContext: state.requestTraceContext }),
+ },
+ }),
+ );
};
return {
diff --git a/packages/eve/src/tracing/eve/agent-invocation-coordinator.test.ts b/packages/eve/src/tracing/eve/agent-invocation-coordinator.test.ts
index 556873179..41ca2382d 100644
--- a/packages/eve/src/tracing/eve/agent-invocation-coordinator.test.ts
+++ b/packages/eve/src/tracing/eve/agent-invocation-coordinator.test.ts
@@ -4,7 +4,7 @@ import { ContextContainer, contextStorage } from "#context/container.js";
import { SessionTraceSeedKey } from "#context/keys.js";
import { serializeContext } from "#context/serialize.js";
import { resolveToolCallAgentTrace } from "#tracing/eve/agent-invocation-coordinator.js";
-import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";
+import { ContextAgentTraceStateStore } from "#tracing/eve/agent-trace-context-store.js";
import { actionIdempotencyKey } from "#instrumentation/lifecycle.js";
import type { ConversationContext } from "#shared/conversation-context.js";
@@ -32,14 +32,14 @@ describe("resolveToolCallAgentTrace", () => {
});
await contextStorage.run(context, () => {
const store = new ContextAgentTraceStateStore();
- store.setTurn("session-1", "turn-1", {
+ store.set("turn", JSON.stringify(["session-1", "turn-1"]), {
context: action.parent,
rootSessionId: "session-1",
traceSessionId: "session-1",
sequence: 0,
startTimeMs: 1,
});
- if (recorded) store.setActionAnchor(actionKey, action);
+ if (recorded) store.set("anchor", actionKey, action);
});
const dispatch = resolveToolCallAgentTrace({
@@ -52,7 +52,7 @@ describe("resolveToolCallAgentTrace", () => {
expect(dispatch.parentTraceContext).toMatchObject({
...action.parent,
- spanId: recorded ? action.spanId : action.parent.spanId,
+ spanId: recorded ? action.context.spanId : action.parent.spanId,
decision: { action: "record", recordInputs: false, recordOutputs: false },
});
expect(dispatch.originAudience).toBe("public");
@@ -64,6 +64,7 @@ function workflowAction() {
return {
attemptIndex: 0,
callId: "workflow",
+ context: { spanId: "3".repeat(16), traceId: "1".repeat(32), traceFlags: 1 },
channelAudience: "private" as const,
kind: "tool-call" as const,
name: "coordinate",
@@ -73,10 +74,7 @@ function workflowAction() {
traceId: "1".repeat(32),
},
rootSessionId: "session-1",
- traceSessionId: "session-1",
sessionId: "session-1",
- spanId: "3".repeat(16),
- startTimeMs: 1,
stepIndex: 0,
turnId: "turn-1",
};
diff --git a/packages/eve/src/tracing/eve/agent-invocation-coordinator.ts b/packages/eve/src/tracing/eve/agent-invocation-coordinator.ts
index 936891113..dfc7d615f 100644
--- a/packages/eve/src/tracing/eve/agent-invocation-coordinator.ts
+++ b/packages/eve/src/tracing/eve/agent-invocation-coordinator.ts
@@ -10,7 +10,7 @@ import {
import {
readActionTraceContext,
readTurnTraceContext,
-} from "#tracing/agent-trace-context-store.js";
+} from "#tracing/eve/agent-trace-context-store.js";
export interface AgentChildTraceDispatch {
readonly conversationId?: string;
@@ -31,22 +31,9 @@ export function resolveToolCallAgentTrace(input: {
readonly turnId: string;
}): AgentChildTraceDispatch {
const { serializedContext, sessionId, turnId } = input;
- return toChildTraceDispatch({
- conversation: input.conversation,
- conversationId: readConversationId(serializedContext[ConversationIdKey.name]),
- stored:
- readActionTraceContext(serializedContext, sessionId, turnId, input.callId) ??
- readTurnTraceContext(serializedContext, sessionId, turnId),
- });
-}
-
-/** Caps a stored parent trace context to the live delivery audience. */
-function toChildTraceDispatch(input: {
- readonly conversation?: ConversationContext;
- readonly conversationId?: string;
- readonly stored?: SessionTraceContext;
-}): AgentChildTraceDispatch {
- const { stored } = input;
+ const stored =
+ readActionTraceContext(serializedContext, sessionId, turnId, input.callId) ??
+ readTurnTraceContext(serializedContext, sessionId, turnId);
const liveAudience = input.conversation?.audience ?? "unknown";
const environment = input.conversation?.environment ?? "production";
const forwardedTracePolicy = readForwardedTraceAssertion(stored?.forwardedTracePolicy);
@@ -63,7 +50,7 @@ function toChildTraceDispatch(input: {
),
};
return {
- conversationId: input.conversationId,
+ conversationId: readConversationId(serializedContext[ConversationIdKey.name]),
originAudience: forwardedTracePolicy?.originAudience ?? liveAudience,
parentTraceContext,
};
diff --git a/packages/eve/src/tracing/eve/agent-memory-instrumentation.ts b/packages/eve/src/tracing/eve/agent-memory-instrumentation.ts
new file mode 100644
index 000000000..d51f69b5e
--- /dev/null
+++ b/packages/eve/src/tracing/eve/agent-memory-instrumentation.ts
@@ -0,0 +1,85 @@
+import { ROOT_CONTEXT, context, trace } from "#compiled/@opentelemetry/api/index.js";
+import { contextStorage } from "#context/container.js";
+import { SessionTraceSeedKey } from "#context/keys.js";
+import type {
+ InstrumentationMemoryExecutionOperation,
+ MemoryInstrumentationResult,
+} from "#instrumentation/memory.js";
+import { normalizeChannelAudience } from "#shared/channel-audience.js";
+import {
+ applyLiveDeliveryAudienceCeiling,
+ resolveForwardedTraceSeed,
+} from "#shared/forwarded-trace-policy.js";
+import type { ConversationEnvironment } from "#shared/conversation-context.js";
+import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";
+import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";
+import { eveTurnIdentity } from "#tracing/eve/operation-input.js";
+import { type AgentTracing } from "#compiled/@eve/tracing/index.js";
+import { activeTraceOperation } from "#compiled/@eve/tracing/otel.js";
+
+export function createAgentMemoryInstrumentation(input: {
+ tracing: AgentTracing;
+ environment: ConversationEnvironment;
+ recordOutputs?: boolean;
+ stateStore: AgentTraceStateStore;
+}) {
+ return {
+ async runInContext<T>(
+ operation: InstrumentationMemoryExecutionOperation,
+ execute: () => PromiseLike<T>,
+ ): Promise<T> {
+ const session = await input.stateStore.get("session", operation.sessionId);
+ const turn =
+ operation.turnId === undefined
+ ? undefined
+ : await input.stateStore.get(
+ "turn",
+ JSON.stringify([operation.sessionId, operation.turnId]),
+ );
+ const active = activeTraceOperation();
+ const reference = active?.reference ?? turn?.context ?? session?.context;
+ if (reference === undefined) return await execute();
+ const host =
+ active === undefined
+ ? withChannelAudience(
+ trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(reference)),
+ session?.channelAudience,
+ )
+ : context.active();
+ const seed = resolveForwardedTraceSeed(contextStorage.getStore()?.get(SessionTraceSeedKey));
+ const decision = seed?.decision ?? session?.decision;
+ const effective =
+ decision === undefined
+ ? undefined
+ : applyLiveDeliveryAudienceCeiling(
+ decision,
+ normalizeChannelAudience(session?.channelAudience),
+ seed?.forwardedTracePolicy,
+ input.environment,
+ );
+ const capture = {
+ emit: (reference.traceFlags & 1) !== 0,
+ recordInputs: effective?.action === "record" && effective.recordInputs,
+ recordOutputs:
+ input.recordOutputs === true && effective?.action === "record" && effective.recordOutputs,
+ };
+ const data = {
+ identity: eveTurnIdentity({ ...operation, turnId: operation.turnId ?? "" }),
+ parent: reference,
+ context: host,
+ capture,
+ phase: operation.phase,
+ slot: operation.slot,
+ storeId: operation.storeId,
+ describe(value: T) {
+ const result = value as MemoryInstrumentationResult<unknown>;
+ return { recordCount: result.recordCount, records: result.outputRecords };
+ },
+ };
+ const memory = input.tracing.memory;
+ return operation.operationName === "search_memory"
+ ? await memory.search(data, execute)
+ : await memory.write(data, execute);
+ },
+ };
+}
diff --git a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts b/packages/eve/src/tracing/eve/agent-otel-provider.test.ts
index 2ba86ad5d..d408c4fa7 100644
--- a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts
+++ b/packages/eve/src/tracing/eve/agent-otel-provider.test.ts
@@ -1,4 +1,10 @@
-import { SpanKind, SpanStatusCode, trace as apiTrace } from "@opentelemetry/api";
+import {
+ SpanKind,
+ SpanStatusCode,
+ context as apiContext,
+ trace as apiTrace,
+} from "@opentelemetry/api";
+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
import {
BasicTracerProvider,
InMemorySpanExporter,
@@ -25,10 +31,12 @@ import { createActionResultEvent, createActionsRequestedEvent } from "#protocol/
import {
createAgentOtelInstrumentation,
type AgentOtelInstrumentationInput,
-} from "#tracing/agent-otel-provider.js";
-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
-import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";
-import type { AgentTraceStateStore } from "#tracing/agent-trace-state.js";
+} from "#tracing/eve/agent-otel-provider.js";
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
+import { otelTelemetry } from "#compiled/@eve/tracing/index.js";
+import { eveOutputMapping } from "#tracing/eve/profile.js";
+import { ContextAgentTraceStateStore } from "#tracing/eve/agent-trace-context-store.js";
+import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";
import { InMemoryAgentTraceStateStore } from "#internal/testing/in-memory-agent-trace-state-store.js";
import {
createInstrumentationHooks,
@@ -44,14 +52,10 @@ import {
} from "#instrumentation/lifecycle.js";
import type { ChannelAudience } from "#shared/channel-audience.js";
import { channelAudienceFromContext } from "#tracing/eve/channel-audience-context.js";
-import {
- agentToolContentPolicy,
- agentToolSpanContext,
- annotateAgentToolSpan,
-} from "#tracing/agent-tool-span-context.js";
+import { activeTraceOperation } from "#compiled/@eve/tracing/otel.js";
import { contentFilteringProcessor } from "#tracing/eve/content-span-processor.js";
import { parseLocalTraceSegment } from "#tracing/local/trace-reader.js";
-import { CONTENT_ATTRIBUTE_LIMIT } from "#tracing/agent-otel-content.js";
+import { CONTENT_ATTRIBUTE_LIMIT } from "#compiled/@eve/tracing/otel.js";
import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";
import type { TraceCaptureContext } from "#shared/trace-policy.js";
import {
@@ -95,7 +99,7 @@ function createRuntime(
recordOutputs: true,
}),
extraSpanProcessors: readonly SpanProcessor[] = [],
- samplesTrace?: AgentOtelInstrumentationInput["samplesTrace"],
+ samplesTrace?: import("#tracing/eve/otel-registration.js").RegisteredOtelPipeline["samplesTrace"],
): TestRuntime {
const exporter = new InMemorySpanExporter();
const idGenerator = new AgentSpanIdGenerator();
@@ -110,12 +114,17 @@ function createRuntime(
// Pin the environment so an inherited EVE_DEV/VERCEL_ENV cannot relax audience ceilings.
environment: "production",
frameworkVersion: "test",
- idGenerator,
recordInputs: true,
recordOutputs: true,
- samplesTrace,
stateStore,
- tracer,
+ telemetry: otelTelemetry({
+ provider,
+ tracerName: "eve.agent",
+ idGenerator,
+ samplesTrace: samplesTrace ?? (() => true),
+ mapping: eveOutputMapping(),
+ }),
+ idGenerator,
};
if (tracePolicy !== null) agentOtelInput.tracePolicy = tracePolicy;
const agentOtel = createAgentOtelInstrumentation(agentOtelInput);
@@ -493,54 +502,6 @@ describe("createAgentOtelInstrumentation", () => {
},
);
- it.each(["completed", "failed", "cancelled"] as const)(
- "exports the %s turn outcome after content redaction and event removal",
- async (outcome) => {
- const metadata = new InMemorySpanExporter();
- const runtime = createRuntime(undefined, undefined, [
- contentFilteringProcessor(new SimpleSpanProcessor(metadata), {
- span: () => ({ redact: true, inputs: true, outputs: true }),
- }),
- ]);
- await publishTurnStarted({
- hooks: runtime.hooks,
- sessionId: "session-1",
- turnId: "turn_0",
- turnSequence: 0,
- });
- const identity = {
- idempotencyKey: turnIdempotencyKey("session-1", "turn_0"),
- sessionId: "session-1",
- turnId: "turn_0",
- };
- await runtime.hooks.publish(
- outcome === "failed"
- ? { ...identity, type: "turn.failed", error: new Error("private failure") }
- : { ...identity, type: outcome === "completed" ? "turn.completed" : "turn.cancelled" },
- );
- await runtime.hooks.publish({
- idempotencyKey: sessionIdempotencyKey("session-1"),
- sessionId: "session-1",
- turnId: "turn_0",
- type: "session.waiting",
- });
- await runtime.provider.forceFlush();
- const spans = metadata.getFinishedSpans();
- const serialized = new TextDecoder().decode(
- JsonTraceSerializer.serializeRequest(
- spans.map((span) => ({ ...span, events: [], spanContext: () => span.spanContext() })),
- )!,
- );
- const turn = parseLocalTraceSegment(serialized, spans[0]!.spanContext().traceId).find(
- (span) => span.name === "invoke_agent weather",
- )!;
- expect(turn.attributes["agent.turn.outcome"]).toBe(outcome);
- expect(turn.statusCode).toBe(outcome === "failed" ? 2 : 0);
- expect(serialized).not.toContain("private failure");
- await runtime.provider.shutdown();
- },
- );
-
it("prepares a stable stream trace before lifecycle hooks observe the turn", async () => {
const runtime = createRuntime();
const sessionEvent = {
@@ -612,7 +573,7 @@ describe("createAgentOtelInstrumentation", () => {
turnId: "turn-1",
type: "turn.started",
});
- const turn = await store.getTurn("session-1", "turn-1");
+ const turn = await store.get("turn", JSON.stringify(["session-1", "turn-1"]));
expect(turn?.currentPrincipal).toStrictEqual(
includesId ? { id: "current-secret", type: "service" } : { type: "service" },
);
@@ -662,12 +623,14 @@ describe("createAgentOtelInstrumentation", () => {
const contextWith = vi.spyOn(context, "with");
await runtime.runInContext(
{
- idempotencyKey: operation.idempotencyKey,
- sessionId: operation.sessionId,
- turnId: operation.turnId,
+ ...operation,
type: "memory.operation",
},
- async () => undefined,
+ async () => ({
+ value: undefined,
+ recordCount: 1,
+ outputRecords: [{ content: "The user prefers dark mode.", id: "preference" }],
+ }),
);
const memoryContext = contextWith.mock.calls[0]?.[0];
contextWith.mockRestore();
@@ -847,7 +810,6 @@ describe("createAgentOtelInstrumentation", () => {
[],
samplesTrace,
);
- const setSession = vi.spyOn(store, "setSession");
const sessionId = "session-early";
await runtime.prepareSessionTrace({
agentName: "weather",
@@ -900,7 +862,6 @@ describe("createAgentOtelInstrumentation", () => {
"agent.session.origin": "schedule",
"agent.session.title": "Prepared before sampling",
});
- expect(setSession).toHaveBeenCalledTimes(1);
});
it("falls back to fresh ids when no trace seed is present", async () => {
@@ -1089,104 +1050,109 @@ describe("createAgentOtelInstrumentation", () => {
},
);
- it("maps one channel delivery onto its activation with an HTTP request link", async () => {
- const runtime = createRuntime();
- const ctx = new ContextContainer();
- const requestTraceContext = {
- spanId: "2222222222222222",
- traceFlags: 1,
- traceId: "22222222222222222222222222222222",
- };
- const started = {
- agentName: "support",
- delivery: {
+ it.each(["private", "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)])(
+ "maps a channel delivery onto its activation despite bounded content (case %#)",
+ async (message) => {
+ const runtime = createRuntime();
+ const ctx = new ContextContainer();
+ const requestTraceContext = {
+ spanId: "2222222222222222",
+ traceFlags: 1,
+ traceId: "22222222222222222222222222222222",
+ };
+ const started = {
+ agentName: "support",
+ delivery: {
+ channelAudience: "public" as const,
+ channelKind: "channel:slack",
+ channelName: "slack",
+ deliveryId: "delivery-1",
+ requestId: "iad1::request-1",
+ requestTraceContext,
+ },
+ idempotencyKey: "channel-delivery:session-1:delivery-1",
+ input: { message },
+ rootSessionId: "session-1",
+ sequence: 0,
+ sessionId: "session-1",
+ turnId: "turn_0",
+ type: "channel.delivery.started" as const,
+ };
+ const completed = {
+ agentName: started.agentName,
+ delivery: started.delivery,
+ idempotencyKey: started.idempotencyKey,
+ outcome: "completed" as const,
+ rootSessionId: started.rootSessionId,
+ sequence: 0,
+ sessionId: started.sessionId,
+ turnId: "turn_0",
+ type: "channel.delivery.completed" as const,
+ };
+ const sessionEvent = {
+ agentName: "support",
channelAudience: "public" as const,
channelKind: "channel:slack",
- channelName: "slack",
- deliveryId: "delivery-1",
- requestId: "iad1::request-1",
- requestTraceContext,
- },
- idempotencyKey: "channel-delivery:session-1:delivery-1",
- input: { message: "private" },
- rootSessionId: "session-1",
- sequence: 0,
- sessionId: "session-1",
- turnId: "turn_0",
- type: "channel.delivery.started" as const,
- };
- const completed = {
- agentName: started.agentName,
- delivery: started.delivery,
- idempotencyKey: started.idempotencyKey,
- outcome: "completed" as const,
- rootSessionId: started.rootSessionId,
- sequence: 0,
- sessionId: started.sessionId,
- turnId: "turn_0",
- type: "channel.delivery.completed" as const,
- };
- const sessionEvent = {
- agentName: "support",
- channelAudience: "public" as const,
- channelKind: "channel:slack",
- idempotencyKey: sessionIdempotencyKey("session-1"),
- rootSessionId: "session-1",
- sessionId: "session-1",
- type: "session.started" as const,
- };
- const turnEvent = {
- idempotencyKey: turnIdempotencyKey("session-1", "turn_0"),
- rootSessionId: "session-1",
- sequence: 0,
- sessionId: "session-1",
- turnId: "turn_0",
- type: "turn.started" as const,
- };
+ idempotencyKey: sessionIdempotencyKey("session-1"),
+ rootSessionId: "session-1",
+ sessionId: "session-1",
+ type: "session.started" as const,
+ };
+ const turnEvent = {
+ idempotencyKey: turnIdempotencyKey("session-1", "turn_0"),
+ rootSessionId: "session-1",
+ sequence: 0,
+ sessionId: "session-1",
+ turnId: "turn_0",
+ type: "turn.started" as const,
+ };
- await contextStorage.run(ctx, async () => {
- await runtime.prepareSessionTrace(sessionEvent);
- await runtime.prepareTurnTrace(turnEvent);
- await runtime.hooks.publish(started);
- await runtime.hooks.publish(sessionEvent);
- await runtime.hooks.publish(turnEvent);
- ctx.set(ActiveChannelDeliveriesKey, [
- {
- agentName: started.agentName,
- delivery: started.delivery,
- policyAgentName: started.agentName,
- rootSessionId: started.rootSessionId,
- traceSessionId: started.rootSessionId,
- sequence: 0,
- sessionId: started.sessionId,
- turnId: "turn_0",
- },
- ]);
- await runtime.hooks.publish(completed);
- await completeTurn(runtime.hooks, "session-1", "turn_0");
- });
+ await contextStorage.run(ctx, async () => {
+ await runtime.prepareSessionTrace(sessionEvent);
+ await runtime.prepareTurnTrace(turnEvent);
+ await runtime.hooks.publish(started);
+ await runtime.hooks.publish(sessionEvent);
+ await runtime.hooks.publish(turnEvent);
+ ctx.set(ActiveChannelDeliveriesKey, [
+ {
+ agentName: started.agentName,
+ delivery: started.delivery,
+ policyAgentName: started.agentName,
+ rootSessionId: started.rootSessionId,
+ traceSessionId: started.rootSessionId,
+ sequence: 0,
+ sessionId: started.sessionId,
+ turnId: "turn_0",
+ },
+ ]);
+ await runtime.hooks.publish(completed);
+ await completeTurn(runtime.hooks, "session-1", "turn_0");
+ });
- const spans = runtime.exporter.getFinishedSpans();
- const turn = spans.find((span) => span.name === "invoke_agent support")!;
- expect(byName(spans, "agent.channel.delivery")).toHaveLength(0);
- expect(turn.parentSpanContext).toBeUndefined();
- expect(turn.links).toEqual([
- expect.objectContaining({
- attributes: { "eve.link.type": "channel.request" },
- context: expect.objectContaining(requestTraceContext),
- }),
- ]);
- expect(turn.attributes).toMatchObject({
- "agent.channel.delivery.id": "delivery-1",
- "agent.channel.delivery.input": JSON.stringify({ message: "private" }),
- "agent.channel.kind": "channel:slack",
- "agent.channel.name": "slack",
- "agent.channel.request.id": "iad1::request-1",
- "agent.turn.id": "turn_0",
- "agent.turn.sequence": 0,
- "gen_ai.conversation.id": "session-1",
- });
- });
+ const spans = runtime.exporter.getFinishedSpans();
+ const turn = spans.find((span) => span.name === "invoke_agent support")!;
+ expect(byName(spans, "agent.channel.delivery")).toHaveLength(0);
+ expect(turn.parentSpanContext).toBeUndefined();
+ expect(turn.links).toEqual([
+ expect.objectContaining({
+ attributes: { "eve.link.type": "channel.request" },
+ context: expect.objectContaining(requestTraceContext),
+ }),
+ ]);
+ expect(turn.attributes).toMatchObject({
+ "agent.channel.delivery.id": "delivery-1",
+ "agent.channel.kind": "channel:slack",
+ "agent.channel.name": "slack",
+ "agent.channel.request.id": "iad1::request-1",
+ "agent.turn.id": "turn_0",
+ "agent.turn.sequence": 0,
+ "gen_ai.conversation.id": "session-1",
+ });
+ if (message === "private")
+ expect(turn.attributes["agent.channel.delivery.input"]).toBe(JSON.stringify({ message }));
+ else expect(turn.attributes["agent.channel.delivery.input"]).toBeUndefined();
+ },
+ );
it("does not emit delivery spans when remote deliveries fan in before the session event", async () => {
const runtime = createRuntime();
@@ -1333,11 +1299,11 @@ describe("createAgentOtelInstrumentation", () => {
turnId: "turn_0",
type: "channel.delivery.started",
});
- expect(stateStore.getSession(sessionId)).toBeUndefined();
+ expect(stateStore.get("session", sessionId)).toBeUndefined();
await runtime.prepareSessionTrace(sessionEvent);
await runtime.prepareTurnTrace(turnEvent);
- expect(stateStore.getSession(sessionId)).toMatchObject({
+ expect(stateStore.get("session", sessionId)).toMatchObject({
channelType: "http",
scheduleId: "sched-1",
title: "Investigate the incident",
@@ -1392,12 +1358,12 @@ describe("createAgentOtelInstrumentation", () => {
turnId: "turn_0",
type: "channel.delivery.started",
});
- expect(stateStore.getSession(sessionId)).toBeUndefined();
+ expect(stateStore.get("session", sessionId)).toBeUndefined();
await runtime.prepareSessionTrace(sessionEvent);
await runtime.hooks.publish(sessionEvent);
- expect(stateStore.getSession(sessionId)).toMatchObject({
+ expect(stateStore.get("session", sessionId)).toMatchObject({
channelType: "http",
parentLineage,
});
@@ -1673,7 +1639,7 @@ describe("createAgentOtelInstrumentation", () => {
it("uses the stored session decision when no event or context seed exists", async () => {
const stateStore = new InMemoryAgentTraceStateStore();
- stateStore.setSession("session-policy", {
+ stateStore.set("session", "session-policy", {
channelAudience: "public",
context: spanContext("1", "2"),
decision: { action: "record", recordInputs: false, recordOutputs: true },
@@ -1689,7 +1655,7 @@ describe("createAgentOtelInstrumentation", () => {
it("preserves a private decision already intersected with a forwarded ceiling", async () => {
const stateStore = new InMemoryAgentTraceStateStore();
- stateStore.setSession("session-forwarded-private", {
+ stateStore.set("session", "session-forwarded-private", {
channelAudience: "private",
context: spanContext("1", "2"),
decision: { action: "record", recordInputs: true, recordOutputs: false },
@@ -1811,7 +1777,7 @@ describe("createAgentOtelInstrumentation", () => {
it("reconstructs a sampled public session decision when persisted policy is absent", async () => {
const stateStore = new InMemoryAgentTraceStateStore();
- stateStore.setSession("session-sampled", {
+ stateStore.set("session", "session-sampled", {
channelAudience: "public",
context: spanContext("1", "2"),
rootSessionId: "session-sampled",
@@ -1828,7 +1794,7 @@ describe("createAgentOtelInstrumentation", () => {
it("reconstructs an unsampled session as content-redacted", async () => {
const stateStore = new InMemoryAgentTraceStateStore();
- stateStore.setSession("session-unsampled", {
+ stateStore.set("session", "session-unsampled", {
channelAudience: "public",
context: { ...spanContext("1", "2"), traceFlags: 0 },
rootSessionId: "session-unsampled",
@@ -2423,6 +2389,45 @@ describe("createAgentOtelInstrumentation", () => {
withSpy.mockRestore();
});
+ it("records no third-party spans from a tool call in a dropped turn", async () => {
+ const runtime = createRuntime(undefined, () => false);
+ const scope: InstrumentationAttemptScope = {
+ attemptId: "session-dropped:turn-1:0:0",
+ attemptIndex: 0,
+ channelAudience: "private",
+ functionId: "weather",
+ sessionId: "session-dropped",
+ stepIndex: 0,
+ turnId: "turn-1",
+ };
+ apiContext.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
+ try {
+ await publishTurnStarted({
+ channelAudience: "private",
+ hooks: runtime.hooks,
+ sessionId: scope.sessionId,
+ turnId: scope.turnId,
+ turnSequence: 0,
+ });
+ // An MCP client span started while Alice's dropped turn calls a tool.
+ await runtime.runInContext(
+ {
+ idempotencyKey: "tool:session-dropped:turn-1:tool-1",
+ scope,
+ type: "tool.call",
+ callId: "tool-1",
+ toolName: "linear__get_issue",
+ input: { issue: "ISSUE-1" },
+ },
+ async () => runtime.tracer.startSpan("mcp tools/call").end(),
+ );
+ await runtime.provider.forceFlush();
+ expect(runtime.exporter.getFinishedSpans()).toHaveLength(0);
+ } finally {
+ apiContext.disable();
+ }
+ });
+
it("enriches the existing tool span with MCP attributes under its content policy", async () => {
const runtime = createRuntime(new InMemoryAgentTraceStateStore(), () => ({
emit: true,
@@ -2476,20 +2481,16 @@ describe("createAgentOtelInstrumentation", () => {
async () => {
const active = withSpy.mock.calls[0]?.[0];
expect(active).toBeDefined();
- expect(agentToolContentPolicy(active!)).toEqual({
+ expect(activeTraceOperation(active!)?.capture).toMatchObject({
recordInputs: false,
recordOutputs: false,
});
- annotateAgentToolSpan(
- {
- "eve.connection.name": "linear",
- "gen_ai.operation.name": "execute_tool",
- "gen_ai.tool.name": "get_issue",
- "jsonrpc.request.id": "7",
- "mcp.method.name": "tools/call",
- },
- active!,
- );
+ activeTraceOperation(active!)?.mcp?.update({
+ connectionName: "linear",
+ method: "tools/call",
+ toolName: "get_issue",
+ requestId: "7",
+ });
},
);
withSpy.mockRestore();
@@ -2917,9 +2918,14 @@ describe("createAgentOtelInstrumentation", () => {
});
const agentOtel = createAgentOtelInstrumentation({
frameworkVersion: "test",
- idGenerator,
stateStore: new InMemoryAgentTraceStateStore(),
- tracer: provider.getTracer("eve.agent"),
+ telemetry: otelTelemetry({
+ provider,
+ tracerName: "eve.agent",
+ idGenerator,
+ mapping: eveOutputMapping(),
+ }),
+ idGenerator,
});
const hooks = createInstrumentationHooks([agentOtel.hook]).forTrace!(traceContext("public"));
await emitAttempt({
@@ -2988,7 +2994,7 @@ describe("createAgentOtelInstrumentation", () => {
expect(withSpy).toHaveBeenCalledOnce();
const active = withSpy.mock.calls[0]?.[0];
expect(active).toBeDefined();
- expect(agentToolSpanContext(active!)).toBeUndefined();
+ expect(activeTraceOperation(active!)?.type).toBe("model");
withSpy.mockRestore();
});
@@ -3251,22 +3257,22 @@ describe("createAgentOtelInstrumentation", () => {
input: undefined,
kind: "subagent-call",
});
- const action = (await store.getAction(key))!;
+ const action = (await store.get("action", key))!;
await publishTurnStarted({
hooks: runtime.hooks,
sessionId: "child",
rootSessionId: "parent",
turnId: "turn_0",
turnSequence: 0,
- parentTraceContext: { ...action.parent, spanId: action.spanId },
+ parentTraceContext: action.context,
});
await completeTurn(runtime.hooks, "child", "turn_0");
await runtime.provider.forceFlush();
const spans = runtime.exporter.getFinishedSpans();
expect(byName(spans, "agent.action")).toHaveLength(0);
const child = byName(spans, "invoke_agent weather")[0]!;
- expect(child.parentSpanContext?.spanId).toBe(action.spanId);
- expect(child.spanContext().traceId).toBe(action.parent.traceId);
+ expect(child.parentSpanContext?.spanId).toBe(action.context.spanId);
+ expect(child.spanContext().traceId).toBe(action.context.traceId);
});
it("starts a remote root with a dispatch link and receiver sampling", async () => {
@@ -3459,31 +3465,3 @@ function modelStartedEvent(
type: "model.call.started",
};
}
-
-describe("AgentSpanIdGenerator.withTraceId", () => {
- it("primes the next generateTraceId call", () => {
- const gen = new AgentSpanIdGenerator();
- const primed = "e".repeat(32);
- const result = gen.withTraceId(primed, () => gen.generateTraceId());
- expect(result).toBe(primed);
- // After the callback, a fresh call should produce a different id.
- const next = gen.generateTraceId();
- expect(next).not.toBe(primed);
- });
-
- it("nests inside withSpanId to prime both ids", () => {
- const gen = new AgentSpanIdGenerator();
- const primedTraceId = "f".repeat(32);
- const primedSpanId = "a".repeat(16);
- let capturedTraceId: string | undefined;
- let capturedSpanId: string | undefined;
- gen.withSpanId(primedSpanId, () =>
- gen.withTraceId(primedTraceId, () => {
- capturedTraceId = gen.generateTraceId();
- capturedSpanId = gen.generateSpanId();
- }),
- );
- expect(capturedTraceId).toBe(primedTraceId);
- expect(capturedSpanId).toBe(primedSpanId);
- });
-});
diff --git a/packages/eve/src/tracing/eve/agent-otel-provider.ts b/packages/eve/src/tracing/eve/agent-otel-provider.ts
new file mode 100644
index 000000000..7c42b8cdc
--- /dev/null
+++ b/packages/eve/src/tracing/eve/agent-otel-provider.ts
@@ -0,0 +1,295 @@
+import { ROOT_CONTEXT, context, trace } from "#compiled/@opentelemetry/api/index.js";
+
+import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";
+import { createAgentApprovalInstrumentation } from "#tracing/eve/agent-approval-instrumentation.js";
+import { createAgentChannelDeliveryInstrumentation } from "#tracing/eve/agent-channel-delivery-instrumentation.js";
+import { createAgentToolInstrumentation } from "#tracing/eve/agent-tool-instrumentation.js";
+import { eveActivationMetadata } from "#tracing/eve/metadata.js";
+import { createAgentMemoryInstrumentation } from "#tracing/eve/agent-memory-instrumentation.js";
+import { readGatewayCostData } from "#tracing/eve/gateway.js";
+import { createEveSessionTracing } from "#tracing/eve/session.js";
+import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";
+import { isSampledTrace } from "#shared/trace-policy.js";
+import { createEveCapturePolicy } from "#tracing/eve/policy.js";
+import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";
+import type {
+ InstrumentationStepAttemptMetadataEvent,
+ InstrumentationAttemptScope,
+ InstrumentationStepAttemptStartedEvent,
+ InstrumentationStepAttemptTerminalEvent,
+ InstrumentationContextRunner,
+ InstrumentationModelCallTerminalEvent,
+ InstrumentationModelCallStartedEvent,
+ InstrumentationProviderDefinition,
+ InstrumentationSessionStartedEvent,
+ InstrumentationTraceSeed,
+ InstrumentationSessionTransitionEvent,
+ InstrumentationTurnStartedEvent,
+} from "#instrumentation/lifecycle.js";
+import { resolveInstrumentationEnvironment } from "#internal/application/dev-environment.js";
+import type { ConversationEnvironment } from "#shared/conversation-context.js";
+import { createEveTracing } from "#tracing/eve/operation-input.js";
+import { gatewayCostAttributes } from "#tracing/eve/gateway.js";
+import type {
+ AgentSpanIdGenerator,
+ AgentTelemetry,
+ AttemptOperation,
+ ModelOperation,
+} from "#compiled/@eve/tracing/index.js";
+import { isUserMessageKind } from "#harness/messages.js";
+import { isObject } from "#shared/guards.js";
+
+export interface AgentOtelInstrumentationInput {
+ /** Span output for eve's agent spans; its `ids` make turns durable across workers. */
+ readonly telemetry: AgentTelemetry;
+ readonly idGenerator: AgentSpanIdGenerator;
+ readonly environment?: ConversationEnvironment;
+ /** Whether any destination records model and tool inputs. */
+ readonly recordInputs?: boolean;
+ /** Whether any destination records model and tool outputs. */
+ readonly recordOutputs?: boolean;
+ readonly frameworkVersion: string;
+ readonly stateStore: AgentTraceStateStore;
+ readonly tracePolicy?: TraceCapturePolicy;
+}
+
+/** OTel event definition and its trusted framework context runner. */
+interface AgentOtelInstrumentation {
+ readonly hook: InstrumentationProviderDefinition;
+ readonly prepareSessionTrace: (
+ event: InstrumentationSessionStartedEvent,
+ ) => Promise<InstrumentationTraceSeed>;
+ readonly prepareTurnTrace: (
+ event: InstrumentationTurnStartedEvent,
+ ) => Promise<InstrumentationTraceSeed>;
+ readonly runInContext: InstrumentationContextRunner;
+}
+
+/** Creates OTel instrumentation for eve's structural and GenAI spans. */
+export function createAgentOtelInstrumentation(
+ input: AgentOtelInstrumentationInput,
+): AgentOtelInstrumentation {
+ const environment = input.environment ?? resolveInstrumentationEnvironment();
+ const recordInputs = input.recordInputs ?? false;
+ const recordOutputs = input.recordOutputs ?? false;
+ const tracing = createEveTracing(input);
+ // Attempts run in one process; a lost serverless worker retries the whole step.
+ const steps = new Map<string, AttemptOperation>();
+ const modelSpans = new Map<string, Map<string, ModelOperation>>();
+ const sessionTracing = createEveSessionTracing({ ...input, environment, tracing });
+ const { prepareSessionTrace, prepareTurnTrace } = sessionTracing;
+
+ /** The attempt in its turn's tree, hydrating it when another process started it. */
+ async function attemptFor(scope: InstrumentationAttemptScope) {
+ const known = steps.get(scope.attemptId);
+ if (known !== undefined) return known;
+ const turn = await sessionTracing.turnFor(
+ scope.sessionId,
+ scope.turnId,
+ withChannelAudience(ROOT_CONTEXT, scope.channelAudience),
+ );
+ const index = { stepIndex: scope.stepIndex, attempt: scope.attemptIndex };
+ return turn?.findAttempt(index) ?? (turn?.finished === false ? turn.attempt(index) : undefined);
+ }
+
+ const tools = createAgentToolInstrumentation({
+ tracing,
+ attemptFor,
+ stepFor: (scope) => steps.get(scope.attemptId),
+ recordInputs,
+ recordOutputs,
+ stateStore: input.stateStore,
+ });
+ const approvals = createAgentApprovalInstrumentation({
+ tracing,
+ actionStateFor: tools.stateFor,
+ });
+ const memory = createAgentMemoryInstrumentation({ ...input, environment, tracing });
+
+ const capturePolicy = createEveCapturePolicy({
+ stateStore: input.stateStore,
+ environment,
+ recordInputs,
+ recordOutputs,
+ });
+
+ function turnState(scope: { sessionId: string; turnId: string }) {
+ return input.stateStore.get("turn", JSON.stringify([scope.sessionId, scope.turnId]));
+ }
+
+ const onSessionStarted = async (event: InstrumentationSessionStartedEvent): Promise<void> => {
+ await prepareSessionTrace(event);
+ };
+
+ const onTurnStarted = async (event: InstrumentationTurnStartedEvent): Promise<void> => {
+ await prepareTurnTrace(event);
+ };
+
+ const onStepStarted = async (event: InstrumentationStepAttemptStartedEvent): Promise<void> => {
+ const turn = await turnState(event.scope);
+ if (turn === undefined || !isSampledTrace(turn.context)) return;
+ const session = await input.stateStore.get("session", event.scope.sessionId);
+ const operation = await sessionTracing.turnFor(
+ event.scope.sessionId,
+ event.scope.turnId,
+ withChannelAudience(ROOT_CONTEXT, event.scope.channelAudience),
+ );
+ if (operation === undefined || operation.finished) return;
+ const activeSpanContext = trace.getSpan(context.active())?.spanContext();
+ const attempt = await operation.attempt({
+ stepIndex: event.scope.stepIndex,
+ attempt: event.scope.attemptIndex,
+ runtimeContext: event.runtimeContext,
+ channel: eveActivationMetadata({ session, turn, sessionId: event.scope.sessionId }).channel,
+ links:
+ activeSpanContext === undefined || activeSpanContext.traceId === turn.context.traceId
+ ? undefined
+ : [{ relationship: "execution.delivery", context: activeSpanContext }],
+ });
+ steps.set(event.scope.attemptId, attempt);
+ };
+
+ const onStepTerminal = async (event: InstrumentationStepAttemptTerminalEvent): Promise<void> => {
+ const failure = event.type === "step.attempt.failed" ? { error: event.error } : undefined;
+ await drainOpenSpans(event);
+ await tools.drain(event.scope.attemptId, failure);
+ const attempt = steps.get(event.scope.attemptId) ?? (await attemptFor(event.scope));
+ steps.delete(event.scope.attemptId);
+ if (failure !== undefined) {
+ // A failed attempt also fails the tool calls it dispatched.
+ await attempt?.fail(failure.error);
+ await tools.forgetAttempt(event.scope);
+ } else await attempt?.complete();
+ };
+
+ const onSessionTransition = async (
+ event: InstrumentationSessionTransitionEvent,
+ ): Promise<void> => {
+ await sessionTracing.sessionTransition(event);
+ // `session.waiting` is not terminal — the session may resume with a new
+ // turn that still needs its metadata — so only release session-scoped
+ // state on terminal transitions.
+ if (event.type === "session.completed" || event.type === "session.failed") {
+ await tools.deleteForSession(event.sessionId);
+ await input.stateStore.delete("session", event.sessionId);
+ }
+ };
+
+ const onModelCallStarted = async (event: InstrumentationModelCallStartedEvent): Promise<void> => {
+ const attempt = steps.get(event.scope.attemptId);
+ if (attempt === undefined) return;
+ const operation = await attempt.modelCall(
+ {
+ provider: event.model.provider,
+ modelId: event.model.modelId,
+ messages: recordInputs
+ ? event.input?.messages.map((message) =>
+ isObject(message) && message.role === "user"
+ ? { ...message, kind: isUserMessageKind(message.kind) ? message.kind : undefined }
+ : message,
+ )
+ : undefined,
+ instructions: recordInputs ? event.input?.instructions : undefined,
+ tools: recordInputs ? event.input?.tools : undefined,
+ runtimeContext: event.runtimeContext,
+ },
+ event.idempotencyKey,
+ );
+ let models = modelSpans.get(event.scope.attemptId);
+ if (models === undefined) modelSpans.set(event.scope.attemptId, (models = new Map()));
+ models.set(event.idempotencyKey, operation);
+ };
+
+ const onModelCallTerminal = async (
+ event: InstrumentationModelCallTerminalEvent,
+ ): Promise<void> => {
+ const models = modelSpans.get(event.scope.attemptId);
+ const operation = models?.get(event.idempotencyKey);
+ models?.delete(event.idempotencyKey);
+ if (operation === undefined) return;
+ if (event.type === "model.call.failed") await operation.fail(event.error);
+ else
+ await operation.complete({
+ outcome: "completed",
+ result: { ...event, content: recordOutputs ? event.content : undefined },
+ });
+ };
+
+ const channelDeliveries = createAgentChannelDeliveryInstrumentation({
+ recordInputs,
+ stateStore: input.stateStore,
+ });
+
+ const onStepMetadata = (event: InstrumentationStepAttemptMetadataEvent): void => {
+ const attempt = steps.get(event.scope.attemptId);
+ if (attempt === undefined) return;
+ // Vercel AI Gateway reports per-call cost in providerMetadata.gateway;
+ // attributes exist only when it was actually the gateway serving the call.
+ const cost = readGatewayCostData(event.providerMetadata);
+ if (cost !== undefined) attempt.attributes(gatewayCostAttributes(cost));
+ };
+
+ return {
+ hook: {
+ events: {
+ ...channelDeliveries,
+ ...approvals,
+ "step.attempt.completed": onStepTerminal,
+ "step.attempt.failed": onStepTerminal,
+ "step.attempt.metadata": onStepMetadata,
+ "step.attempt.started": onStepStarted,
+ "model.call.completed": onModelCallTerminal,
+ "model.call.failed": onModelCallTerminal,
+ "model.call.started": onModelCallStarted,
+ "session.completed": onSessionTransition,
+ "session.failed": onSessionTransition,
+ "session.started": onSessionStarted,
+ "session.waiting": onSessionTransition,
+ ...tools.events,
+ "turn.cancelled": sessionTracing.turnTerminal,
+ "turn.completed": sessionTracing.turnTerminal,
+ "turn.failed": sessionTracing.turnTerminal,
+ "turn.started": onTurnStarted,
+ },
+ name: "eve.otel",
+ projectEvent: capturePolicy.projectEvent,
+ tracePolicy: () => ({ emit: true, recordInputs, recordOutputs }),
+ },
+ prepareSessionTrace,
+ prepareTurnTrace,
+ async runInContext(operation, execute) {
+ if (operation.type === "memory.operation") return memory.runInContext(operation, execute);
+ const ceiling = { emit: true, ...(await capturePolicy.forOperation(operation.scope)) };
+ const inTurn = async () => {
+ const turn = await sessionTracing.turnFor(
+ operation.scope.sessionId,
+ operation.scope.turnId,
+ withChannelAudience(ROOT_CONTEXT, operation.scope.channelAudience),
+ );
+ if (turn !== undefined && !turn.finished) return turn.run(execute, ceiling);
+ // Third-party spans, such as MCP calls, must not record a dropped turn's work.
+ const { sessionId, turnId } = operation.scope;
+ return input.telemetry.suppressed !== undefined &&
+ (await sessionTracing.dropped(sessionId, turnId))
+ ? input.telemetry.suppressed(execute)
+ : execute();
+ };
+ if (operation.type === "tool.call")
+ return tools.runInContext(operation, execute, ceiling, inTurn);
+ const model = modelSpans.get(operation.scope.attemptId)?.get(operation.idempotencyKey);
+ return model !== undefined && !model.finished ? model.run(execute, ceiling) : inTurn();
+ },
+ };
+
+ async function drainOpenSpans(event: InstrumentationStepAttemptTerminalEvent): Promise<void> {
+ for (const operation of modelSpans.get(event.scope.attemptId)?.values() ?? []) {
+ if (event.type === "step.attempt.failed") await operation.fail(event.error);
+ else
+ await operation.complete({
+ outcome: "completed",
+ result: { finishReason: "unknown", usage: {} },
+ });
+ }
+ modelSpans.delete(event.scope.attemptId);
+ }
+}
diff --git a/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts b/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts
index 50ba807f8..0295d7fd7 100644
--- a/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts
+++ b/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts
@@ -31,12 +31,15 @@ import {
type InstrumentationAttemptScope,
} from "#instrumentation/lifecycle.js";
import { bindInstrumentationRuntime } from "#instrumentation/runtime.js";
-import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js";
-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
-import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";
-import { AGENT_TRACE_CONTEXT_KEY } from "#tracing/agent-trace-context-codec.js";
+import { createAgentOtelInstrumentation } from "#tracing/eve/agent-otel-provider.js";
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
+import { otelTelemetry } from "#compiled/@eve/tracing/index.js";
+import { eveOutputMapping } from "#tracing/eve/profile.js";
+import { ContextAgentTraceStateStore } from "#tracing/eve/agent-trace-context-store.js";
+import { AGENT_TRACE_CONTEXT_KEY } from "#tracing/eve/agent-trace-context-store.js";
import { resolveToolCallAgentTrace } from "#tracing/eve/agent-invocation-coordinator.js";
import * as instrumentation from "#instrumentation/runtime.js";
+import * as runtimeGlobal from "#instrumentation/runtime-global.js";
import {
assembleLocalTrace,
isAgentTurnSpan,
@@ -78,11 +81,16 @@ function createRuntime() {
});
const agent = createAgentOtelInstrumentation({
frameworkVersion: "test",
- idGenerator,
recordInputs: true,
recordOutputs: true,
stateStore: new ContextAgentTraceStateStore(),
- tracer: provider.getTracer("eve.agent"),
+ telemetry: otelTelemetry({
+ provider,
+ tracerName: "eve.agent",
+ idGenerator,
+ mapping: eveOutputMapping(),
+ }),
+ idGenerator,
});
const hooks = createInstrumentationHooks([agent.hook]);
return {
@@ -1090,7 +1098,7 @@ describe("exported agent telemetry contract", () => {
const runtime = createRuntime();
const hooks = runtime.hooks.forTrace!(traceContext("child", "public"));
const registered = vi
- .spyOn(instrumentation, "getInstrumentationRuntime")
+ .spyOn(runtimeGlobal, "getInstrumentationRuntime")
.mockReturnValue(runtime);
let ctx = contextFor("public");
ctx.set(ParentTraceContextKey, {
diff --git a/packages/eve/src/tracing/eve/agent-tool-instrumentation.ts b/packages/eve/src/tracing/eve/agent-tool-instrumentation.ts
new file mode 100644
index 000000000..103304ae5
--- /dev/null
+++ b/packages/eve/src/tracing/eve/agent-tool-instrumentation.ts
@@ -0,0 +1,310 @@
+import {
+ ROOT_CONTEXT,
+ type Context,
+ type SpanContext,
+ trace,
+} from "#compiled/@opentelemetry/api/index.js";
+
+import type {
+ InstrumentationAttemptScope,
+ InstrumentationExecutionOperation,
+ InstrumentationToolCallStartedEvent,
+ InstrumentationToolCallTerminalEvent,
+} from "#instrumentation/lifecycle.js";
+import { actionIdempotencyKey } from "#instrumentation/lifecycle.js";
+import { normalizeChannelAudience } from "#shared/channel-audience.js";
+import type {
+ AgentActionTraceState,
+ AgentTraceStateStore,
+} from "#tracing/eve/agent-trace-state.js";
+import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";
+import { eveTurnIdentity } from "#tracing/eve/operation-input.js";
+import type {
+ AgentTracing,
+ AttemptOperation,
+ CaptureDecision,
+ ToolInput,
+ ToolOperation,
+} from "#compiled/@eve/tracing/index.js";
+
+export interface AgentToolContext {
+ readonly context: Context;
+ readonly spanContext: SpanContext;
+}
+
+type ToolExecution = Extract<InstrumentationExecutionOperation, { type: "tool.call" }>;
+
+/** What the SDK reported about one tool run, kept until the dispatch or the step settles it. */
+interface Execution {
+ readonly attemptId: string;
+ readonly idempotencyKey: string;
+ readonly tool: ToolOperation;
+ terminal?: {
+ readonly completedAtMs: number;
+ readonly durationMs?: number;
+ readonly failed: boolean;
+ readonly error?: unknown;
+ readonly output?: unknown;
+ };
+}
+
+const OPEN_EXECUTIONS = 10000;
+
+/** Finds an open tool call in its turn's trace tree, in this or a later process. */
+export async function resumeTool(
+ tracing: AgentTracing,
+ state: AgentActionTraceState,
+): Promise<ToolOperation | undefined> {
+ const turn = await tracing.resume({
+ identity: eveTurnIdentity(state),
+ context: withChannelAudience(ROOT_CONTEXT, state.channelAudience),
+ });
+ return turn?.findTool(state.callId);
+}
+
+/**
+ * One `execute_tool` span per call. eve's tool loop reports the durable
+ * dispatch through `tool.call.*` events; the SDK reports the run through the
+ * execution context. Both name the same call ID, so they share one span. A run
+ * without a dispatch closes under its step when the step ends.
+ */
+export function createAgentToolInstrumentation(input: {
+ readonly tracing: AgentTracing;
+ readonly attemptFor: (
+ scope: InstrumentationAttemptScope,
+ ) => Promise<AttemptOperation | undefined>;
+ readonly stepFor: (scope: InstrumentationAttemptScope) => AttemptOperation | undefined;
+ readonly recordInputs: boolean;
+ readonly recordOutputs: boolean;
+ readonly stateStore: AgentTraceStateStore;
+}) {
+ const executions = new Map<string, Execution>();
+ const callKey = (scope: { sessionId: string; turnId: string }, callId: string) =>
+ actionIdempotencyKey(scope.sessionId, scope.turnId, callId);
+
+ async function stateFor(sessionId: string, turnId: string, callId: string) {
+ const direct = await input.stateStore.get(
+ "action",
+ actionIdempotencyKey(sessionId, turnId, callId),
+ );
+ if (direct !== undefined) return direct;
+ return (await input.stateStore.entries("action")).find(
+ ([, state]) => state.sessionId === sessionId && state.callId === callId,
+ )?.[1];
+ }
+
+ /** Reports the call again through its parent, so a later report refines the same span. */
+ async function reenter(state: AgentActionTraceState, report: ToolInput) {
+ const turn = await input.tracing.resume({
+ identity: eveTurnIdentity(state),
+ context: withChannelAudience(ROOT_CONTEXT, state.channelAudience),
+ });
+ if (turn?.findTool(state.callId) === undefined) return undefined;
+ const parent = turn.findAttempt({ stepIndex: state.stepIndex, attempt: state.attemptIndex });
+ return parent === undefined ? turn.findTool(state.callId) : parent.tool(report);
+ }
+
+ /** A dispatched call takes the run's timing and failure; its own terminal ends the span. */
+ async function settleExecution(execution: Execution, dispatched: AgentActionTraceState) {
+ const terminal = execution.terminal;
+ if (terminal === undefined) return;
+ executions.delete(execution.idempotencyKey);
+ if (terminal.failed) execution.tool.recordError(terminal.error);
+ else if (terminal.durationMs !== undefined)
+ execution.tool.attributes({ "gen_ai.execute_tool.duration": terminal.durationMs / 1000 });
+ await input.stateStore.set("action", execution.idempotencyKey, {
+ ...dispatched,
+ toolEndTimeMs: terminal.completedAtMs,
+ });
+ }
+
+ async function onDispatchStarted(event: InstrumentationToolCallStartedEvent): Promise<void> {
+ let state = await input.stateStore.get("action", event.idempotencyKey);
+ if (state === undefined) {
+ const parent = await input.attemptFor(event.scope);
+ if (parent === undefined) return;
+ const tool = await parent.tool({
+ callId: event.callId,
+ name: event.toolName,
+ kind: event.kind ?? "tool-call",
+ arguments: input.recordInputs ? event.input : undefined,
+ startTimeMs: event.startedAtMs,
+ });
+ tool.attributes({ "agent.tool.is_framework": event.frameworkTool === true });
+ state = {
+ attemptId: event.scope.attemptId,
+ attemptIndex: event.scope.attemptIndex,
+ callId: event.callId,
+ channelAudience: normalizeChannelAudience(event.scope.channelAudience),
+ context: { ...tool.reference, isRemote: false },
+ rootSessionId: event.scope.rootSessionId,
+ sessionId: event.scope.sessionId,
+ stepIndex: event.scope.stepIndex,
+ turnId: event.scope.turnId,
+ };
+ await input.stateStore.set("action", event.idempotencyKey, state);
+ const execution = executions.get(event.idempotencyKey);
+ if (execution !== undefined) await settleExecution(execution, state);
+ }
+ if (event.isWorkflowTool === true) {
+ await input.stateStore.set("anchor", event.idempotencyKey, state);
+ }
+ }
+
+ async function onDispatchTerminal(event: InstrumentationToolCallTerminalEvent): Promise<void> {
+ const state = await input.stateStore.get("action", event.idempotencyKey);
+ if (state === undefined) return;
+ try {
+ const tool = await resumeTool(input.tracing, state);
+ const error =
+ event.type === "tool.call.failed"
+ ? event.error
+ : event.output.type === "error"
+ ? event.output.error
+ : undefined;
+ await tool?.complete({
+ outcome: event.outcome ?? (event.type === "tool.call.failed" ? "failed" : "completed"),
+ failed: event.type === "tool.call.failed" || event.output.type === "error",
+ errorType: event.type === "tool.call.failed" ? event.errorCode : undefined,
+ error,
+ output:
+ input.recordOutputs &&
+ event.type === "tool.call.completed" &&
+ event.output.type === "result"
+ ? event.output.output
+ : undefined,
+ usage: event.usage,
+ endTimeMs: event.acceptedAtMs ?? state.toolEndTimeMs,
+ });
+ } finally {
+ await input.stateStore.delete("action", event.idempotencyKey);
+ }
+ }
+
+ async function executionFor(
+ operation: ToolExecution & { callId: string; toolName: string },
+ ceiling: CaptureDecision,
+ ) {
+ const key = callKey(operation.scope, operation.callId);
+ const known = executions.get(key);
+ if (known !== undefined) return known;
+ const state = await stateFor(
+ operation.scope.sessionId,
+ operation.scope.turnId,
+ operation.callId,
+ );
+ const report: ToolInput = {
+ callId: operation.callId,
+ name: operation.toolName,
+ arguments: input.recordInputs && ceiling.recordInputs ? operation.input : undefined,
+ startTimeMs: operation.startedAtMs,
+ };
+ const tool =
+ state === undefined
+ ? await input.stepFor(operation.scope)?.tool(report)
+ : await reenter(state, report);
+ if (tool === undefined || executions.size >= OPEN_EXECUTIONS) return undefined;
+ const execution: Execution = {
+ attemptId: operation.scope.attemptId,
+ idempotencyKey: key,
+ tool,
+ };
+ executions.set(key, execution);
+ return execution;
+ }
+
+ async function finishExecution(execution: Execution, terminal: Execution["terminal"]) {
+ execution.terminal = terminal;
+ const key = execution.idempotencyKey;
+ const dispatched = await input.stateStore.get("action", key);
+ if (dispatched !== undefined) await settleExecution(execution, dispatched);
+ }
+
+ return {
+ stateFor,
+ async contextFor(sessionId: string, turnId: string, callId: string) {
+ const state = await stateFor(sessionId, turnId, callId);
+ return state === undefined ? undefined : toolContext(state);
+ },
+ async deleteForSession(sessionId: string) {
+ for (const kind of ["action", "anchor"] as const)
+ for (const [key, state] of await input.stateStore.entries(kind))
+ if (state.sessionId === sessionId) await input.stateStore.delete(kind, key);
+ },
+ // A failed attempt closes its calls in the trace tree; only the locators remain.
+ async forgetAttempt(scope: InstrumentationAttemptScope) {
+ for (const [key, state] of await input.stateStore.entries("action"))
+ if (state.attemptId === scope.attemptId) await input.stateStore.delete("action", key);
+ },
+ /** Closes SDK runs that no dispatch claimed before their step ended. */
+ async drain(attemptId: string, failure?: { readonly error: unknown }) {
+ for (const [key, execution] of executions) {
+ if (execution.attemptId !== attemptId) continue;
+ executions.delete(key);
+ if ((await input.stateStore.get("action", key)) !== undefined) continue;
+ const terminal = execution.terminal;
+ if (failure !== undefined) {
+ await execution.tool.fail(failure.error);
+ continue;
+ }
+ if (terminal?.durationMs !== undefined && !terminal.failed)
+ execution.tool.attributes({ "gen_ai.execute_tool.duration": terminal.durationMs / 1000 });
+ await execution.tool.complete({
+ failed: terminal?.failed,
+ error: terminal?.error,
+ output: terminal?.output,
+ endTimeMs: terminal?.completedAtMs,
+ });
+ }
+ },
+ /** Runs the SDK's tool call inside its span, or calls `fallback` when it has none. */
+ async runInContext<T>(
+ operation: ToolExecution,
+ execute: () => PromiseLike<T>,
+ ceiling: CaptureDecision,
+ fallback: () => Promise<T>,
+ ): Promise<T> {
+ if (operation.callId === undefined || operation.toolName === undefined) return fallback();
+ const execution = await executionFor(
+ { ...operation, callId: operation.callId, toolName: operation.toolName },
+ ceiling,
+ );
+ if (execution === undefined || execution.tool.finished) return fallback();
+ execution.tool.attributes({ "agent.tool.is_framework": operation.frameworkTool === true });
+ const startedAtMs = operation.startedAtMs ?? Date.now();
+ try {
+ const result = await execution.tool.run(execute, ceiling);
+ const completedAtMs = operation.completedAtMs ?? Date.now();
+ await finishExecution(execution, {
+ completedAtMs,
+ durationMs: completedAtMs - startedAtMs,
+ failed: operation.failed === true,
+ output: input.recordOutputs && ceiling.recordOutputs ? result : undefined,
+ });
+ return result;
+ } catch (error) {
+ await finishExecution(execution, {
+ completedAtMs: Date.now(),
+ failed: true,
+ error: ceiling.recordOutputs ? error : undefined,
+ });
+ throw error;
+ }
+ },
+ events: {
+ "tool.call.completed": onDispatchTerminal,
+ "tool.call.failed": onDispatchTerminal,
+ "tool.call.started": onDispatchStarted,
+ },
+ };
+}
+
+function toolContext(state: AgentActionTraceState): AgentToolContext {
+ return {
+ context: withChannelAudience(
+ trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(state.context)),
+ state.channelAudience,
+ ),
+ spanContext: state.context,
+ };
+}
diff --git a/packages/eve/src/tracing/eve/agent-trace-context-store.test.ts b/packages/eve/src/tracing/eve/agent-trace-context-store.test.ts
index c24cd6cb1..1e5cd69ae 100644
--- a/packages/eve/src/tracing/eve/agent-trace-context-store.test.ts
+++ b/packages/eve/src/tracing/eve/agent-trace-context-store.test.ts
@@ -3,29 +3,26 @@ import { describe, expect, it } from "vitest";
import { ContextContainer, contextStorage } from "#context/container.js";
import { SessionTraceSeedKey } from "#context/keys.js";
import { deserializeContext, serializeContext } from "#context/serialize.js";
-import { INSTRUMENTATION_PRINCIPAL_TYPES } from "#instrumentation/lifecycle.js";
-import { deserializeAgentTraceContextState } from "#tracing/agent-trace-context-codec.js";
-import { CONTENT_ATTRIBUTE_LIMIT } from "#tracing/agent-otel-content.js";
import {
ContextAgentTraceStateStore,
preserveSerializedAgentTraceState,
readActionTraceContext,
readTurnTraceContext,
-} from "#tracing/agent-trace-context-store.js";
+} from "#tracing/eve/agent-trace-context-store.js";
describe("ContextAgentTraceStateStore", () => {
it("restores serializable session and turn context", async () => {
const context = new ContextContainer();
await contextStorage.run(context, () => {
const store = new ContextAgentTraceStateStore();
- store.setSession("session-1", {
+ store.set("session", "session-1", {
agentName: "weather",
channelType: "http",
context: spanContext("1", "2"),
rootSessionId: "session-1",
traceSessionId: "session-1",
});
- store.setTurn("session-1", "turn-1", {
+ store.set("turn", JSON.stringify(["session-1", "turn-1"]), {
channelDelivery: {
channelKind: "channel:slack",
channelName: "slack",
@@ -37,16 +34,13 @@ describe("ContextAgentTraceStateStore", () => {
context: spanContext("1", "3"),
currentPrincipal: { id: "user-123", type: "user" },
initiatorPrincipal: { type: "none" },
- inputMessagesAttribute: '[{"parts":[{"content":"hello","type":"text"}],"role":"user"}]',
- modelUsage: { inputTokens: 12, outputTokens: 4 },
- outputMessagesAttribute: '[{"parts":[{"content":"hi","type":"text"}],"role":"assistant"}]',
caller: { ...spanContext("4", "2"), isRemote: true },
rootSessionId: "session-1",
traceSessionId: "session-1",
sequence: 0,
startTimeMs: 1_700_000_000_000,
subagentName: "researcher",
- terminal: { error: new Error("failed"), type: "turn.failed" },
+ terminal: { outcome: "failed", errorName: "Error", errorMessage: "failed" },
});
});
@@ -54,12 +48,14 @@ describe("ContextAgentTraceStateStore", () => {
const restored = await deserializeContext(serialized);
await contextStorage.run(restored, () => {
const store = new ContextAgentTraceStateStore();
- expect(store.getSession("session-1")).toMatchObject({
+ expect(store.get("session", "session-1")).toMatchObject({
channelType: "http",
context: spanContext("1", "2"),
});
- expect(store.getTurn("session-1", "turn-1")?.context).toEqual(spanContext("1", "3"));
- expect(store.getTurn("session-1", "turn-1")).toMatchObject({
+ expect(store.get("turn", JSON.stringify(["session-1", "turn-1"]))?.context).toEqual(
+ spanContext("1", "3"),
+ );
+ expect(store.get("turn", JSON.stringify(["session-1", "turn-1"]))).toMatchObject({
channelDelivery: {
channelKind: "channel:slack",
channelName: "slack",
@@ -70,80 +66,31 @@ describe("ContextAgentTraceStateStore", () => {
},
currentPrincipal: { id: "user-123", type: "user" },
initiatorPrincipal: { type: "none" },
- inputMessagesAttribute: '[{"parts":[{"content":"hello","type":"text"}],"role":"user"}]',
- modelUsage: { inputTokens: 12, outputTokens: 4 },
- outputMessagesAttribute: '[{"parts":[{"content":"hi","type":"text"}],"role":"assistant"}]',
+ terminal: { outcome: "failed", errorName: "Error", errorMessage: "failed" },
caller: { ...spanContext("4", "2"), isRemote: true },
startTimeMs: 1_700_000_000_000,
subagentName: "researcher",
});
- expect(store.getTurn("session-1", "turn-1")?.initiatorPrincipal).toStrictEqual({
+ expect(
+ store.get("turn", JSON.stringify(["session-1", "turn-1"]))?.initiatorPrincipal,
+ ).toStrictEqual({
type: "none",
});
- expect(JSON.stringify(store.getTurn("session-1", "turn-1"))).not.toContain("attributes");
- const terminal = store.getTurn("session-1", "turn-1")?.terminal;
- expect(terminal?.type).toBe("turn.failed");
- expect(terminal?.type === "turn.failed" ? terminal.error : undefined).toMatchObject({
- message: "failed",
- });
- });
- });
-
- it("rejects oversized persisted turn content attributes", () => {
- const state = deserializeAgentTraceContextState({
- turns: {
- turn: {
- context: spanContext("1", "2"),
- inputMessagesAttribute: "x".repeat(CONTENT_ATTRIBUTE_LIMIT + 1),
- outputMessagesAttribute: "x".repeat(CONTENT_ATTRIBUTE_LIMIT + 1),
- rootSessionId: "session-1",
- sequence: 0,
- startTimeMs: 1,
- },
- },
+ expect(
+ JSON.stringify(store.get("turn", JSON.stringify(["session-1", "turn-1"]))),
+ ).not.toContain("attributes");
});
-
- expect(state.turns.turn).toBeDefined();
- expect(state.turns.turn?.inputMessagesAttribute).toBeUndefined();
- expect(state.turns.turn?.outputMessagesAttribute).toBeUndefined();
- });
-
- it.each([
- ...INSTRUMENTATION_PRINCIPAL_TYPES.map((type) => ({
- input: { type, id: "user-123" },
- expected: type === "none" ? { type } : { type, id: "user-123" },
- })),
- { input: { type: "unrecognized", id: "user-123" }, expected: undefined },
- ...[undefined, null, 123, "", "x".repeat(1025), String.fromCodePoint(0x1f600).repeat(257)].map(
- (id) => ({
- input: { type: "user", id },
- expected: { type: "user" },
- }),
- ),
- ])("restores bounded principal summaries from $input", ({ input, expected }) => {
- const state = deserializeAgentTraceContextState({
- turns: {
- turn: {
- context: spanContext("1", "2"),
- currentPrincipal: input,
- initiatorPrincipal: input,
- startTimeMs: 1,
- },
- },
- });
- expect(state.turns.turn?.currentPrincipal).toStrictEqual(expected);
- expect(state.turns.turn?.initiatorPrincipal).toStrictEqual(expected);
});
it("removes terminal state", () => {
contextStorage.run(new ContextContainer(), () => {
const store = new ContextAgentTraceStateStore();
- store.setSession("session-1", {
+ store.set("session", "session-1", {
context: spanContext("1", "2"),
rootSessionId: "session-1",
traceSessionId: "session-1",
});
- store.setTurn("session-1", "turn-1", {
+ store.set("turn", JSON.stringify(["session-1", "turn-1"]), {
context: spanContext("1", "3"),
caller: spanContext("4", "2"),
rootSessionId: "session-1",
@@ -152,18 +99,18 @@ describe("ContextAgentTraceStateStore", () => {
startTimeMs: 1_700_000_000_000,
});
- store.deleteTurn("session-1", "turn-1");
- store.deleteSession("session-1");
+ store.delete("turn", JSON.stringify(["session-1", "turn-1"]));
+ store.delete("session", "session-1");
- expect(store.getTurn("session-1", "turn-1")).toBeUndefined();
- expect(store.getSession("session-1")).toBeUndefined();
+ expect(store.get("turn", JSON.stringify(["session-1", "turn-1"]))).toBeUndefined();
+ expect(store.get("session", "session-1")).toBeUndefined();
});
});
it("composes atomic turn updates without recreating deleted turns", () => {
contextStorage.run(new ContextContainer(), () => {
const store = new ContextAgentTraceStateStore();
- store.setTurn("session-1", "turn-1", {
+ store.set("turn", JSON.stringify(["session-1", "turn-1"]), {
context: spanContext("1", "3"),
caller: spanContext("4", "2"),
rootSessionId: "session-1",
@@ -172,32 +119,32 @@ describe("ContextAgentTraceStateStore", () => {
startTimeMs: 1_700_000_000_000,
});
- store.updateTurn("session-1", "turn-1", (turn) => ({
+ store.update("turn", JSON.stringify(["session-1", "turn-1"]), (turn) => ({
...turn,
- modelUsage: { inputTokens: 12, outputTokens: 4 },
+ terminal: { outcome: "completed" },
}));
- store.updateTurn("session-1", "turn-1", (turn) => ({
+ store.update("turn", JSON.stringify(["session-1", "turn-1"]), (turn) => ({
...turn,
- terminal: { type: "turn.completed" },
+ channelDelivery: { channelKind: "http", channelName: "http", deliveryId: "delivery" },
}));
- expect(store.getTurn("session-1", "turn-1")).toMatchObject({
- modelUsage: { inputTokens: 12, outputTokens: 4 },
- terminal: { type: "turn.completed" },
+ expect(store.get("turn", JSON.stringify(["session-1", "turn-1"]))).toMatchObject({
+ terminal: { outcome: "completed" },
+ channelDelivery: { deliveryId: "delivery" },
});
- store.deleteTurn("session-1", "turn-1");
- store.updateTurn("session-1", "turn-1", (turn) => ({
+ store.delete("turn", JSON.stringify(["session-1", "turn-1"]));
+ store.update("turn", JSON.stringify(["session-1", "turn-1"]), (turn) => ({
...turn,
- modelUsage: { inputTokens: 99 },
+ terminal: { outcome: "cancelled" },
}));
- expect(store.getTurn("session-1", "turn-1")).toBeUndefined();
+ expect(store.get("turn", JSON.stringify(["session-1", "turn-1"]))).toBeUndefined();
});
});
it("preserves only trace state from an interrupted context", async () => {
const context = new ContextContainer();
await contextStorage.run(context, () => {
- new ContextAgentTraceStateStore().setSession("session-1", {
+ new ContextAgentTraceStateStore().set("session", "session-1", {
context: spanContext("1", "2"),
rootSessionId: "session-1",
traceSessionId: "session-1",
@@ -226,13 +173,13 @@ describe("readTurnTraceContext", () => {
traceFlags: 1,
traceId: "1".repeat(32),
});
- new ContextAgentTraceStateStore().setSession("session-1", {
+ new ContextAgentTraceStateStore().set("session", "session-1", {
context: spanContext("1", "2"),
decision: { action: "record", recordInputs: true, recordOutputs: false },
rootSessionId: "session-1",
traceSessionId: "session-1",
});
- new ContextAgentTraceStateStore().setTurn("session-1", "turn-1", {
+ new ContextAgentTraceStateStore().set("turn", JSON.stringify(["session-1", "turn-1"]), {
context: spanContext("3", "4"),
rootSessionId: "session-1",
traceSessionId: "session-1",
@@ -258,13 +205,13 @@ describe("readTurnTraceContext", () => {
const context = new ContextContainer();
await contextStorage.run(context, () => {
const store = new ContextAgentTraceStateStore();
- store.setSession("session-1", {
+ store.set("session", "session-1", {
context: spanContext("1", "2"),
decision: { action: "record", recordInputs: false, recordOutputs: true },
rootSessionId: "session-1",
traceSessionId: "session-1",
});
- store.setTurn("session-1", "turn-1", {
+ store.set("turn", JSON.stringify(["session-1", "turn-1"]), {
context: spanContext("3", "4"),
rootSessionId: "session-1",
traceSessionId: "session-1",
@@ -294,19 +241,8 @@ describe("readActionTraceContext", () => {
traceFlags: 1,
traceId: "1".repeat(32),
});
- new ContextAgentTraceStateStore().setAction("action:session-1:turn-1:call-1", {
- attemptIndex: 0,
- callId: "call-1",
- kind: "subagent-call",
- name: "researcher",
- parent: spanContext("1", "2"),
- rootSessionId: "session-1",
- traceSessionId: "session-1",
- sessionId: "session-1",
- spanId: "3".repeat(16),
- startTimeMs: 1_700_000_000_000,
- stepIndex: 0,
- turnId: "turn-1",
+ new ContextAgentTraceStateStore().set("action", "action:session-1:turn-1:call-1", {
+ ...actionState(),
});
});
const serialized = await serializeContext(context);
@@ -329,25 +265,14 @@ describe("readActionTraceContext", () => {
const context = new ContextContainer();
await contextStorage.run(context, () => {
const store = new ContextAgentTraceStateStore();
- store.setSession("session-1", {
+ store.set("session", "session-1", {
context: spanContext("1", "2"),
decision: { action: "record", recordInputs: true, recordOutputs: false },
rootSessionId: "session-1",
traceSessionId: "session-1",
});
- store.setAction("action:session-1:turn-1:call-1", {
- attemptIndex: 0,
- callId: "call-1",
- kind: "subagent-call",
- name: "researcher",
- parent: spanContext("1", "2"),
- rootSessionId: "session-1",
- traceSessionId: "session-1",
- sessionId: "session-1",
- spanId: "3".repeat(16),
- startTimeMs: 1_700_000_000_000,
- stepIndex: 0,
- turnId: "turn-1",
+ store.set("action", "action:session-1:turn-1:call-1", {
+ ...actionState(),
});
});
@@ -362,3 +287,14 @@ describe("readActionTraceContext", () => {
function spanContext(traceId: string, spanId: string) {
return { spanId: spanId.repeat(16), traceFlags: 1, traceId: traceId.repeat(32) };
}
+function actionState() {
+ return {
+ attemptIndex: 0,
+ callId: "call-1",
+ context: spanContext("1", "3"),
+ name: "researcher",
+ sessionId: "session-1",
+ stepIndex: 0,
+ turnId: "turn-1",
+ };
+}
diff --git a/packages/eve/src/tracing/eve/agent-trace-context-store.ts b/packages/eve/src/tracing/eve/agent-trace-context-store.ts
new file mode 100644
index 000000000..3cf4e2dc0
--- /dev/null
+++ b/packages/eve/src/tracing/eve/agent-trace-context-store.ts
@@ -0,0 +1,277 @@
+import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";
+
+import type { SessionTraceContext } from "#channel/types.js";
+import { contextStorage, loadContext } from "#context/container.js";
+import { ContextKey } from "#context/key.js";
+import type { ContextAccessor } from "#context/key.js";
+import { SessionTraceSeedKey, type SessionTraceSeed } from "#context/keys.js";
+import type {
+ AgentActionTraceState,
+ AgentTraceStateStore,
+ AgentTraceValues,
+ AgentSessionTraceState,
+ AgentTurnTraceState,
+} from "#tracing/eve/agent-trace-state.js";
+import { actionIdempotencyKey } from "#instrumentation/lifecycle.js";
+import type { SessionStateMap } from "#harness/types.js";
+import { getBlockingWorkflowToolRuns } from "#harness/workflow-tool-runs.js";
+
+import { createLogger } from "#internal/logging.js";
+import type { InstrumentationDecision } from "#shared/instrumentation-decision.js";
+import {
+ decisionToTraceContentCeiling,
+ resolveForwardedTraceSeed,
+} from "#shared/forwarded-trace-policy.js";
+import { parseJsonObject } from "#shared/json.js";
+
+export const AGENT_TRACE_CONTEXT_KEY = "eve.harness.agentTrace";
+interface AgentTraceContextState {
+ readonly entries: Readonly<Record<string, AgentTraceEntry>>;
+}
+type AgentTraceEntry =
+ | { kind: "session"; value: AgentSessionTraceState }
+ | { kind: "turn"; value: AgentTurnTraceState }
+ | { kind: "checkpoint"; value: AgentTraceValues["checkpoint"] }
+ | { kind: "action"; value: AgentActionTraceState; active: boolean; retained: boolean };
+const traceStateKey = (kind: AgentTraceEntry["kind"], key: string) => JSON.stringify([kind, key]);
+function traceStateValue<K extends AgentTraceEntry["kind"]>(
+ state: AgentTraceContextState | undefined,
+ kind: K,
+ key: string,
+): AgentTraceValues[K] | undefined {
+ return state?.entries[traceStateKey(kind, key)]?.value as AgentTraceValues[K] | undefined;
+}
+const emptyAgentTraceContextState = (): AgentTraceContextState => ({ entries: {} });
+function deserializeAgentTraceContextState(data: unknown): AgentTraceContextState {
+ try {
+ const state = parseJsonObject(data);
+ if (typeof state.entries !== "object" || state.entries === null || Array.isArray(state.entries))
+ return emptyAgentTraceContextState();
+ const entries: Record<string, AgentTraceEntry> = {};
+ for (const [key, value] of Object.entries(state.entries)) {
+ if (value === null || typeof value !== "object" || Array.isArray(value)) continue;
+ const record = value as import("#shared/json.js").JsonObject;
+ if (
+ record.kind !== "session" &&
+ record.kind !== "turn" &&
+ record.kind !== "checkpoint" &&
+ record.kind !== "action"
+ )
+ continue;
+ if (record.value === null || typeof record.value !== "object" || Array.isArray(record.value))
+ continue;
+ entries[key] = value as object as AgentTraceEntry;
+ }
+ return { entries };
+ } catch {
+ return emptyAgentTraceContextState();
+ }
+}
+
+const AgentTraceContextKey = new ContextKey<AgentTraceContextState>(AGENT_TRACE_CONTEXT_KEY, {
+ codec: {
+ deserialize: deserializeAgentTraceContextState,
+ serialize: (state) => state,
+ },
+});
+
+export function currentTraceSessionId(runId: string): string | undefined {
+ return traceStateValue(contextStorage.getStore()?.get(AgentTraceContextKey), "session", runId)
+ ?.traceSessionId;
+}
+
+/** Run after task-provider commits, so a pending workflow tool call keeps its anchor. */
+export function pruneAgentTraceState(
+ context: ContextAccessor,
+ sessionId: string,
+ sessionState: SessionStateMap | undefined,
+): void {
+ try {
+ pruneTraceOwnership(context, sessionId, sessionState);
+ } catch (error) {
+ createLogger("tracing.retention").warn(
+ "could not reconcile trace ownership; preserving trace state",
+ { error },
+ );
+ }
+}
+
+function pruneTraceOwnership(
+ context: ContextAccessor,
+ sessionId: string,
+ sessionState: SessionStateMap | undefined,
+): void {
+ const state = context.get(AgentTraceContextKey);
+ if (state === undefined) return;
+ const calls = new Set(getBlockingWorkflowToolRuns(sessionState).map((run) => run.callId));
+ const entries = Object.fromEntries(
+ Object.entries(state.entries).filter(
+ ([, entry]) =>
+ entry.kind !== "action" ||
+ entry.value.sessionId !== sessionId ||
+ entry.active ||
+ calls.has(entry.value.callId),
+ ),
+ );
+ context.set(AgentTraceContextKey, { ...state, entries });
+}
+
+export function readSessionTraceDecision(
+ context: ContextAccessor,
+ sessionId: string,
+): InstrumentationDecision | undefined {
+ return traceStateValue(context.get(AgentTraceContextKey), "session", sessionId)?.decision;
+}
+
+/** Keeps only framework trace state from an interrupted step's context changes. */
+export function preserveSerializedAgentTraceState(
+ original: Record<string, unknown>,
+ interrupted: Record<string, unknown>,
+): Record<string, unknown> {
+ const traceState = interrupted[AgentTraceContextKey.name];
+ return traceState === undefined
+ ? original
+ : { ...original, [AgentTraceContextKey.name]: traceState };
+}
+
+/** Reads the active turn context straight out of a serialized Workflow context. */
+export function readTurnTraceContext(
+ serializedContext: Readonly<Record<string, unknown>>,
+ sessionId: string,
+ turnId: string,
+): SessionTraceContext | undefined {
+ const raw = serializedContext[AgentTraceContextKey.name];
+ if (raw === undefined) return undefined;
+ const state = deserializeAgentTraceContextState(raw);
+ const turn = traceStateValue(state, "turn", JSON.stringify([sessionId, turnId]));
+ return turn === undefined
+ ? undefined
+ : withTraceDecision(
+ serializedContext,
+ turn.context,
+ traceStateValue(state, "session", sessionId)?.decision,
+ );
+}
+
+/** Reads the durable action span that should parent a dispatched child agent. */
+export function readActionTraceContext(
+ serializedContext: Readonly<Record<string, unknown>>,
+ sessionId: string,
+ turnId: string,
+ callId: string,
+): SessionTraceContext | undefined {
+ const raw = serializedContext[AgentTraceContextKey.name];
+ if (raw === undefined) return undefined;
+ const state = deserializeAgentTraceContextState(raw);
+ const key = actionIdempotencyKey(sessionId, turnId, callId);
+ const action = traceStateValue(state, "action", key);
+ if (action === undefined) return undefined;
+ return withTraceDecision(
+ serializedContext,
+ { ...action.context, isRemote: false },
+ traceStateValue(state, "session", action.sessionId)?.decision,
+ );
+}
+
+function withTraceDecision(
+ serializedContext: Readonly<Record<string, unknown>>,
+ context: SpanContext,
+ storedDecision?: InstrumentationDecision,
+): SessionTraceContext {
+ const seed = serializedContext[SessionTraceSeedKey.name] as SessionTraceSeed | undefined;
+ const traceState = resolveForwardedTraceSeed({
+ decision: storedDecision ?? seed?.decision,
+ forwardedTracePolicy: seed?.forwardedTracePolicy,
+ traceFlags: context.traceFlags,
+ })!;
+ const decision = traceState.decision;
+ const forwardedTracePolicy = traceState.forwardedTracePolicy;
+ const ceiling = decisionToTraceContentCeiling(decision);
+ const resolvedContext = { ...context, traceFlags: traceState.traceFlags };
+ if (forwardedTracePolicy === undefined) {
+ return decision === undefined ? resolvedContext : { ...resolvedContext, decision };
+ }
+ const narrowedForwardedTracePolicy =
+ ceiling === undefined ? forwardedTracePolicy : { ...forwardedTracePolicy, ceiling };
+ if (decision === undefined) {
+ return { ...resolvedContext, forwardedTracePolicy: narrowedForwardedTracePolicy };
+ }
+ return {
+ ...resolvedContext,
+ decision,
+ forwardedTracePolicy: narrowedForwardedTracePolicy,
+ };
+}
+
+/** Durable trace state backed by eve's serialized Workflow context. */
+export class ContextAgentTraceStateStore implements AgentTraceStateStore {
+ get<K extends keyof AgentTraceValues>(kind: K, key: string): AgentTraceValues[K] | undefined {
+ const entry = contextStorage.getStore()?.get(AgentTraceContextKey)?.entries[
+ traceStateKey(kind === "anchor" ? "action" : kind, key)
+ ];
+ if (
+ entry === undefined ||
+ (entry.kind === "action" && !(kind === "anchor" ? entry.retained : entry.active))
+ )
+ return undefined;
+ return entry.value as AgentTraceValues[K];
+ }
+ set<K extends keyof AgentTraceValues>(kind: K, key: string, value: AgentTraceValues[K]): void {
+ updateState((state) => {
+ const id = traceStateKey(kind === "anchor" ? "action" : kind, key);
+ const previous = state.entries[id];
+ const entry: AgentTraceEntry =
+ kind === "action" || kind === "anchor"
+ ? {
+ kind: "action",
+ value: value as AgentActionTraceState,
+ active: kind === "action" || (previous?.kind === "action" && previous.active),
+ retained: kind === "anchor" || (previous?.kind === "action" && previous.retained),
+ }
+ : ({ kind, value } as AgentTraceEntry);
+ return { ...state, entries: { ...state.entries, [id]: entry } };
+ });
+ }
+ delete(kind: keyof AgentTraceValues, key: string): void {
+ updateState((state) => {
+ const id = traceStateKey(kind === "anchor" ? "action" : kind, key);
+ const entry = state.entries[id];
+ const entries = { ...state.entries };
+ if (entry?.kind === "action" && kind === "action" && entry.retained)
+ entries[id] = { ...entry, active: false };
+ else if (entry?.kind === "action" && kind === "anchor" && entry.active)
+ entries[id] = { ...entry, retained: false };
+ else delete entries[id];
+ return { ...state, entries };
+ });
+ }
+ update<K extends keyof AgentTraceValues>(
+ kind: K,
+ key: string,
+ update: (value: AgentTraceValues[K]) => AgentTraceValues[K],
+ ): void {
+ const value = this.get(kind, key);
+ if (value !== undefined) this.set(kind, key, update(value));
+ }
+ entries<K extends keyof AgentTraceValues>(kind: K): [string, AgentTraceValues[K]][] {
+ return Object.entries(
+ contextStorage.getStore()?.get(AgentTraceContextKey)?.entries ?? {},
+ ).flatMap(([key, entry]) =>
+ entry.kind === (kind === "anchor" ? "action" : kind) &&
+ (entry.kind !== "action" || (kind === "anchor" ? entry.retained : entry.active))
+ ? [
+ [JSON.parse(key)[1] as string, entry.value as AgentTraceValues[K]] as [
+ string,
+ AgentTraceValues[K],
+ ],
+ ]
+ : [],
+ );
+ }
+}
+
+function updateState(update: (state: AgentTraceContextState) => AgentTraceContextState): void {
+ loadContext().set(AgentTraceContextKey, (state) =>
+ update(state ?? emptyAgentTraceContextState()),
+ );
+}
diff --git a/packages/eve/src/tracing/eve/agent-trace-context.ts b/packages/eve/src/tracing/eve/agent-trace-context.ts
index 6f5b76364..1a7742871 100644
--- a/packages/eve/src/tracing/eve/agent-trace-context.ts
+++ b/packages/eve/src/tracing/eve/agent-trace-context.ts
@@ -1,4 +1,5 @@
import { createContextKey, type Context } from "#compiled/@opentelemetry/api/index.js";
+import { activeTraceOperation } from "#compiled/@eve/tracing/otel.js";
// OTel context keys use Symbol.for(description), so this marker survives duplicate
// module evaluations while remaining local to the process and async context.
@@ -6,6 +7,7 @@ const AGENT_TRACE_KEY = createContextKey("eve.agent.trace");
export function isAgentTraceContext(context: unknown): boolean {
if (typeof context !== "object" || context === null) return false;
+ if (activeTraceOperation(context as Context) !== undefined) return true;
const getValue = Reflect.get(context, "getValue");
return (
typeof getValue === "function" && Reflect.apply(getValue, context, [AGENT_TRACE_KEY]) === true
diff --git a/packages/eve/src/tracing/eve/agent-trace-state.ts b/packages/eve/src/tracing/eve/agent-trace-state.ts
new file mode 100644
index 000000000..95aaa4ded
--- /dev/null
+++ b/packages/eve/src/tracing/eve/agent-trace-state.ts
@@ -0,0 +1,99 @@
+import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";
+
+import type {
+ InstrumentationParentLineage,
+ InstrumentationPrincipalSummary,
+} from "#instrumentation/lifecycle.js";
+import type { ChannelAudience } from "#shared/channel-audience.js";
+import type { InstrumentationDecision } from "#shared/instrumentation-decision.js";
+import type { TraceSnapshot } from "#compiled/@eve/tracing/index.js";
+
+export interface AgentSessionTraceState {
+ readonly traceSessionId: string;
+ readonly channelAudience?: ChannelAudience;
+ readonly agentName?: string;
+ readonly channelKind?: string;
+ readonly channelType?: string;
+ readonly context: SpanContext;
+ readonly decision?: InstrumentationDecision;
+ readonly parentLineage?: InstrumentationParentLineage;
+ readonly rootSessionId: string;
+ readonly scheduleId?: string;
+ readonly title?: string;
+}
+
+export interface AgentTurnTraceState {
+ readonly agentName?: string;
+ /** Recorded at the turn's terminal event; the span ends at the session transition. */
+ readonly terminal?: {
+ readonly outcome: "cancelled" | "completed" | "failed";
+ readonly errorName?: string;
+ readonly errorMessage?: string;
+ };
+ readonly traceSessionId: string;
+ readonly caller?: SpanContext;
+ readonly channelDelivery?: AgentTurnChannelDeliveryTraceState;
+ readonly context: SpanContext;
+ readonly currentPrincipal?: InstrumentationPrincipalSummary;
+ readonly initiatorPrincipal?: InstrumentationPrincipalSummary;
+ readonly parentLineage?: InstrumentationParentLineage;
+ readonly rootSessionId: string;
+ readonly sequence: number;
+ readonly startTimeMs: number;
+ readonly subagentName?: string;
+}
+
+export interface AgentTurnChannelDeliveryTraceState {
+ readonly channelKind: string;
+ readonly channelName: string;
+ readonly deliveryId: string;
+ readonly inputAttribute?: string;
+ readonly requestId?: string;
+ readonly requestTraceContext?: SpanContext;
+}
+
+/** Locates a dispatched tool call in its turn's trace tree, after the turn's own state is gone. */
+export interface AgentActionTraceState {
+ readonly attemptId?: string;
+ readonly attemptIndex: number;
+ readonly callId: string;
+ readonly channelAudience?: ChannelAudience;
+ readonly context: SpanContext;
+ readonly rootSessionId?: string;
+ readonly sessionId: string;
+ readonly stepIndex: number;
+ /** When the SDK execution finished, for dispatches that report no acceptance time. */
+ readonly toolEndTimeMs?: number;
+ readonly turnId: string;
+}
+
+/** Provider-owned serializable storage for durable agent trace state. */
+export interface AgentTraceStateStore {
+ get<K extends keyof AgentTraceValues>(
+ kind: K,
+ key: string,
+ ): AgentTraceValues[K] | undefined | PromiseLike<AgentTraceValues[K] | undefined>;
+ set<K extends keyof AgentTraceValues>(
+ kind: K,
+ key: string,
+ value: AgentTraceValues[K],
+ ): void | PromiseLike<void>;
+ delete(kind: keyof AgentTraceValues, key: string): void | PromiseLike<void>;
+ update<K extends keyof AgentTraceValues>(
+ kind: K,
+ key: string,
+ update: (value: AgentTraceValues[K]) => AgentTraceValues[K],
+ ): void | PromiseLike<void>;
+ entries<K extends keyof AgentTraceValues>(
+ kind: K,
+ ):
+ | readonly [string, AgentTraceValues[K]][]
+ | PromiseLike<readonly [string, AgentTraceValues[K]][]>;
+}
+export interface AgentTraceValues {
+ session: AgentSessionTraceState;
+ turn: AgentTurnTraceState;
+ action: AgentActionTraceState;
+ anchor: AgentActionTraceState;
+ checkpoint: TraceSnapshot;
+}
diff --git a/packages/eve/src/tracing/eve/batch-span-processor.test.ts b/packages/eve/src/tracing/eve/batch-span-processor.test.ts
index 0cfddea7c..d7dd26fb8 100644
--- a/packages/eve/src/tracing/eve/batch-span-processor.test.ts
+++ b/packages/eve/src/tracing/eve/batch-span-processor.test.ts
@@ -1,11 +1,21 @@
-import type { SpanExporter } from "#compiled/@vercel/otel/index.js";
+import type { SpanExporter } from "./otel-configuration.js";
import { context, TraceFlags } from "@opentelemetry/api";
import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
import { isTracingSuppressed } from "@opentelemetry/core";
import { afterEach, describe, expect, it, vi } from "vitest";
-import { batchSpanProcessor } from "#tracing/eve/batch-span-processor.js";
-import { captureLogRecords } from "#internal/testing/log-records.js";
+import { batchSpanProcessor as createBatchSpanProcessor } from "./batch-span-processor.js";
+const diagnostics = vi.fn();
+
+function batchSpanProcessor(
+ exporter: SpanExporter,
+ options: Parameters<typeof createBatchSpanProcessor>[1] = {},
+) {
+ return createBatchSpanProcessor(exporter, {
+ ...options,
+ diagnostic: diagnostics,
+ });
+}
interface RecordingExporter extends SpanExporter {
readonly batches: readonly (readonly unknown[])[];
@@ -35,6 +45,7 @@ function batchIndexes(batches: readonly (readonly unknown[])[]): number[][] {
describe("batchSpanProcessor", () => {
afterEach(() => {
+ diagnostics.mockClear();
vi.useRealTimers();
context.disable();
});
@@ -81,16 +92,13 @@ describe("batchSpanProcessor", () => {
});
it("logs a refused export rather than failing the turn that produced it", async () => {
- const logs = captureLogRecords();
const processor = batchSpanProcessor(recordingExporter({ fail: true }), {
maxExportBatchSize: 1,
});
processor.onEnd(span(1));
await expect(processor.forceFlush()).resolves.toBeUndefined();
- expect(logs.records).toContainEqual(
- expect.objectContaining({ level: "warn", message: "span export failed" }),
- );
+ expect(diagnostics).toHaveBeenCalledWith("span export failed", expect.anything());
});
it("exports only sampled spans", async () => {
@@ -126,7 +134,6 @@ describe("batchSpanProcessor", () => {
});
it("does not overlap a later batch after the first times out", async () => {
- const logs = captureLogRecords();
vi.useFakeTimers();
const batches: (readonly unknown[])[] = [];
const callbacks: Array<(result: { code: number }) => void> = [];
@@ -159,16 +166,13 @@ describe("batchSpanProcessor", () => {
expect(batchIndexes(batches)).toStrictEqual([[1], [2]]);
callbacks[1]?.({ code: 0 });
await flushed;
- expect(logs.records).toContainEqual(
- expect.objectContaining({
- level: "warn",
- message: "span export timed out; preserving exporter serialization",
- }),
+ expect(diagnostics).toHaveBeenCalledWith(
+ "span export timed out; preserving exporter serialization",
+ expect.anything(),
);
});
it("continues exporting after exporter forceFlush times out", async () => {
- const logs = captureLogRecords();
vi.useFakeTimers();
const exporter = recordingExporter() as RecordingExporter & {
forceFlush: () => Promise<void>;
@@ -188,16 +192,13 @@ describe("batchSpanProcessor", () => {
await vi.advanceTimersByTimeAsync(0);
expect(batchIndexes(exporter.batches)).toStrictEqual([[1], [2]]);
- expect(logs.records).toContainEqual(
- expect.objectContaining({
- level: "warn",
- message: "exporter flush timed out; preserving exporter serialization",
- }),
+ expect(diagnostics).toHaveBeenCalledWith(
+ "exporter flush timed out; preserving exporter serialization",
+ expect.anything(),
);
});
it("drains queued batches and closes the exporter after shutdown times out", async () => {
- const logs = captureLogRecords();
vi.useFakeTimers();
const batches: (readonly unknown[])[] = [];
const callbacks: Array<(result: { code: number }) => void> = [];
@@ -228,11 +229,9 @@ describe("batchSpanProcessor", () => {
callbacks[1]?.({ code: 0 });
await vi.advanceTimersByTimeAsync(0);
expect(shutdown).toHaveBeenCalledOnce();
- expect(logs.records).toContainEqual(
- expect.objectContaining({
- level: "warn",
- message: "span export timed out; preserving exporter serialization",
- }),
+ expect(diagnostics).toHaveBeenCalledWith(
+ "span export timed out; preserving exporter serialization",
+ expect.anything(),
);
});
diff --git a/packages/eve/src/tracing/eve/batch-span-processor.ts b/packages/eve/src/tracing/eve/batch-span-processor.ts
index c343352c6..6148731a8 100644
--- a/packages/eve/src/tracing/eve/batch-span-processor.ts
+++ b/packages/eve/src/tracing/eve/batch-span-processor.ts
@@ -1,9 +1,5 @@
import { TraceFlags, context, createContextKey } from "#compiled/@opentelemetry/api/index.js";
-import type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js";
-
-import { createLogger, formatError } from "#internal/logging.js";
-
-const log = createLogger("harness.batch-span-processor");
+import type { SpanExporter, SpanProcessor } from "./otel-configuration.js";
// This is the Symbol.for-backed key used by @opentelemetry/core's
// suppressTracing(), without pulling that package into eve's runtime.
@@ -19,6 +15,10 @@ const DEFAULTS = {
/** @internal — exposed for tests; authors get the defaults. */
interface BatchSpanProcessorOptions {
+ readonly diagnostic?: (
+ message: string,
+ details: { error?: unknown; timeoutMillis?: number },
+ ) => void;
readonly exportTimeoutMillis?: number;
readonly maxExportBatchSize?: number;
readonly maxQueueSize?: number;
@@ -40,6 +40,11 @@ export function batchSpanProcessor(
exporter: SpanExporter,
options: BatchSpanProcessorOptions = {},
): SpanProcessor {
+ const report = (message: string, details: { error?: unknown; timeoutMillis?: number }) => {
+ try {
+ options.diagnostic?.(message, details);
+ } catch {}
+ };
const exportTimeoutMillis = options.exportTimeoutMillis ?? DEFAULTS.exportTimeoutMillis;
const maxExportBatchSize = options.maxExportBatchSize ?? DEFAULTS.maxExportBatchSize;
const maxQueueSize = options.maxQueueSize ?? DEFAULTS.maxQueueSize;
@@ -93,7 +98,7 @@ export function batchSpanProcessor(
} catch (error: unknown) {
// A destination that cannot take spans is not a reason to fail the turn
// that produced them.
- log.warn("span export failed", { error: formatError(error) });
+ report("span export failed", { error });
}
}
@@ -150,7 +155,7 @@ export function batchSpanProcessor(
let timer: ReturnType<typeof setTimeout> | undefined;
const timeout = new Promise<false>((resolve) => {
timer = setTimeout(() => {
- log.warn(`${name} timed out; preserving exporter serialization`, {
+ report(`${name} timed out; preserving exporter serialization`, {
timeoutMillis: exportTimeoutMillis,
});
resolve(false);
@@ -169,7 +174,7 @@ export function batchSpanProcessor(
try {
await operation.call(exporter);
} catch (error: unknown) {
- log.warn(`${name} failed`, { error: formatError(error) });
+ report(`${name} failed`, { error });
}
}
}
diff --git a/packages/eve/src/tracing/eve/content-span-processor.ts b/packages/eve/src/tracing/eve/content-span-processor.ts
index 5f5f2969b..43c9aca00 100644
--- a/packages/eve/src/tracing/eve/content-span-processor.ts
+++ b/packages/eve/src/tracing/eve/content-span-processor.ts
@@ -3,7 +3,7 @@ import type { SpanProcessor } from "#compiled/@vercel/otel/index.js";
import {
withoutDeclinedContent,
type ResolvedContentOptions,
-} from "#tracing/content-attributes.js";
+} from "#compiled/@eve/tracing/otel.js";
import { hasConversationRelease, type LocalTracesProcessor } from "#tracing/local/traces.js";
import { normalizeChannelAudience } from "#shared/channel-audience.js";
import type { ChannelAudience } from "#shared/channel-audience.js";
diff --git a/packages/eve/src/tracing/eve/delegation.ts b/packages/eve/src/tracing/eve/delegation.ts
new file mode 100644
index 000000000..98980c117
--- /dev/null
+++ b/packages/eve/src/tracing/eve/delegation.ts
@@ -0,0 +1,87 @@
+import { createAgentDelegationTransport } from "#compiled/@eve/tracing/delegation.js";
+import type { AgentSessionContext } from "#execution/agent-sessions/context.js";
+import type { SessionParent, SessionTraceContext } from "#channel/types.js";
+import { resolveConversationId } from "#shared/conversation-identity.js";
+import { intersectInstrumentationDecisions } from "#shared/instrumentation-decision.js";
+import { readConversationBaggage } from "#tracing/eve/conversation-context.js";
+
+const transport = createAgentDelegationTransport();
+
+export const agentDelegation = {
+ fetch: transport.fetch,
+ resume<T>(
+ input: { context: Pick<AgentSessionContext, "trace" | "parent">; agentName: string },
+ execute: () => T,
+ ): T {
+ const { context } = input;
+ const caller = context.trace.parentTraceContext;
+ if (caller === undefined) return execute();
+ const decision = caller.decision;
+ return transport.resume(
+ {
+ caller,
+ conversationId:
+ context.trace.conversationId ?? resolveConversationId(context.parent.rootSessionId),
+ parentRunId: context.parent.sessionId,
+ parentCallId: context.parent.callId,
+ agentName: input.agentName,
+ capture: {
+ emit: caller.traceFlags === 1 && decision?.action !== "drop",
+ recordInputs: decision?.action === "record" && decision.recordInputs,
+ recordOutputs: decision?.action === "record" && decision.recordOutputs,
+ },
+ },
+ execute,
+ );
+ },
+ receive<T>(
+ input: {
+ headers: Headers;
+ delegated: boolean;
+ trusted: boolean;
+ parent?: SessionParent;
+ callerTrace?: SessionTraceContext;
+ parentTraceContext?: SessionTraceContext;
+ },
+ execute: (context: { conversationId?: string; parentTraceContext?: SessionTraceContext }) => T,
+ ): T {
+ return transport.receive(
+ input.headers,
+ (handoff) =>
+ input.delegated &&
+ input.trusted &&
+ input.parent !== undefined &&
+ handoff.parentRunId === input.parent.sessionId &&
+ handoff.parentCallId === input.parent.callId &&
+ input.callerTrace?.traceId === handoff.caller.traceId &&
+ input.callerTrace.spanId === handoff.caller.spanId,
+ (handoff) => {
+ const parentTrace = input.parentTraceContext;
+ const decision = handoff?.capture.emit
+ ? {
+ action: "record" as const,
+ recordInputs: handoff.capture.recordInputs,
+ recordOutputs: handoff.capture.recordOutputs,
+ }
+ : { action: "drop" as const };
+ const inheritedTrace =
+ handoff === undefined || parentTrace === undefined
+ ? parentTrace
+ : {
+ ...parentTrace,
+ decision:
+ parentTrace.decision === undefined
+ ? decision
+ : intersectInstrumentationDecisions(parentTrace.decision, decision),
+ traceFlags: handoff.capture.emit ? parentTrace.traceFlags : 0,
+ };
+ return execute({
+ conversationId: input.delegated
+ ? (handoff?.conversationId ?? readConversationBaggage(input.headers.get("baggage")))
+ : undefined,
+ parentTraceContext: inheritedTrace,
+ });
+ },
+ );
+ },
+};
diff --git a/packages/eve/src/tracing/eve/gateway.ts b/packages/eve/src/tracing/eve/gateway.ts
new file mode 100644
index 000000000..4d255597b
--- /dev/null
+++ b/packages/eve/src/tracing/eve/gateway.ts
@@ -0,0 +1,41 @@
+/** Converts Gateway response metadata to cost data. */
+import type { Attributes } from "#compiled/@eve/tracing/index.js";
+export function gatewayCostAttributes(input: {
+ cost?: number;
+ gatewayCost?: number;
+ inputCost?: number;
+ outputCost?: number;
+ generationId?: string;
+}): Attributes {
+ return {
+ "gen_ai.usage.cost": input.cost,
+ "gen_ai.usage.gateway_cost": input.gatewayCost,
+ "gen_ai.usage.input_cost": input.inputCost,
+ "gen_ai.usage.output_cost": input.outputCost,
+ "gen_ai.generation.id": input.generationId,
+ };
+}
+export function readGatewayCostData(
+ providerMetadata: Readonly<Record<string, unknown>> | undefined,
+) {
+ if (providerMetadata === undefined) return undefined;
+ const gateway = providerMetadata.gateway;
+ if (typeof gateway !== "object" || gateway === null || Array.isArray(gateway)) return undefined;
+ const data = gateway as Record<string, unknown>;
+ return {
+ cost: readUsd(data.cost),
+ gatewayCost: readUsd(data.gatewayCost),
+ inputCost: readUsd(data.inputInferenceCost),
+ outputCost: readUsd(data.outputInferenceCost),
+ generationId:
+ typeof data.generationId === "string" && data.generationId.length > 0
+ ? data.generationId
+ : undefined,
+ };
+}
+
+function readUsd(value: unknown): number | undefined {
+ if (typeof value !== "string" || value.trim() === "") return undefined;
+ const parsed = Number(value);
+ return Number.isFinite(parsed) ? parsed : undefined;
+}
diff --git a/packages/eve/src/tracing/eve/install-instrumentation-runtime.test.ts b/packages/eve/src/tracing/eve/install-instrumentation-runtime.test.ts
index b7c4b217f..6180f9e21 100644
--- a/packages/eve/src/tracing/eve/install-instrumentation-runtime.test.ts
+++ b/packages/eve/src/tracing/eve/install-instrumentation-runtime.test.ts
@@ -12,22 +12,20 @@ const { forceFlush, internalTerminalState, shutdown } = vi.hoisted(() => ({
shutdown: vi.fn(async () => undefined),
}));
-vi.mock("#tracing/eve/otel-registration.js", async (importOriginal) => {
- const actual = await importOriginal<typeof import("#tracing/eve/otel-registration.js")>();
- return {
- ...actual,
- registerOtelPipeline: () => ({
- forceFlush,
- idGenerator: {
- allocateSpanId: () => "1".repeat(16),
- withSpanId: (_spanId: string, run: () => unknown) => run(),
- },
- shutdown,
- }),
- };
-});
+vi.mock("#tracing/eve/otel-registration.js", () => ({
+ registerOtel: () => ({
+ forceFlush,
+ shutdown,
+ provider: { getTracer: () => ({}) },
+ idGenerator: {
+ allocateSpanId: () => "1".repeat(16),
+ withSpanId: (_spanId: string, run: () => unknown) => run(),
+ },
+ samplesTrace: () => true,
+ }),
+}));
-vi.mock("#tracing/agent-otel-provider.js", () => ({
+vi.mock("#tracing/eve/agent-otel-provider.js", () => ({
createAgentOtelInstrumentation: () => ({
hook: {
events: {
diff --git a/packages/eve/src/tracing/eve/install-instrumentation-runtime.ts b/packages/eve/src/tracing/eve/install-instrumentation-runtime.ts
index 5cfd2a751..753b4f683 100644
--- a/packages/eve/src/tracing/eve/install-instrumentation-runtime.ts
+++ b/packages/eve/src/tracing/eve/install-instrumentation-runtime.ts
@@ -1,4 +1,3 @@
-import { trace } from "#compiled/@opentelemetry/api/index.js";
import type { SpanProcessor } from "#compiled/@vercel/otel/index.js";
import { contextStorage } from "#context/container.js";
@@ -13,15 +12,15 @@ import {
} from "#instrumentation/runtime.js";
import { createLogger, formatError } from "#internal/logging.js";
import { resolveInstrumentationEnvironment } from "#internal/application/dev-environment.js";
-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
-import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";
-import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js";
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
+import { ContextAgentTraceStateStore } from "#tracing/eve/agent-trace-context-store.js";
+import { createAgentOtelInstrumentation } from "#tracing/eve/agent-otel-provider.js";
import { hasConversationRelease, type LocalTracesProcessor } from "#tracing/local/traces.js";
import type { CollectedOtel, RuntimeContextResolver } from "#tracing/eve/otel-declaration.js";
-import {
- registerOtelPipeline,
- type RegisteredOtelPipeline,
-} from "#tracing/eve/otel-registration.js";
+import { otelTelemetry } from "#compiled/@eve/tracing/index.js";
+import type { RegisteredOtelPipeline } from "#tracing/eve/otel-registration.js";
+import { registerOtel } from "#tracing/eve/otel-registration.js";
+import { eveOutputMapping } from "#tracing/eve/profile.js";
import { readConversationId } from "#shared/conversation-identity.js";
const log = createLogger("tracing.install-instrumentation-runtime");
@@ -48,21 +47,29 @@ export function installInstrumentationRuntime(input: {
let prepareSessionTrace: InstrumentationRuntime["prepareSessionTrace"];
let prepareTurnTrace: InstrumentationRuntime["prepareTurnTrace"];
let runInContext: InstrumentationRuntime["runInContext"] = (_operation, execute) => execute();
+ let idGenerator: AgentSpanIdGenerator;
if (input.collected.declared) {
- otelRuntime = registerOtelPipeline({
- pipeline: input.collected.pipeline,
+ otelRuntime = registerOtel({
serviceName: input.serviceName,
+ otel: input.collected.configuration,
});
+ idGenerator = otelRuntime.idGenerator;
const agentOtel = createAgentOtelInstrumentation({
environment: resolveInstrumentationEnvironment(),
frameworkVersion: input.frameworkVersion,
- idGenerator: otelRuntime.idGenerator,
+ telemetry: otelTelemetry({
+ provider: otelRuntime.provider,
+ tracerName: "eve.agent",
+ tracerVersion: input.frameworkVersion,
+ idGenerator,
+ samplesTrace: otelRuntime.samplesTrace,
+ mapping: eveOutputMapping(),
+ }),
+ idGenerator,
recordInputs: input.collected.settings.recordInputs,
recordOutputs: input.collected.settings.recordOutputs,
- samplesTrace: otelRuntime.samplesTrace,
stateStore: new ContextAgentTraceStateStore(),
- tracer: trace.getTracer("eve.agent", input.frameworkVersion),
tracePolicy: input.collected.settings.tracePolicy,
});
// The span must exist before authored providers observe the lifecycle event.
@@ -71,10 +78,12 @@ export function installInstrumentationRuntime(input: {
prepareTurnTrace = agentOtel.prepareTurnTrace;
runInContext = agentOtel.runInContext;
- const releasable = input.collected.pipeline.spanProcessors
+ const releasable = input.collected.configuration.spanProcessors
.filter(isSpanProcessor)
.filter(hasConversationRelease);
if (releasable.length > 0) serialAfter.push(sessionReleaseProvider(releasable));
+ } else {
+ idGenerator = new AgentSpanIdGenerator();
}
const allProviders = [...serialBefore, ...input.providers, ...serialAfter];
@@ -89,7 +98,7 @@ export function installInstrumentationRuntime(input: {
serialAfter,
serialBefore,
}),
- idGenerator: otelRuntime?.idGenerator ?? new AgentSpanIdGenerator(),
+ idGenerator,
memoryOperations: otelRuntime !== undefined || input.providers.some(hasMemoryOperationHandler),
otelSettings: input.collected.declared ? input.collected.settings : undefined,
ownsAgentSpans: otelRuntime !== undefined,
diff --git a/packages/eve/src/tracing/eve/invoke-tool-span.ts b/packages/eve/src/tracing/eve/invoke-tool-span.ts
index 09ae1b377..5ddd23e55 100644
--- a/packages/eve/src/tracing/eve/invoke-tool-span.ts
+++ b/packages/eve/src/tracing/eve/invoke-tool-span.ts
@@ -1,11 +1,4 @@
-import {
- context as otelContext,
- SpanKind,
- trace,
- type Attributes,
- type Context,
- type Span,
-} from "#compiled/@opentelemetry/api/index.js";
+import { context as otelContext, type Context } from "#compiled/@opentelemetry/api/index.js";
import type { InvokeToolResult } from "#channel/invoke-tool.js";
import type { SessionAuthContext } from "#channel/types.js";
@@ -21,21 +14,19 @@ import { resolveInstrumentationEnvironment } from "#internal/application/dev-env
import { createLogger, formatError } from "#internal/logging.js";
import { applyLiveDeliveryAudienceCeiling } from "#shared/forwarded-trace-policy.js";
import { resolveTracePolicy } from "#shared/trace-policy.js";
-import { agentTraceIdentityAttributes } from "#tracing/agent-otel-attributes.js";
-import { contentAttribute } from "#tracing/agent-otel-content.js";
-import { recordAgentSpanError } from "#tracing/agent-span-error.js";
-import {
- DIRECT_TOOL_CALL_ATTRIBUTE,
- DIRECT_TOOL_CALL_VALUE,
-} from "#tracing/agent-span-contract.js";
-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";
+import { otelTelemetry, type Attributes, type SpanWriter } from "#compiled/@eve/tracing/index.js";
+import { contentAttribute, mcpLifecycle, withErrorContent } from "#compiled/@eve/tracing/otel.js";
+import { DIRECT_TOOL_CALL_ATTRIBUTE, DIRECT_TOOL_CALL_VALUE } from "#tracing/local/inspection.js";
import { markAgentTraceContext } from "#tracing/eve/agent-trace-context.js";
-import { withAgentToolSpanContext } from "#tracing/agent-tool-span-context.js";
import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";
-import { withErrorContent } from "#tracing/error-content-context.js";
-import { suppressTracing } from "#tracing/suppress-tracing.js";
+import { operationConversationId } from "#tracing/eve/conversation-context.js";
+import { MCP_SERIALIZER } from "#tracing/eve/mcp.js";
+import { eveOutputMapping } from "#tracing/eve/profile.js";
const log = createLogger("invoke-tool");
+// Built per call: the global tracer provider can be registered after this module loads.
+const agentTelemetry = () =>
+ otelTelemetry({ tracerName: "eve.agent", mapping: eveOutputMapping() });
/** Where a direct tool call came from, so trace policy can classify it like a conversation. */
export interface InvokeToolTraceOrigin {
@@ -109,6 +100,7 @@ async function traced(
): Promise<InvokeToolResult> {
const runtime = getInstrumentationRuntime();
if (runtime === undefined) return await run(silentObserver);
+ const telemetry = agentTelemetry();
const conversation = buildConversationContext(
{
@@ -147,7 +139,7 @@ async function traced(
}),
);
if (decision.action === "drop") {
- return await otelContext.with(suppressTracing(base), () => run(events.observer(undefined)));
+ return await telemetry.suppressed!(() => run(events.observer(undefined)));
}
const content = applyLiveDeliveryAudienceCeiling(
{
@@ -164,7 +156,7 @@ async function traced(
const spanName = `execute_tool ${input.toolName}`;
const parent = withChannelAudience(base, conversation.audience);
- const attributes: Attributes = {
+ const attributes: Record<string, Attributes[string]> = {
"agent.tool.is_framework": false,
"gen_ai.agent.name": input.agentName,
"gen_ai.operation.name": "execute_tool",
@@ -172,30 +164,30 @@ async function traced(
"gen_ai.tool.name": input.toolName,
"gen_ai.tool.type": "function",
[DIRECT_TOOL_CALL_ATTRIBUTE]: DIRECT_TOOL_CALL_VALUE,
- ...agentSpanNamingAttributes(spanName, "execute_tool"),
- ...agentTraceIdentityAttributes({
- rootSessionId: input.sessionId,
- sessionId: input.sessionId,
- traceSessionId: input.sessionId,
- }),
+ "operation.name": "execute_tool",
+ "resource.name": spanName,
+ "agent.run.id": input.sessionId,
+ "agent.trace.schema.version": 4,
+ "gen_ai.conversation.id": operationConversationId({ sessionId: input.sessionId }),
};
+ if (process.env.VERCEL_ENV !== undefined) attributes["vercel.session_id"] = input.sessionId;
if (input.origin !== undefined) {
attributes["eve.channel.kind"] = conversation.channel.kind;
attributes["eve.channel.name"] = input.origin.channelName;
}
- const span = trace
- .getTracer("eve.agent")
- .startSpan(spanName, { attributes, kind: SpanKind.INTERNAL }, parent);
- // As on a conversation's tool span: error text inside the call follows the output decision.
- const active = markAgentTraceContext(
- withAgentToolSpanContext(withErrorContent(trace.setSpan(parent, span), recordOutputs), {
- recordInputs,
- recordOutputs,
- recordError: (error, errorType) =>
- recordAgentSpanError(span, recordOutputs ? error : undefined, errorType),
- setAttributes: (attributes) => setDefined(span, attributes),
- }),
+ const span = telemetry.startSpan(
+ { type: "tool", operationId: input.callId, name: spanName, kind: "INTERNAL", attributes },
+ { context: parent },
);
+ const capture = { emit: true, recordInputs, recordOutputs };
+ // As on a conversation's tool span: error text inside the call follows the output decision.
+ const host = markAgentTraceContext(withErrorContent(parent, recordOutputs));
+ const mcp = mcpLifecycle({
+ serializer: MCP_SERIALIZER,
+ ...capture,
+ write: (attributes) => setDefined(span, attributes),
+ error: (error, errorType) => span.fail(recordOutputs ? error : undefined, errorType),
+ });
let failure: { readonly error: unknown } | undefined;
const observer = events.observer({
@@ -206,7 +198,11 @@ async function traced(
span,
});
try {
- const result = await otelContext.with(active, () => run(observer));
+ const result = await telemetry.run(
+ { type: "tool", reference: span.reference, capture, mcp },
+ () => run(observer),
+ host,
+ );
span.setAttribute("eve.tool.outcome", result.status);
if (result.status === "completed" && recordOutputs) {
const output = contentAttribute(result.output);
@@ -214,18 +210,17 @@ async function traced(
}
if (result.status === "invalid-input") {
// A model-issued call with invalid input never runs, so it has no span to compare to.
- recordAgentSpanError(span, undefined, result.status);
+ span.fail(undefined, result.status);
} else if (result.status === "failed") {
// The conversation's rule: the error itself when outputs are recorded, else a generic one.
- recordAgentSpanError(
- span,
+ span.fail(
recordOutputs ? failure?.error : undefined,
failure === undefined ? result.status : undefined,
);
}
return result;
} catch (error) {
- recordAgentSpanError(span, recordOutputs ? error : undefined);
+ span.fail(recordOutputs ? error : undefined);
throw error;
} finally {
span.end();
@@ -260,7 +255,7 @@ function toolCallEvents(
| {
readonly failedWith: (error: unknown) => void;
readonly recordInputs: boolean;
- readonly span: Span;
+ readonly span: SpanWriter;
}
| undefined,
): InvokeToolObserver {
@@ -311,7 +306,7 @@ function toolCallEvents(
};
}
-function setDefined(span: Span, attributes: Attributes): void {
+function setDefined(span: SpanWriter, attributes: Attributes): void {
for (const [name, value] of Object.entries(attributes)) {
if (value !== undefined) span.setAttribute(name, value);
}
diff --git a/packages/eve/src/tracing/eve/logging.integration.test.ts b/packages/eve/src/tracing/eve/logging.integration.test.ts
new file mode 100644
index 000000000..6cdccfd7d
--- /dev/null
+++ b/packages/eve/src/tracing/eve/logging.integration.test.ts
@@ -0,0 +1,43 @@
+import { context, trace } from "@opentelemetry/api";
+import {
+ BasicTracerProvider,
+ InMemorySpanExporter,
+ SimpleSpanProcessor,
+} from "@opentelemetry/sdk-trace-base";
+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
+import { expect, it, vi } from "vitest";
+import { withCapture } from "#compiled/@eve/tracing/otel.js";
+import { createLogger, logError } from "#internal/logging.js";
+
+it("keeps logger-to-span error details behind the active output policy", async () => {
+ const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
+ const exporter = new InMemorySpanExporter();
+ const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] });
+ const manager = new AsyncLocalStorageContextManager().enable();
+ context.setGlobalContextManager(manager);
+ try {
+ for (const allowed of [false, true]) {
+ const span = provider.getTracer("logger").startSpan(`logger-${allowed}`);
+ const active = withCapture(trace.setSpan(context.active(), span), {
+ emit: true,
+ recordInputs: false,
+ recordOutputs: allowed,
+ });
+ context.with(active, () =>
+ logError(createLogger("test"), "operation failed", new Error("sensitive payload")),
+ );
+ span.end();
+ }
+ const [privateSpan, publicSpan] = exporter.getFinishedSpans();
+ expect(privateSpan?.status.code).toBe(2);
+ expect(privateSpan?.status.message).toBeUndefined();
+ expect(privateSpan?.events).toEqual([]);
+ expect(publicSpan?.status.message).toContain("sensitive payload");
+ expect(publicSpan?.events[0]?.attributes?.["exception.message"]).toContain("sensitive payload");
+ } finally {
+ context.disable();
+ manager.disable();
+ consoleError.mockRestore();
+ await provider.shutdown();
+ }
+});
diff --git a/packages/eve/src/tracing/eve/mcp.ts b/packages/eve/src/tracing/eve/mcp.ts
new file mode 100644
index 000000000..7125069b4
--- /dev/null
+++ b/packages/eve/src/tracing/eve/mcp.ts
@@ -0,0 +1,374 @@
+import {
+ context as otelContext,
+ propagation,
+ trace,
+ type Context as OtelContext,
+ type TextMapSetter,
+} from "#compiled/@opentelemetry/api/index.js";
+
+import {
+ otelTelemetry,
+ type CaptureDecision,
+ type ContentSerializer,
+} from "#compiled/@eve/tracing/index.js";
+import {
+ activeTraceOperation,
+ contentAttribute,
+ mcpLifecycle,
+ truncateTelemetryText,
+ type McpLifecycle,
+ type McpUpdate,
+} from "#compiled/@eve/tracing/otel.js";
+import { eveOutputMapping } from "./profile.js";
+function isObject(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+type Context = Pick<OtelContext, "getValue">;
+const MAX_MCP_TRACE_REQUEST_BYTES = 1024 * 1024;
+const MAX_MCP_TRACE_CONTEXT_BYTES = 8192;
+const utf8Encoder = new TextEncoder();
+const traceContextSetter: TextMapSetter<Record<string, string>> = {
+ set(carrier, key, value) {
+ carrier[key] = value;
+ },
+};
+
+export function createMcpTraceFetch(input: {
+ readonly connectionName: string;
+ readonly fetcher?: typeof fetch;
+ readonly getActiveContext?: () => Context;
+ readonly getProtocolVersion: () => string | undefined;
+ readonly injectContext?: McpTraceContextInjector;
+ readonly filterCarrier?: (carrier: Record<string, string>) => void;
+}): typeof fetch {
+ const fetcher = input.fetcher ?? ((request, init) => globalThis.fetch(request, init));
+ const getActiveContext = input.getActiveContext ?? (() => otelContext.active());
+ const injectContext = input.injectContext ?? injectOpenTelemetryTraceContext;
+
+ return async (request, init) => {
+ const message = readJsonRpcRequest(init?.body);
+ if (message === undefined) return await fetcher(request, init);
+
+ const activeContext = getActiveContext();
+ const propagated = injectMcpTraceContext(
+ message,
+ activeContext,
+ injectContext,
+ input.filterCarrier,
+ );
+ const annotateRequest =
+ message.method === "tools/call" ||
+ (message.method === "tools/list" && activeTraceOperation(activeContext)?.type === "mcp");
+ if (annotateRequest) {
+ activeTraceOperation(activeContext)?.mcp?.update(
+ mcpRequestMetadata({
+ connectionName: input.connectionName,
+ message,
+ params: isObject(message.params) ? message.params : {},
+ protocolVersion:
+ headerValue(init?.headers, "mcp-protocol-version") ?? input.getProtocolVersion(),
+ }),
+ );
+ }
+
+ const response = await fetcher(
+ request,
+ propagated === undefined ? init : { ...init, body: JSON.stringify(propagated) },
+ );
+ if (annotateRequest) {
+ const sessionId = response.headers.get("mcp-session-id");
+ if (sessionId !== null) {
+ activeTraceOperation(activeContext)?.mcp?.update({
+ sessionId: truncateTelemetryText(sessionId, 512),
+ });
+ }
+ }
+ return response;
+ };
+}
+
+function injectMcpTraceContext(
+ message: JsonRpcRequest,
+ activeContext: Context,
+ injectContext: McpTraceContextInjector = injectOpenTelemetryTraceContext,
+ filterCarrier?: (carrier: Record<string, string>) => void,
+): JsonRpcRequest | undefined {
+ if (!withinTraceRequestLimit(JSON.stringify(message))) return undefined;
+
+ if (message.params !== undefined && !isObject(message.params)) return undefined;
+ const propagated: Record<string, string> = {};
+ injectContext(activeContext, propagated);
+ if (!withinTraceContextLimit(propagated)) return undefined;
+ filterCarrier?.(propagated);
+
+ const params = isObject(message.params) ? message.params : {};
+ const existingMeta = isObject(params["_meta"]) ? params["_meta"] : {};
+ return {
+ ...message,
+ params: {
+ ...params,
+ _meta: { ...existingMeta, ...propagated },
+ },
+ };
+}
+
+const noContent = () => undefined;
+/** Records only JSON tool arguments and results, as MCP spans carry no model content. */
+export const MCP_SERIALIZER: ContentSerializer = {
+ json: contentAttribute,
+ text: noContent,
+ inputMessages: noContent,
+ instructions: noContent,
+ toolDefinitions: noContent,
+ outputMessages: noContent,
+ toolResults: noContent,
+};
+
+export function createMcpTracing() {
+ const telemetry = otelTelemetry({ tracerName: "eve.mcp", mapping: eveOutputMapping() });
+ function start(input: {
+ method: "tools/list" | "tools/call";
+ connectionName: string;
+ toolName?: string;
+ protocolVersion?: string;
+ executionContext: OtelContext;
+ capture: CaptureDecision;
+ }) {
+ const span = telemetry.startSpan(
+ {
+ type: "mcp",
+ operationId: `${input.connectionName}:${input.method}`,
+ name:
+ input.method === "tools/call"
+ ? `tools/call ${input.toolName ?? "unknown"}`
+ : input.method,
+ kind: "CLIENT",
+ parent: trace.getSpan(input.executionContext)?.spanContext(),
+ attributes: {
+ "agent.connection.name": input.connectionName,
+ "mcp.method.name": input.method,
+ "network.protocol.name": "http",
+ "network.transport": "tcp",
+ "mcp.protocol.version": input.protocolVersion,
+ "gen_ai.operation.name": input.method === "tools/call" ? "execute_tool" : undefined,
+ "gen_ai.tool.name": input.toolName,
+ },
+ },
+ { context: input.executionContext },
+ );
+ const semantic = mcpLifecycle({
+ serializer: MCP_SERIALIZER,
+ ...input.capture,
+ write(attributes) {
+ for (const [key, value] of Object.entries(attributes))
+ if (value !== undefined) span.setAttribute(key, value);
+ },
+ error: span.fail,
+ });
+ return {
+ ...semantic,
+ end: span.end,
+ run<T>(execute: () => T): T {
+ return telemetry.run(
+ { type: "mcp", reference: span.reference, capture: input.capture, mcp: semantic },
+ execute,
+ input.executionContext,
+ );
+ },
+ };
+ }
+ return {
+ async list<T>(input: {
+ readonly connectionName: string;
+ readonly execute: () => Promise<T>;
+ readonly protocolVersion?: string;
+ }): Promise<T> {
+ const parent = otelContext.active();
+ const span = start({
+ method: "tools/list",
+ connectionName: input.connectionName,
+ protocolVersion: input.protocolVersion,
+ executionContext: parent,
+ capture: activeTraceOperation()?.capture ?? {
+ emit: true,
+ recordInputs: false,
+ recordOutputs: false,
+ },
+ });
+ try {
+ return await span.run(async () => {
+ try {
+ return await input.execute();
+ } catch (error) {
+ const code = jsonRpcErrorCode(error);
+ if (code !== undefined) {
+ span.update({ statusCode: code });
+ }
+ span.error(error, errorType(error, code));
+ throw error;
+ }
+ });
+ } finally {
+ span.end();
+ }
+ },
+ async call<T>(input: {
+ readonly arguments: unknown;
+ readonly connectionName: string;
+ readonly execute: () => Promise<T>;
+ readonly protocolVersion?: string;
+ readonly toolName: string;
+ }): Promise<T> {
+ const parent = otelContext.active();
+ const existing = activeTraceOperation();
+ if (existing?.type === "tool" && existing.mcp !== undefined) {
+ existing.mcp.update({ ...input, method: "tools/call" });
+ return await runMcpToolCall(input, undefined);
+ }
+
+ const span = start({
+ method: "tools/call",
+ connectionName: input.connectionName,
+ toolName: truncateTelemetryText(input.toolName, 128),
+ protocolVersion: input.protocolVersion,
+ executionContext: parent,
+ capture: existing?.capture ?? { emit: true, recordInputs: false, recordOutputs: false },
+ });
+ try {
+ return await span.run(() => runMcpToolCall(input, span));
+ } finally {
+ span.end();
+ }
+ },
+ };
+}
+
+function runMcpToolCall<T>(
+ input: {
+ readonly arguments: unknown;
+ readonly execute: () => Promise<T>;
+ },
+ fallbackSpan: McpLifecycle | undefined,
+): Promise<T> {
+ if (fallbackSpan !== undefined) {
+ fallbackSpan.arguments(input.arguments);
+ }
+
+ return input
+ .execute()
+ .then((result) => {
+ if (isToolErrorResult(result)) {
+ activeTraceOperation()?.mcp?.error(undefined, "tool_error");
+ }
+ if (fallbackSpan !== undefined) {
+ fallbackSpan.result(result);
+ }
+ return result;
+ })
+ .catch((error: unknown) => {
+ const code = jsonRpcErrorCode(error);
+ if (code !== undefined) {
+ activeTraceOperation()?.mcp?.update({ statusCode: code });
+ }
+ if (fallbackSpan !== undefined) {
+ activeTraceOperation()?.mcp?.error(error, errorType(error, code));
+ }
+ throw error;
+ });
+}
+
+function mcpRequestMetadata(input: {
+ readonly connectionName: string;
+ readonly message?: JsonRpcRequest;
+ readonly method?: string;
+ readonly params?: Record<string, unknown>;
+ readonly protocolVersion?: string;
+ readonly toolName?: string;
+}): McpUpdate {
+ const method = input.method ?? input.message?.method;
+ const params = input.params ?? (isObject(input.message?.params) ? input.message.params : {});
+ const toolName =
+ input.toolName ?? (typeof params["name"] === "string" ? params["name"] : undefined);
+ const requestId = input.message?.id;
+ const meta = params["_meta"];
+ const protocolVersion =
+ input.protocolVersion ??
+ (isObject(meta) && typeof meta["io.modelcontextprotocol/protocolVersion"] === "string"
+ ? meta["io.modelcontextprotocol/protocolVersion"]
+ : undefined);
+ return {
+ connectionName: input.connectionName,
+ method: method ?? "unknown",
+ toolName,
+ protocolVersion,
+ requestId:
+ typeof requestId === "number" || typeof requestId === "string"
+ ? String(requestId)
+ : undefined,
+ };
+}
+
+interface JsonRpcRequest extends Record<string, unknown> {
+ readonly method: string;
+ readonly params?: unknown;
+}
+
+type McpTraceContextInjector = (context: Context, carrier: Record<string, string>) => void;
+
+function injectOpenTelemetryTraceContext(context: Context, carrier: Record<string, string>): void {
+ propagation.inject(context as OtelContext, carrier, traceContextSetter);
+}
+
+function readJsonRpcRequest(body: RequestInit["body"]): JsonRpcRequest | undefined {
+ if (typeof body !== "string" || !withinTraceRequestLimit(body)) return undefined;
+ try {
+ const value: unknown = JSON.parse(body);
+ return isObject(value) && typeof value["method"] === "string"
+ ? (value as JsonRpcRequest)
+ : undefined;
+ } catch {
+ return undefined;
+ }
+}
+
+function withinTraceRequestLimit(value: string): boolean {
+ return (
+ value.length <= MAX_MCP_TRACE_REQUEST_BYTES &&
+ utf8Encoder.encode(value).byteLength <= MAX_MCP_TRACE_REQUEST_BYTES
+ );
+}
+
+function withinTraceContextLimit(carrier: Readonly<Record<string, string>>): boolean {
+ let bytes = 0;
+ for (const [key, value] of Object.entries(carrier)) {
+ if (key.length + value.length > MAX_MCP_TRACE_CONTEXT_BYTES) return false;
+ bytes += utf8Encoder.encode(key).byteLength + utf8Encoder.encode(value).byteLength + 2;
+ if (bytes > MAX_MCP_TRACE_CONTEXT_BYTES) return false;
+ }
+ return bytes > 0;
+}
+
+function headerValue(headers: RequestInit["headers"], name: string): string | undefined {
+ if (headers === undefined) return undefined;
+ try {
+ return new Headers(headers).get(name) ?? undefined;
+ } catch {
+ return undefined;
+ }
+}
+
+function isToolErrorResult(value: unknown): boolean {
+ return isObject(value) && value["isError"] === true;
+}
+
+function jsonRpcErrorCode(error: unknown): string | undefined {
+ if (!isObject(error)) return undefined;
+ const code = error["code"];
+ return typeof code === "number" || typeof code === "string" ? String(code) : undefined;
+}
+
+function errorType(error: unknown, code: string | undefined): string | undefined {
+ if (code !== undefined) return code;
+ return error instanceof Error ? error.name || "Error" : undefined;
+}
diff --git a/packages/eve/src/tracing/eve/metadata.ts b/packages/eve/src/tracing/eve/metadata.ts
new file mode 100644
index 000000000..964cf8dcc
--- /dev/null
+++ b/packages/eve/src/tracing/eve/metadata.ts
@@ -0,0 +1,71 @@
+import type {
+ AgentSessionTraceState,
+ AgentTurnTraceState,
+} from "#tracing/eve/agent-trace-state.js";
+import { normalizeInstrumentationChannelKind } from "#internal/instrumentation.js";
+import type { TurnInput } from "#compiled/@eve/tracing/index.js";
+import { checkpointContent } from "#tracing/eve/operation-input.js";
+
+export function eveActivationMetadata(input: {
+ readonly session?: AgentSessionTraceState;
+ readonly turn: AgentTurnTraceState;
+ readonly sessionId: string;
+}): Pick<TurnInput, "sequence" | "lineage" | "channel" | "attributes"> {
+ const { session, turn, sessionId } = input;
+ const lineage = turn.parentLineage ?? session?.parentLineage;
+ const owns = lineage === undefined || turn.traceSessionId === sessionId;
+ const delivery = turn.channelDelivery;
+ const kind =
+ delivery?.channelKind ??
+ (!owns
+ ? undefined
+ : (session?.channelKind ??
+ (session?.channelType === undefined
+ ? undefined
+ : normalizeInstrumentationChannelKind(session.channelType))));
+ return {
+ sequence: turn.sequence,
+ lineage:
+ lineage === undefined
+ ? undefined
+ : {
+ agentName: turn.subagentName,
+ parentCallId: lineage.callId,
+ parentRunId: lineage.sessionId,
+ },
+ channel: {
+ kind,
+ origin:
+ !owns || kind === undefined
+ ? undefined
+ : session?.scheduleId === undefined
+ ? "channel"
+ : "schedule",
+ },
+ attributes: {
+ "agent.channel.audience": session?.channelAudience,
+ "agent.session.title":
+ owns && session?.decision?.action === "record" && session.decision.recordInputs
+ ? session.title
+ : undefined,
+ "agent.schedule.id": lineage === undefined ? session?.scheduleId : undefined,
+ "agent.principal.current.type": turn.currentPrincipal?.type,
+ "agent.principal.current.id":
+ turn.currentPrincipal && "id" in turn.currentPrincipal
+ ? turn.currentPrincipal.id
+ : undefined,
+ "agent.principal.initiator.type": turn.initiatorPrincipal?.type,
+ "agent.principal.initiator.id":
+ turn.initiatorPrincipal && "id" in turn.initiatorPrincipal
+ ? turn.initiatorPrincipal.id
+ : undefined,
+ "agent.channel.name": delivery?.channelName,
+ "agent.channel.delivery.id": delivery?.deliveryId,
+ "agent.channel.request.id": delivery?.requestId,
+ "agent.channel.delivery.input":
+ delivery?.inputAttribute === undefined
+ ? undefined
+ : JSON.stringify(checkpointContent(delivery.inputAttribute)),
+ },
+ };
+}
diff --git a/packages/eve/src/tracing/eve/operation-input.ts b/packages/eve/src/tracing/eve/operation-input.ts
new file mode 100644
index 000000000..04cf3772d
--- /dev/null
+++ b/packages/eve/src/tracing/eve/operation-input.ts
@@ -0,0 +1,63 @@
+import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";
+
+import { resolveConversationId } from "#shared/conversation-identity.js";
+import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";
+import {
+ createAgentTracing,
+ type AgentTelemetry,
+ type AgentTracing,
+ type TraceCheckpointer,
+} from "#compiled/@eve/tracing/index.js";
+
+export function traceSessionIdOf(scope: {
+ readonly traceSessionId?: string;
+ readonly rootSessionId?: string;
+ readonly sessionId: string;
+}): string {
+ return scope.traceSessionId ?? scope.rootSessionId ?? scope.sessionId;
+}
+
+export function checkpointContent(value: string | undefined): unknown {
+ if (value === undefined) return undefined;
+ // Historical checkpoints contain truncated JSON; omit payloads, not the operation.
+ try {
+ return JSON.parse(value);
+ } catch {
+ return undefined;
+ }
+}
+
+/** Identity of one eve turn within the shared tracing library. */
+export function eveTurnIdentity(input: {
+ readonly sessionId: string;
+ readonly rootSessionId?: string;
+ readonly turnId: string;
+}) {
+ return {
+ conversationId: resolveConversationId(input.rootSessionId ?? input.sessionId),
+ runId: input.sessionId,
+ turnId: input.turnId,
+ };
+}
+
+/** Content flags are recorded by eve's content processor, not withheld by the library. */
+export function eveCapture(reference: SpanContext | undefined) {
+ return {
+ emit: ((reference?.traceFlags ?? 1) & 1) !== 0,
+ recordInputs: true,
+ recordOutputs: true,
+ };
+}
+
+/** eve's agent tracing, checkpointed in its workflow-context trace state. */
+export function createEveTracing(input: {
+ readonly telemetry: AgentTelemetry;
+ readonly stateStore: AgentTraceStateStore;
+}): AgentTracing {
+ const checkpointer: TraceCheckpointer = {
+ get: (key) => input.stateStore.get("checkpoint", key),
+ set: (key, value) => input.stateStore.set("checkpoint", key, value),
+ delete: (key) => input.stateStore.delete("checkpoint", key),
+ };
+ return createAgentTracing({ telemetry: input.telemetry, checkpointer });
+}
diff --git a/packages/eve/src/tracing/eve/otel-configuration.ts b/packages/eve/src/tracing/eve/otel-configuration.ts
new file mode 100644
index 000000000..9f170056e
--- /dev/null
+++ b/packages/eve/src/tracing/eve/otel-configuration.ts
@@ -0,0 +1,84 @@
+export interface SpanProcessor {
+ forceFlush(): Promise<void>;
+ onEnd(span: unknown): void;
+ onStart(span: unknown, parentContext: unknown): void;
+ shutdown(): Promise<void>;
+}
+
+export type SpanProcessorOrName = SpanProcessor | "auto";
+
+export interface IdGenerator {
+ generateSpanId(): string;
+ generateTraceId(): string;
+}
+
+/**
+ * A `SpanExporter`. Structural for the same reason the propagator is: the
+ * instance comes from whichever `@opentelemetry/*` build the app installed, and
+ * the host only ever hands it spans and waits for the callback.
+ *
+ * `code` is `ExportResultCode`: `0` succeeded, `1` failed.
+ */
+export interface SpanExporter {
+ export(
+ spans: readonly unknown[],
+ resultCallback: (result: { code: number; error?: Error }) => void,
+ ): void;
+ forceFlush?(): Promise<void>;
+ shutdown(): Promise<void>;
+}
+
+/**
+ * A `TextMapPropagator`, or one of the names `@vercel/otel` resolves for you.
+ * Structural rather than imported: the instance comes from whichever
+ * `@opentelemetry/*` build the app installed, not from the host's.
+ */
+export type PropagatorOrName =
+ | { inject(...args: never[]): void; extract(...args: never[]): unknown; fields(): string[] }
+ | "auto"
+ | "none"
+ | "tracecontext"
+ | "baggage";
+
+/** A `Sampler`, or one of the names `@vercel/otel` resolves for you. */
+export type SamplerOrName =
+ | { shouldSample(...args: never[]): unknown; toString(): string }
+ | "auto"
+ | "always_off"
+ | "always_on"
+ | "parentbased_always_off"
+ | "parentbased_always_on"
+ | "parentbased_traceidratio"
+ | "traceidratio";
+
+/**
+ * A `MetricReader`. Structural for the same reason the exporter is: the
+ * instance comes from whichever `@opentelemetry/sdk-metrics` build the app
+ * installed, and the host only ever hands it to `registerOTel`.
+ */
+export interface MetricReader {
+ forceFlush(): Promise<void>;
+ shutdown(): Promise<void>;
+}
+
+export interface Configuration {
+ readonly attributes?: Readonly<Record<string, unknown>>;
+ readonly autoDetectResources?: boolean;
+ readonly idGenerator?: IdGenerator;
+ readonly instrumentations?: readonly unknown[];
+ readonly metricReaders?: readonly MetricReader[];
+ readonly propagators?: readonly PropagatorOrName[];
+ readonly serviceName?: string;
+ readonly spanProcessors?: readonly SpanProcessorOrName[];
+ readonly traceSampler?: SamplerOrName;
+}
+
+/** Resolved process-wide settings. Hosts discover and merge their own declarations. */
+export interface OtelConfiguration {
+ readonly instrumentations?: readonly unknown[];
+ readonly metricReaders?: readonly MetricReader[];
+ readonly propagators?: readonly PropagatorOrName[];
+ readonly resource?: Readonly<Record<string, unknown>>;
+ readonly sampler?: SamplerOrName;
+ readonly spanProcessors: readonly SpanProcessorOrName[];
+}
diff --git a/packages/eve/src/tracing/eve/otel-declaration.test.ts b/packages/eve/src/tracing/eve/otel-declaration.test.ts
index fe46c0784..958592653 100644
--- a/packages/eve/src/tracing/eve/otel-declaration.test.ts
+++ b/packages/eve/src/tracing/eve/otel-declaration.test.ts
@@ -5,7 +5,6 @@ import {
agentRunsIntegration,
collectOtelPipeline,
isOtelDeclaration,
- isOtelIntegration,
managedOtelIntegration,
otel,
otelIntegration,
@@ -47,16 +46,6 @@ describe("otel", () => {
});
describe("otelIntegration", () => {
- it("passes declared processors through untouched", () => {
- const first = processor();
- const integration = otelIntegration({
- spanProcessors: [first],
- });
-
- expect(isOtelIntegration(integration)).toBe(true);
- expect(integration.spanProcessors).toStrictEqual([first]);
- });
-
it("wraps an exporter in a batching processor, after any declared ones", () => {
const first = processor();
const integration = otelIntegration({
@@ -81,15 +70,6 @@ describe("otelIntegration", () => {
});
});
-describe("agentRunsIntegration", () => {
- it("declares the Agent Runs runtime processor", () => {
- const integration = agentRunsIntegration();
-
- expect(integration.spanProcessors).toHaveLength(1);
- expect(integration.spanProcessors[0]).not.toBe("auto");
- });
-});
-
describe("destination export policy", () => {
it("does not redact content unless the export pipeline requests it", () => {
let visibleAttributes: Readonly<Record<string, unknown>> | undefined;
@@ -163,7 +143,7 @@ describe("collectOtelPipeline", () => {
]);
expect(collected.declared).toBe(true);
- expect(collected.pipeline.spanProcessors).toStrictEqual([first, second, third]);
+ expect(collected.configuration.spanProcessors).toStrictEqual([first, second, third]);
});
it("is declared by a destination alone, with no otel() beside it", () => {
@@ -189,7 +169,7 @@ describe("collectOtelPipeline", () => {
}),
]);
- expect(collected.pipeline).toMatchObject({
+ expect(collected.configuration).toMatchObject({
propagators: ["tracecontext"],
resource: { "service.version": "abc" },
sampler: "always_on",
@@ -203,12 +183,6 @@ describe("collectOtelPipeline", () => {
});
});
- it("captures complete spans whenever a destination is declared", () => {
- const collected = collectOtelPipeline([otelIntegration()]);
-
- expect(collected.settings).toMatchObject({ recordInputs: true, recordOutputs: true });
- });
-
// A process has one tracer provider, so letting the first declaration win
// would silently discard the second — the failure this throw exists to stop.
it("refuses a second otel() rather than picking one", () => {
@@ -221,12 +195,6 @@ describe("collectOtelPipeline", () => {
const instrumentation = { name: "test-instrumentation" };
const collected = collectOtelPipeline([otel({ instrumentations: [instrumentation] })]);
- expect(collected.pipeline.instrumentations).toStrictEqual([instrumentation]);
- });
-
- it("defaults to undefined instrumentations when none are declared", () => {
- const collected = collectOtelPipeline([otel()]);
-
- expect(collected.pipeline.instrumentations).toBeUndefined();
+ expect(collected.configuration.instrumentations).toStrictEqual([instrumentation]);
});
});
diff --git a/packages/eve/src/tracing/eve/otel-declaration.ts b/packages/eve/src/tracing/eve/otel-declaration.ts
index 50f2b626b..5f82409b6 100644
--- a/packages/eve/src/tracing/eve/otel-declaration.ts
+++ b/packages/eve/src/tracing/eve/otel-declaration.ts
@@ -13,6 +13,9 @@ import type { JsonObject } from "#shared/json.js";
import { batchSpanProcessor } from "#tracing/eve/batch-span-processor.js";
import { contentFilteringProcessor } from "#tracing/eve/content-span-processor.js";
import { vercelRuntimeSpanProcessor } from "#tracing/eve/vercel-runtime-span-exporter.js";
+import type { OtelConfiguration } from "#tracing/eve/otel-configuration.js";
+import { createLogger, formatError } from "#internal/logging.js";
+const log = createLogger("tracing.export");
import type { TraceCapturePolicy } from "#shared/trace-policy.js";
export type {
TraceCaptureContext,
@@ -178,7 +181,16 @@ function createOtelIntegration(options: OtelIntegrationOptions): OtelIntegration
const spanProcessors =
options.traceExporter === undefined
? declared
- : [...declared, batchSpanProcessor(options.traceExporter)];
+ : [
+ ...declared,
+ batchSpanProcessor(options.traceExporter, {
+ diagnostic: (message, details) =>
+ log.warn(message, {
+ ...details,
+ error: details.error === undefined ? undefined : formatError(details.error),
+ }),
+ }),
+ ];
return {
[OTEL_INTEGRATION]: true,
@@ -226,14 +238,6 @@ export function isOtelIntegration(value: unknown): value is OtelIntegration {
}
/** The one pipeline a process can register. @internal */
-export interface OtelPipeline {
- readonly instrumentations?: readonly unknown[];
- readonly metricReaders?: readonly MetricReader[];
- readonly propagators?: readonly PropagatorOrName[];
- readonly resource?: Readonly<Record<string, unknown>>;
- readonly sampler?: SamplerOrName;
- readonly spanProcessors: readonly SpanProcessorOrName[];
-}
/** What the harness reads at turn time, as opposed to at registration. @internal */
export interface OtelHarnessSettings {
@@ -258,7 +262,7 @@ export interface CollectedOtel {
* global tracer provider slot alone rather than register an empty pipeline.
*/
readonly declared: boolean;
- readonly pipeline: OtelPipeline;
+ readonly configuration: OtelConfiguration;
readonly runtimeContextResolvers: readonly RuntimeContextResolver[];
readonly settings: OtelHarnessSettings;
}
@@ -314,7 +318,7 @@ export function collectOtelPipeline(values: readonly unknown[]): CollectedOtel {
}
return {
declared,
- pipeline: {
+ configuration: {
instrumentations: options.instrumentations,
metricReaders: metricReaders.length > 0 ? metricReaders : undefined,
propagators: options.propagators,
diff --git a/packages/eve/src/tracing/eve/otel-registration.scenario.test.ts b/packages/eve/src/tracing/eve/otel-registration.scenario.test.ts
index 694c77766..47f4d925c 100644
--- a/packages/eve/src/tracing/eve/otel-registration.scenario.test.ts
+++ b/packages/eve/src/tracing/eve/otel-registration.scenario.test.ts
@@ -14,10 +14,8 @@ import {
metrics as runtimeMetrics,
trace as runtimeTrace,
} from "#compiled/@opentelemetry/api/index.js";
-import { registerOtelPipeline } from "#tracing/eve/otel-registration.js";
-import { withErrorContent } from "#tracing/error-content-context.js";
-import { withConversationId } from "#tracing/eve/conversation-context.js";
-import { createLogger, logError } from "#internal/logging.js";
+import { registerOtelPipeline } from "./otel-registration.js";
+import { withConversationId } from "./conversation-context.js";
const require = createRequire(import.meta.url);
const authoredApi = require("@opentelemetry/api") as typeof import("@opentelemetry/api");
@@ -37,7 +35,7 @@ afterEach(() => {
(runtimeTrace as typeof runtimeTrace & { disable(): void }).disable();
});
-describe("registerOtelPipeline", () => {
+describe("OTel registration", () => {
it("samples using the real activation name and attributes without exporting probes", async () => {
const exporter = new InMemorySpanExporter();
const sampler = {
@@ -56,7 +54,7 @@ describe("registerOtelPipeline", () => {
toString: () => "activation-sampler",
};
const runtime = registerOtelPipeline({
- pipeline: { sampler, spanProcessors: [new SimpleSpanProcessor(exporter)] },
+ otel: { sampler, spanProcessors: [new SimpleSpanProcessor(exporter)] },
serviceName: "researcher",
});
const operation = {
@@ -84,7 +82,7 @@ describe("registerOtelPipeline", () => {
async (sampler) => {
vi.stubEnv("OTEL_TRACES_SAMPLER_ARG", "0");
const runtime = registerOtelPipeline({
- pipeline: { sampler, spanProcessors: [] },
+ otel: { sampler, spanProcessors: [] },
serviceName: "test",
});
expect(runtime.samplesTrace("a".repeat(32), { name: "invoke_agent test" })).toBe(false);
@@ -92,31 +90,6 @@ describe("registerOtelPipeline", () => {
},
);
- it("keeps logger-to-span error details behind the active output policy", async () => {
- vi.spyOn(console, "error").mockImplementation(() => {});
- const exporter = new InMemorySpanExporter();
- const runtime = registerOtelPipeline({
- pipeline: { spanProcessors: [new SimpleSpanProcessor(exporter)] },
- serviceName: "test",
- });
- for (const allowed of [false, true]) {
- const span = runtimeTrace.getTracer("eve.agent").startSpan(`logger-${allowed}`);
- const active = withErrorContent(runtimeTrace.setSpan(COMPILED_ROOT_CONTEXT, span), allowed);
- await runtimeContext.with(active, async () => {
- logError(createLogger("test"), "operation failed", new Error("sensitive payload"));
- });
- span.end();
- }
- await runtime.forceFlush();
- const [privateSpan, publicSpan] = exporter.getFinishedSpans();
- expect(privateSpan?.status.code).toBe(2);
- expect(privateSpan?.status.message).toBeUndefined();
- expect(privateSpan?.events).toEqual([]);
- expect(publicSpan?.status.message).toContain("sensitive payload");
- expect(publicSpan?.events[0]?.attributes?.["exception.message"]).toContain("sensitive payload");
- await runtime.shutdown();
- });
-
it("delegates an authored tracer cached before registration", async () => {
const authoredTracer = authoredApi.trace.getTracer("authored");
expect(authoredTracer.startSpan("before-registration").isRecording()).toBe(false);
@@ -124,7 +97,7 @@ describe("registerOtelPipeline", () => {
const exporter = new InMemorySpanExporter();
const processor = new SimpleSpanProcessor(exporter);
const runtime = registerOtelPipeline({
- pipeline: { spanProcessors: [processor] },
+ otel: { spanProcessors: [processor] },
serviceName: "weather",
});
const parent = runtimeTrace.getTracer("eve").startSpan("eve.parent");
@@ -168,38 +141,19 @@ describe("registerOtelPipeline", () => {
await runtime.shutdown();
});
- it("verifies tracer ownership when the sampler records nothing", () => {
- expect(() =>
- registerOtelPipeline({
- pipeline: { sampler: "always_off", spanProcessors: [] },
- serviceName: "weather",
- }),
- ).not.toThrow();
- });
-
- it("reports the installed sampler's verdict for pre-allocated trace ids", () => {
- const runtime = registerOtelPipeline({
- pipeline: { sampler: "always_off", spanProcessors: [] },
- serviceName: "weather",
- });
-
- expect(runtime.samplesTrace(runtime.idGenerator.generateTraceId())).toBe(false);
- });
-
it("passes the pre-allocated trace id to a custom sampler without exporting the probe", async () => {
const seen: string[] = [];
- const sampler: NonNullable<Parameters<typeof registerOtelPipeline>[0]["pipeline"]["sampler"]> =
- {
- shouldSample: (_context: unknown, traceId: string) => {
- seen.push(traceId);
- return { decision: traceId.startsWith("a") ? 2 : 0 };
- },
- toString: () => "test-sampler",
- };
+ const sampler: NonNullable<Parameters<typeof registerOtelPipeline>[0]["otel"]["sampler"]> = {
+ shouldSample: (_context: unknown, traceId: string) => {
+ seen.push(traceId);
+ return { decision: traceId.startsWith("a") ? 2 : 0 };
+ },
+ toString: () => "test-sampler",
+ };
const exporter = new InMemorySpanExporter();
const processor = new SimpleSpanProcessor(exporter);
const runtime = registerOtelPipeline({
- pipeline: {
+ otel: {
sampler,
spanProcessors: [processor],
},
@@ -231,7 +185,7 @@ describe("registerOtelPipeline", () => {
expect(() =>
registerOtelPipeline({
- pipeline: {
+ otel: {
spanProcessors: [
{ forceFlush: async () => {}, onEnd: () => {}, onStart: () => {}, shutdown },
],
@@ -260,7 +214,7 @@ describe("registerOtelPipeline", () => {
expect(() =>
registerOtelPipeline({
- pipeline: { spanProcessors: [] },
+ otel: { spanProcessors: [] },
serviceName: "weather",
}),
).toThrow(/another runtime already owns the global tracer provider/u);
@@ -284,7 +238,7 @@ describe("registerOtelPipeline", () => {
shutdown: vi.fn(async () => {}),
};
const runtime = registerOtelPipeline({
- pipeline: { metricReaders: [reader], spanProcessors: [] },
+ otel: { metricReaders: [reader], spanProcessors: [] },
serviceName: "weather",
});
@@ -305,7 +259,7 @@ describe("registerOtelPipeline", () => {
setTracerProvider: vi.fn(),
};
const runtime = registerOtelPipeline({
- pipeline: { instrumentations: [instrumentation], spanProcessors: [] },
+ otel: { instrumentations: [instrumentation], spanProcessors: [] },
serviceName: "weather",
});
expect(instrumentation.enable).toHaveBeenCalledOnce();
@@ -314,23 +268,6 @@ describe("registerOtelPipeline", () => {
await runtime.shutdown();
expect(instrumentation.disable).toHaveBeenCalledOnce();
});
-
- it("does not export the private registration span", async () => {
- const exporter = new InMemorySpanExporter();
- const processor = new SimpleSpanProcessor(exporter);
- const shutdown = vi.spyOn(processor, "shutdown");
- const runtime = registerOtelPipeline({
- pipeline: { spanProcessors: [processor] },
- serviceName: "weather",
- });
-
- trace.getTracer("test").startSpan("user.work").end();
- await processor.forceFlush();
-
- expect(exporter.getFinishedSpans().map((span) => span.name)).toEqual(["user.work"]);
- await runtime.shutdown();
- expect(shutdown).toHaveBeenCalledOnce();
- });
});
function currentTracerDelegate(): unknown {
diff --git a/packages/eve/src/tracing/eve/otel-registration.test.ts b/packages/eve/src/tracing/eve/otel-registration.test.ts
index 2fcd9f4f9..3b4f33df3 100644
--- a/packages/eve/src/tracing/eve/otel-registration.test.ts
+++ b/packages/eve/src/tracing/eve/otel-registration.test.ts
@@ -1,12 +1,12 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
-import { registerOtelPipeline } from "#tracing/eve/otel-registration.js";
+import { registerOtelPipeline } from "./otel-registration.js";
const { registerOTel } = vi.hoisted(() => ({ registerOTel: vi.fn() }));
vi.mock("#compiled/@vercel/otel/index.js", () => ({ registerOTel }));
-describe("registerOtelPipeline", () => {
+describe("OTel registration", () => {
beforeEach(() => {
delete (globalThis as Record<symbol, unknown>)[Symbol.for("eve.otel.replay-deduplication")];
});
@@ -16,7 +16,7 @@ describe("registerOtelPipeline", () => {
expect(() =>
registerOtelPipeline({
- pipeline: {
+ otel: {
propagators: ["tracecontext"],
resource: { "service.version": "abc" },
sampler: "always_on",
@@ -42,7 +42,7 @@ describe("registerOtelPipeline", () => {
registerOTel.mockImplementation(() => undefined);
expect(() =>
- registerOtelPipeline({ pipeline: { spanProcessors: [] }, serviceName: "weather" }),
+ registerOtelPipeline({ otel: { spanProcessors: [] }, serviceName: "weather" }),
).toThrow(/already owns the global tracer provider/u);
const configuration = registerOTel.mock.calls.at(-1)?.[0] as Record<string, unknown>;
@@ -51,43 +51,11 @@ describe("registerOtelPipeline", () => {
expect(configuration["propagators"]).toEqual(["auto", expect.any(Object)]);
});
- it("filters the private registration span from authored processors", () => {
- const downstream = {
- forceFlush: vi.fn(async () => {}),
- onEnd: vi.fn(),
- onStart: vi.fn(),
- shutdown: vi.fn(async () => {}),
- };
- registerOTel.mockImplementation(() => undefined);
-
- expect(() =>
- registerOtelPipeline({
- pipeline: { spanProcessors: [downstream] },
- serviceName: "weather",
- }),
- ).toThrow();
-
- const configuration = registerOTel.mock.calls.at(-1)?.[0] as {
- spanProcessors: {
- onEnd(span: unknown): void;
- onStart(span: unknown, parentContext: unknown): void;
- }[];
- };
- const processor = configuration.spanProcessors[0]!;
- processor.onStart({ name: "eve.otel.registration" }, {});
- processor.onEnd({ name: "eve.otel.registration" });
- processor.onStart({ name: "agent.turn" }, {});
- processor.onEnd({ name: "agent.turn" });
-
- expect(downstream.onStart).toHaveBeenCalledExactlyOnceWith({ name: "agent.turn" }, {});
- expect(downstream.onEnd).toHaveBeenCalledExactlyOnceWith({ name: "agent.turn" });
- });
-
it("passes Vercel's automatic processor through", () => {
registerOTel.mockImplementation(() => undefined);
expect(() =>
- registerOtelPipeline({ pipeline: { spanProcessors: ["auto"] }, serviceName: "weather" }),
+ registerOtelPipeline({ otel: { spanProcessors: ["auto"] }, serviceName: "weather" }),
).toThrow();
const configuration = registerOTel.mock.calls.at(-1)?.[0] as {
@@ -113,7 +81,7 @@ describe("registerOtelPipeline", () => {
expect(() =>
registerOtelPipeline({
- pipeline: { spanProcessors: [first, second] },
+ otel: { spanProcessors: [first, second] },
serviceName: "weather",
}),
).toThrow();
@@ -153,7 +121,7 @@ describe("registerOtelPipeline", () => {
registerOTel.mockImplementation(() => undefined);
expect(() =>
registerOtelPipeline({
- pipeline: { spanProcessors: [downstream] },
+ otel: { spanProcessors: [downstream] },
serviceName: "weather",
}),
).toThrow();
@@ -176,11 +144,43 @@ describe("registerOtelPipeline", () => {
expect(downstream.onEnd.mock.calls).toEqual([[parent], [child]]);
});
+ it("filters the private registration span from authored processors", () => {
+ const downstream = {
+ forceFlush: vi.fn(async () => {}),
+ onEnd: vi.fn(),
+ onStart: vi.fn(),
+ shutdown: vi.fn(async () => {}),
+ };
+ registerOTel.mockImplementation(() => undefined);
+
+ expect(() =>
+ registerOtelPipeline({
+ otel: { spanProcessors: [downstream] },
+ serviceName: "weather",
+ }),
+ ).toThrow();
+
+ const configuration = registerOTel.mock.calls.at(-1)?.[0] as {
+ spanProcessors: {
+ onEnd(span: unknown): void;
+ onStart(span: unknown, parentContext: unknown): void;
+ }[];
+ };
+ const processor = configuration.spanProcessors[0]!;
+ processor.onStart({ name: "eve.otel.registration" }, {});
+ processor.onEnd({ name: "eve.otel.registration" });
+ processor.onStart({ name: "agent.turn" }, {});
+ processor.onEnd({ name: "agent.turn" });
+
+ expect(downstream.onStart).toHaveBeenCalledExactlyOnceWith({ name: "agent.turn" }, {});
+ expect(downstream.onEnd).toHaveBeenCalledExactlyOnceWith({ name: "agent.turn" });
+ });
+
it("throws when the registration never reached a processor", () => {
registerOTel.mockImplementation(() => undefined);
expect(() =>
- registerOtelPipeline({ pipeline: { spanProcessors: [] }, serviceName: "weather" }),
+ registerOtelPipeline({ otel: { spanProcessors: [] }, serviceName: "weather" }),
).toThrow(/another runtime already owns/u);
});
@@ -261,7 +261,7 @@ function filteringProcessor() {
registerOTel.mockImplementation(() => undefined);
expect(() =>
registerOtelPipeline({
- pipeline: { spanProcessors: [downstream] },
+ otel: { spanProcessors: [downstream] },
serviceName: "weather",
}),
).toThrow();
diff --git a/packages/eve/src/tracing/eve/otel-registration.ts b/packages/eve/src/tracing/eve/otel-registration.ts
index 850b55805..071054cd0 100644
--- a/packages/eve/src/tracing/eve/otel-registration.ts
+++ b/packages/eve/src/tracing/eve/otel-registration.ts
@@ -7,21 +7,18 @@ import {
trace,
SpanKind,
type Context,
+ type TracerProvider,
} from "#compiled/@opentelemetry/api/index.js";
-import {
- registerOTel,
- type Configuration,
- type SpanProcessor,
- type SpanProcessorOrName,
-} from "#compiled/@vercel/otel/index.js";
+import { registerOTel } from "#compiled/@vercel/otel/index.js";
+import type { Configuration, SpanProcessor, SpanProcessorOrName } from "./otel-configuration.js";
+
+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";
+import type { OtelConfiguration } from "./otel-configuration.js";
+import { type Attributes } from "#compiled/@eve/tracing/index.js";
+import { invocationName } from "#compiled/@eve/tracing/otel.js";
+export type SamplingOperation = { readonly name: string; readonly attributes?: Attributes };
-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
import { conversationIdFromContext } from "#tracing/eve/conversation-context.js";
-import type { OtelPipeline } from "#tracing/eve/otel-declaration.js";
-import {
- agentInvocationSpanName,
- type AgentSamplingOperation,
-} from "#tracing/agent-span-contract.js";
const REGISTRATION_SPAN_NAME = "eve.otel.registration";
const REPLAY_DEDUPLICATION_LIMIT = 100_000;
@@ -163,10 +160,10 @@ class PrivateSpanFilteringProcessor implements SpanProcessor {
* propagator, then verifies both through the global APIs.
*/
export function registerOtelPipeline(input: {
- readonly pipeline: OtelPipeline;
+ readonly otel: OtelConfiguration;
readonly serviceName: string;
}): RegisteredOtelPipeline {
- const { pipeline } = input;
+ const pipeline = input.otel;
// A tracer cached before registration retains this private proxy even after
// the vendored API takes the global slot. Delegate it only after ownership is proven.
const optionalPeerTracerProxy = captureOptionalPeerTracerProxy();
@@ -218,6 +215,7 @@ export function registerOtelPipeline(input: {
// global is a no-op provider with no lifecycle methods.
const meterProvider = runtimeMeterProvider();
return {
+ provider: trace.getTracerProvider() as TracerProvider,
forceFlush: async () => {
await Promise.all([provider.forceFlush!(), meterProvider.forceFlush?.()]);
},
@@ -260,17 +258,19 @@ function replayDeduplicationRegistry(): Set<string> {
/** Lifecycle retained from the providers that own every destination. */
export interface RegisteredOtelPipeline {
+ readonly provider: TracerProvider;
readonly forceFlush: () => Promise<void>;
readonly idGenerator: AgentSpanIdGenerator;
/** Whether the installed sampler would record a trace with this id. */
- readonly samplesTrace: (traceId: string, operation?: AgentSamplingOperation) => boolean;
+ readonly samplesTrace: (traceId: string, operation?: SamplingOperation) => boolean;
readonly shutdown: () => Promise<void>;
}
+export const registerOtel = registerOtelPipeline;
function samplerAdmitsTrace(
idGenerator: AgentSpanIdGenerator,
traceId: string,
- operation: AgentSamplingOperation = { name: agentInvocationSpanName(undefined) },
+ operation: SamplingOperation = { name: invocationName(undefined) },
): boolean {
const probe = idGenerator.withTraceId(traceId, () =>
trace.getTracer("eve.registration").startSpan(operation.name, {
@@ -296,20 +296,6 @@ interface ProxyTracerProvider {
getDelegate(): unknown;
}
-interface OptionalPeerTracerProxy {
- setDelegate(delegate: unknown): void;
-}
-
-function captureOptionalPeerTracerProxy(): OptionalPeerTracerProxy | undefined {
- try {
- const api = require("@opentelemetry/api") as typeof import("@opentelemetry/api");
- const provider = api.trace.getTracerProvider();
- return provider instanceof api.ProxyTracerProvider ? provider : undefined;
- } catch {
- return undefined;
- }
-}
-
function rollbackRegistration(input: {
readonly ownsPropagator: boolean;
readonly ownsTracer: boolean;
@@ -417,3 +403,17 @@ function addBounded(values: Set<string>, value: string): void {
function isSpanProcessor(processor: SpanProcessorOrName): processor is SpanProcessor {
return processor !== "auto";
}
+
+interface OptionalPeerTracerProxy {
+ setDelegate(delegate: unknown): void;
+}
+
+function captureOptionalPeerTracerProxy(): OptionalPeerTracerProxy | undefined {
+ try {
+ const api = require("@opentelemetry/api") as typeof import("@opentelemetry/api");
+ const provider = api.trace.getTracerProvider();
+ return provider instanceof api.ProxyTracerProvider ? provider : undefined;
+ } catch {
+ return undefined;
+ }
+}
diff --git a/packages/eve/src/tracing/eve/policy.ts b/packages/eve/src/tracing/eve/policy.ts
new file mode 100644
index 000000000..88164cf82
--- /dev/null
+++ b/packages/eve/src/tracing/eve/policy.ts
@@ -0,0 +1,115 @@
+import { contextStorage } from "#context/container.js";
+import { SessionTraceSeedKey } from "#context/keys.js";
+import { withoutInstrumentationContent } from "#instrumentation/content.js";
+import { instrumentationEventForTraceDecision } from "#instrumentation/content-policy.js";
+import type {
+ InstrumentationEvent,
+ InstrumentationAttemptScope,
+} from "#instrumentation/lifecycle.js";
+import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";
+import { isSampledTrace, resolveTracePolicyDecision } from "#shared/trace-policy.js";
+import {
+ applyLiveDeliveryAudienceCeiling,
+ resolveForwardedTraceSeed,
+} from "#shared/forwarded-trace-policy.js";
+import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";
+import { normalizeChannelAudience, type ChannelAudience } from "#shared/channel-audience.js";
+import type { ConversationEnvironment } from "#shared/conversation-context.js";
+
+export function createEveCapturePolicy(input: {
+ readonly stateStore: AgentTraceStateStore;
+ readonly environment: ConversationEnvironment;
+ readonly recordInputs: boolean;
+ readonly recordOutputs: boolean;
+}) {
+ return {
+ async projectEvent(event: InstrumentationEvent): Promise<InstrumentationEvent> {
+ const session = await input.stateStore.get(
+ "session",
+ "scope" in event ? event.scope.sessionId : event.sessionId,
+ );
+ const audience = audienceForEvent(event, session?.channelAudience);
+ const eventSeed = "traceSeed" in event ? event.traceSeed : undefined;
+ const contextSeed = contextStorage.getStore()?.get(SessionTraceSeedKey);
+ const contextTraceState = resolveForwardedTraceSeed(contextSeed);
+ const eventTraceState = resolveForwardedTraceSeed(
+ eventSeed,
+ contextTraceState?.forwardedTracePolicy,
+ );
+ const decisionForTrace = (trace: { readonly traceFlags: number } | undefined) =>
+ trace === undefined
+ ? undefined
+ : resolveTracePolicyDecision(isSampledTrace(trace), {
+ audience,
+ environment: input.environment,
+ });
+ const decision =
+ eventTraceState?.decision ??
+ contextTraceState?.decision ??
+ readInstrumentationDecision(session?.decision) ??
+ decisionForTrace(eventSeed) ??
+ decisionForTrace(contextSeed) ??
+ decisionForTrace(session?.context);
+ if (decision === undefined) return withoutInstrumentationContent(event);
+ const normalizedEvent =
+ eventTraceState === undefined || !("traceSeed" in event) || event.traceSeed === undefined
+ ? event
+ : {
+ ...event,
+ traceSeed: {
+ ...event.traceSeed,
+ decision: eventTraceState.decision,
+ traceFlags: eventTraceState.traceFlags,
+ },
+ };
+ return instrumentationEventForTraceDecision(
+ normalizedEvent,
+ applyLiveDeliveryAudienceCeiling(
+ decision.action === "drop"
+ ? decision
+ : {
+ action: "record",
+ recordInputs: input.recordInputs && decision.recordInputs,
+ recordOutputs: input.recordOutputs && decision.recordOutputs,
+ },
+ audience,
+ eventTraceState?.forwardedTracePolicy ?? contextTraceState?.forwardedTracePolicy,
+ input.environment,
+ ),
+ { audience, environment: input.environment },
+ { applyAudienceCeiling: false },
+ );
+ },
+ async forOperation(scope: InstrumentationAttemptScope) {
+ const session = await input.stateStore.get("session", scope.sessionId);
+ const seed = resolveForwardedTraceSeed(contextStorage.getStore()?.get(SessionTraceSeedKey));
+ const decision = seed?.decision ?? session?.decision;
+ const effective =
+ decision === undefined
+ ? undefined
+ : applyLiveDeliveryAudienceCeiling(
+ decision,
+ normalizeChannelAudience(scope.channelAudience),
+ seed?.forwardedTracePolicy,
+ input.environment,
+ );
+ return {
+ recordInputs:
+ input.recordInputs && effective?.action === "record" && effective.recordInputs,
+ recordOutputs:
+ input.recordOutputs && effective?.action === "record" && effective.recordOutputs,
+ };
+ },
+ };
+}
+
+function audienceForEvent(
+ event: InstrumentationEvent,
+ sessionAudience: ChannelAudience | undefined,
+): ChannelAudience {
+ if ("delivery" in event) return normalizeChannelAudience(event.delivery.channelAudience);
+ if ("scope" in event && event.scope.channelAudience !== undefined)
+ return normalizeChannelAudience(event.scope.channelAudience);
+ if (event.type === "session.started") return normalizeChannelAudience(event.channelAudience);
+ return normalizeChannelAudience(sessionAudience);
+}
diff --git a/packages/eve/src/tracing/eve/profile.ts b/packages/eve/src/tracing/eve/profile.ts
new file mode 100644
index 000000000..58269bdb5
--- /dev/null
+++ b/packages/eve/src/tracing/eve/profile.ts
@@ -0,0 +1,51 @@
+import type { Attributes, MappingContext, OutputMapping } from "#compiled/@eve/tracing/index.js";
+import { currentTraceSessionId } from "#tracing/eve/agent-trace-context-store.js";
+
+/** eve publishes tool call kind and outcome under its schema-4 `agent.action.*` names. */
+const TOOL_ATTRIBUTES: Readonly<Record<string, string>> = {
+ "agent.tool.kind": "agent.action.kind",
+ "agent.tool.outcome": "agent.action.outcome",
+};
+
+export function eveOutputMapping(options?: {
+ readonly resolve: (span: MappingContext) => {
+ readonly traceSessionId?: string;
+ readonly platform: "vercel" | "other";
+ };
+}): OutputMapping {
+ return {
+ attributes(span, attributes) {
+ const output: Record<string, Attributes[string]> = {};
+ for (const [key, value] of Object.entries(attributes)) {
+ if (key === "agent.connection.name") output["eve.connection.name"] = value;
+ else if (span.type === "tool" && TOOL_ATTRIBUTES[key] !== undefined)
+ output[TOOL_ATTRIBUTES[key]] = value;
+ else if (span.type === "approval" && key === "gen_ai.tool.name")
+ output["agent.action.name"] = value;
+ else if (span.type === "channelRequest" && key === "agent.channel.name")
+ output["eve.channel.name"] = value;
+ else if (span.type === "channelRequest" && key === "agent.channel.kind")
+ output["eve.channel.kind"] = value;
+ else output[key] = value;
+ }
+ if (span.type === "tool" && attributes["agent.tool.kind"] !== undefined)
+ output["agent.action.name"] = attributes["gen_ai.tool.name"];
+ if (attributes["agent.run.id"] !== undefined) {
+ const context = options?.resolve(span) ?? {
+ platform: process.env.VERCEL_ENV === undefined ? "other" : "vercel",
+ traceSessionId:
+ typeof attributes["agent.run.id"] === "string"
+ ? currentTraceSessionId(attributes["agent.run.id"])
+ : undefined,
+ };
+ if (context?.platform === "vercel" && context.traceSessionId !== undefined)
+ output["vercel.session_id"] = context.traceSessionId;
+ }
+ return output;
+ },
+ link: (_span, link) => ({
+ "eve.link.type":
+ link.relationship === "execution.delivery" ? "workflow.delivery" : link.relationship,
+ }),
+ };
+}
diff --git a/packages/eve/src/tracing/eve/session.ts b/packages/eve/src/tracing/eve/session.ts
new file mode 100644
index 000000000..3f040d0c2
--- /dev/null
+++ b/packages/eve/src/tracing/eve/session.ts
@@ -0,0 +1,355 @@
+import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";
+
+import {
+ type InstrumentationSessionStartedEvent,
+ type InstrumentationTraceContext,
+ type InstrumentationTraceSeed,
+ type InstrumentationTurnStartedEvent,
+ type InstrumentationTurnTerminalEvent,
+ type InstrumentationSessionTransitionEvent,
+} from "#instrumentation/lifecycle.js";
+import { normalizeChannelAudience } from "#shared/channel-audience.js";
+import type { ChannelAudience } from "#shared/channel-audience.js";
+import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";
+import {
+ isSampledTrace,
+ resolveTracePolicy,
+ resolveTracePolicyDecision,
+} from "#shared/trace-policy.js";
+import type {
+ AgentSessionTraceState,
+ AgentTraceStateStore,
+} from "#tracing/eve/agent-trace-state.js";
+import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";
+import { eveActivationMetadata } from "#tracing/eve/metadata.js";
+import { eveCapture, eveTurnIdentity } from "#tracing/eve/operation-input.js";
+import type { AgentTurnTraceState } from "#tracing/eve/agent-trace-state.js";
+import { applyPrincipalTraceDecision } from "#instrumentation/principal-summary.js";
+import { normalizeInstrumentationChannelKind } from "#internal/instrumentation.js";
+import type { ConversationEnvironment } from "#shared/conversation-context.js";
+import { traceSessionIdOf } from "#tracing/eve/operation-input.js";
+import { ROOT_CONTEXT } from "#compiled/@opentelemetry/api/index.js";
+import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";
+import { withOperationConversation } from "#tracing/eve/conversation-context.js";
+import type {
+ AgentSpanIdGenerator,
+ AgentTracing,
+ ExecutionContext,
+ TraceLink,
+ TurnOperation,
+} from "#compiled/@eve/tracing/index.js";
+
+interface EveSessionTracingInput {
+ readonly tracing: AgentTracing;
+ readonly idGenerator: AgentSpanIdGenerator;
+ readonly environment: ConversationEnvironment;
+ readonly frameworkVersion: string;
+ readonly stateStore: AgentTraceStateStore;
+ readonly tracePolicy?: TraceCapturePolicy;
+}
+
+type SessionMetadata = Omit<InstrumentationSessionStartedEvent, "idempotencyKey" | "type">;
+
+interface EveSessionTracing {
+ readonly ensureSessionContext: (event: SessionMetadata) => Promise<AgentSessionTraceState>;
+ readonly prepareSessionTrace: (
+ event: InstrumentationSessionStartedEvent,
+ ) => Promise<InstrumentationTraceSeed>;
+ readonly prepareTurnTrace: (
+ event: InstrumentationTurnStartedEvent,
+ ) => Promise<InstrumentationTraceSeed>;
+ /** Hydrates a turn this process or an earlier one started. */
+ turnFor(
+ sessionId: string,
+ turnId: string,
+ context?: ExecutionContext,
+ ): Promise<TurnOperation | undefined>;
+ /** Whether the trace policy dropped this turn, so its work must not record spans. */
+ dropped(sessionId: string, turnId: string): Promise<boolean>;
+ turnTerminal(event: InstrumentationTurnTerminalEvent): Promise<void>;
+ sessionTransition(event: InstrumentationSessionTransitionEvent): Promise<void>;
+}
+
+export function createEveSessionTracing(input: EveSessionTracingInput): EveSessionTracing {
+ const ensureSessionContext = async (event: SessionMetadata): Promise<AgentSessionTraceState> => {
+ let state = await input.stateStore.get("session", event.sessionId);
+ if (state === undefined) {
+ const channelAudience = normalizeChannelAudience(event.channelAudience);
+ const decision = resolveSessionTraceDecision(
+ event,
+ channelAudience,
+ input.environment,
+ input.tracePolicy,
+ );
+ state = {
+ agentName: event.agentName,
+ channelAudience,
+ channelKind: event.channelKind,
+ channelType: event.channelType,
+ decision,
+ context: initialSessionContext(input, event, decision),
+ parentLineage: event.parentLineage,
+ rootSessionId: event.rootSessionId,
+ traceSessionId: traceSessionIdOf(event),
+ scheduleId: event.scheduleId,
+ title: event.title,
+ };
+ await input.stateStore.set("session", event.sessionId, state);
+ }
+ return state;
+ };
+
+ const prepareSessionTrace = async (
+ event: InstrumentationSessionStartedEvent,
+ ): Promise<InstrumentationTraceSeed> => {
+ const session = await ensureSessionContext(event);
+ return portableSpanContext(session.context, session.decision);
+ };
+
+ const turnKey = (sessionId: string, turnId: string) => JSON.stringify([sessionId, turnId]);
+
+ async function turnFor(sessionId: string, turnId: string, context?: ExecutionContext) {
+ const turn = await input.stateStore.get("turn", turnKey(sessionId, turnId));
+ if (turn === undefined || !isSampledTrace(turn.context)) return undefined;
+ const session = await input.stateStore.get("session", sessionId);
+ return input.tracing.resume({
+ identity: eveTurnIdentity({ sessionId, rootSessionId: turn.rootSessionId, turnId }),
+ context: context ?? hostContext(session?.channelAudience, { sessionId, ...turn }),
+ });
+ }
+
+ async function dropped(sessionId: string, turnId: string): Promise<boolean> {
+ const turn = await input.stateStore.get("turn", turnKey(sessionId, turnId));
+ return turn !== undefined && !isSampledTrace(turn.context);
+ }
+
+ const prepareTurnTrace = async (
+ event: InstrumentationTurnStartedEvent,
+ ): Promise<InstrumentationTraceSeed> => {
+ const prepared = await input.stateStore.get("turn", turnKey(event.sessionId, event.turnId));
+ if (prepared !== undefined) {
+ const session = await input.stateStore.get("session", event.sessionId);
+ return portableSpanContext(prepared.context, session?.decision);
+ }
+
+ const session = await ensureSessionContext(event);
+ const useInitialContext = event.sequence === 0;
+ const caller = useInitialContext ? event.parentTraceContext : undefined;
+ const emit = session.decision?.action === "record";
+ // A local caller's first child turn joins the caller's trace beneath its call.
+ const nested = caller !== undefined && !("isRemote" in caller && caller.isRemote === true);
+ const reserved = useInitialContext
+ ? {
+ ...session.context,
+ traceId: nested ? caller.traceId : session.context.traceId,
+ isRemote: false,
+ }
+ : {
+ traceId: input.idGenerator.deriveTraceId(`turn:${event.idempotencyKey}`),
+ spanId: input.idGenerator.deriveSpanId(`turn:${event.idempotencyKey}`),
+ traceFlags: emit ? 1 : 0,
+ isRemote: false,
+ };
+ const agentName =
+ session.agentName ?? (event.parentLineage ?? session.parentLineage)?.subagentName;
+ let turn: AgentTurnTraceState = {
+ agentName,
+ caller: caller === undefined ? undefined : adoptedSpanContext(caller),
+ context: reserved,
+ currentPrincipal: applyPrincipalTraceDecision(event.currentPrincipal, session.decision),
+ initiatorPrincipal: applyPrincipalTraceDecision(event.initiatorPrincipal, session.decision),
+ parentLineage: event.parentLineage ?? session.parentLineage,
+ rootSessionId: event.rootSessionId,
+ traceSessionId: traceSessionIdOf(event),
+ sequence: event.sequence,
+ startTimeMs: Date.now(),
+ subagentName: (event.parentLineage ?? session.parentLineage)?.subagentName,
+ };
+ if (isSampledTrace(reserved)) {
+ const operation = await input.tracing.turn({
+ ...eveActivationMetadata({ session, turn, sessionId: event.sessionId }),
+ agentName,
+ identity: eveTurnIdentity({ ...event, sessionId: event.sessionId }),
+ framework: { name: "eve", version: input.frameworkVersion },
+ capture: eveCapture(reserved),
+ reference: reserved,
+ parent: nested ? adoptedSpanContext(caller) : undefined,
+ links: activationLinks(turn),
+ startTimeMs: turn.startTimeMs,
+ context: hostContext(session.channelAudience, event),
+ });
+ turn = { ...turn, context: { ...reserved, traceFlags: operation.reference.traceFlags } };
+ }
+ await input.stateStore.set("turn", turnKey(event.sessionId, event.turnId), turn);
+ return portableSpanContext(turn.context, session.decision);
+ };
+
+ async function recordTerminal(
+ sessionId: string,
+ turnId: string,
+ terminal: NonNullable<AgentTurnTraceState["terminal"]>,
+ error?: unknown,
+ ) {
+ await input.stateStore.update("turn", turnKey(sessionId, turnId), (turn) => ({
+ ...turn,
+ terminal:
+ terminal.outcome !== "failed"
+ ? terminal
+ : {
+ outcome: "failed",
+ errorName: error instanceof Error ? error.name : undefined,
+ errorMessage: error instanceof Error ? error.message : undefined,
+ },
+ }));
+ }
+
+ return {
+ ensureSessionContext,
+ prepareSessionTrace,
+ prepareTurnTrace,
+ turnFor,
+ dropped,
+ async turnTerminal(event) {
+ await recordTerminal(
+ event.sessionId,
+ event.turnId,
+ {
+ outcome:
+ event.type === "turn.failed"
+ ? "failed"
+ : event.type === "turn.cancelled"
+ ? "cancelled"
+ : "completed",
+ },
+ event.type === "turn.failed" ? event.error : undefined,
+ );
+ },
+ async sessionTransition(event) {
+ if (event.turnId === undefined) return;
+ if (event.type === "session.failed")
+ await recordTerminal(event.sessionId, event.turnId, { outcome: "failed" }, event.error);
+ const turn = await input.stateStore.get("turn", turnKey(event.sessionId, event.turnId));
+ if (turn === undefined) return;
+ const session = await input.stateStore.get("session", event.sessionId);
+ const operation = await turnFor(event.sessionId, event.turnId);
+ if (operation !== undefined) {
+ const metadata = eveActivationMetadata({ session, turn, sessionId: event.sessionId });
+ operation.attributes(metadata.attributes ?? {});
+ operation.links(activationLinks(turn) ?? []);
+ const terminal = turn.terminal;
+ // A turn the session left without a terminal event ends with no outcome, as on main.
+ if (terminal === undefined) await operation.complete({ outcomeUnknown: true });
+ else if (terminal.outcome === "failed") {
+ const error = new Error(terminal.errorMessage);
+ error.name = terminal.errorName ?? "Error";
+ await operation.complete({
+ outcome: "failed",
+ failed: true,
+ error,
+ errorType: terminal.errorName,
+ });
+ } else await operation.complete({ outcome: terminal.outcome });
+ }
+ await input.stateStore.delete("turn", turnKey(event.sessionId, event.turnId));
+ },
+ };
+}
+
+/** Spans this process starts inherit the channel audience and the operation's conversation. */
+function hostContext(
+ audience: ChannelAudience | undefined,
+ operation: { readonly sessionId: string; readonly rootSessionId?: string },
+) {
+ return withOperationConversation(withChannelAudience(ROOT_CONTEXT, audience), operation);
+}
+
+function activationLinks(turn: AgentTurnTraceState): TraceLink[] | undefined {
+ const links: TraceLink[] = [];
+ // A local caller is the turn's parent; only a remote one is linked.
+ if (turn.caller?.isRemote === true)
+ links.push({ context: turn.caller, relationship: "agent.dispatch" });
+ if (turn.channelDelivery?.requestTraceContext !== undefined)
+ links.push({
+ context: turn.channelDelivery.requestTraceContext,
+ relationship: "channel.request",
+ });
+ return links.length === 0 ? undefined : links;
+}
+
+function portableSpanContext(
+ spanContext: SpanContext,
+ decision?: InstrumentationTraceSeed["decision"],
+): InstrumentationTraceSeed {
+ return {
+ decision,
+ spanId: spanContext.spanId,
+ traceFlags: spanContext.traceFlags,
+ traceId: spanContext.traceId,
+ };
+}
+
+function adoptedSpanContext(handed: InstrumentationTraceContext): SpanContext {
+ return {
+ isRemote: "isRemote" in handed && handed.isRemote === true,
+ spanId: handed.spanId,
+ traceFlags: handed.traceFlags,
+ traceId: handed.traceId,
+ };
+}
+
+function initialSessionContext(
+ input: EveSessionTracingInput,
+ event: SessionMetadata,
+ decision: ReturnType<typeof resolveTracePolicy>,
+): SpanContext {
+ const handed = event.traceSeed;
+ if (handed !== undefined) {
+ return {
+ ...adoptedSpanContext(handed),
+ traceFlags: decision.action === "drop" ? 0 : handed.traceFlags,
+ };
+ }
+ const key = `session:${event.sessionId}`;
+ const parent = event.parentTraceContext;
+ const local = parent !== undefined && !("isRemote" in parent && parent.isRemote === true);
+ return {
+ traceId: local ? parent.traceId : input.idGenerator.deriveTraceId(key),
+ spanId: input.idGenerator.deriveSpanId(key),
+ traceFlags: decision.action === "record" ? 1 : 0,
+ };
+}
+
+function resolveSessionTraceDecision(
+ event: SessionMetadata,
+ audience: ChannelAudience,
+ environment: ConversationEnvironment,
+ policy: TraceCapturePolicy | undefined,
+): ReturnType<typeof resolveTracePolicy> {
+ const content = { audience, environment };
+ if (event.parentTraceContext !== undefined && !isSampledTrace(event.parentTraceContext)) {
+ return { action: "drop" };
+ }
+ if (event.traceSeed?.decision !== undefined) {
+ return readInstrumentationDecision(event.traceSeed.decision) ?? { action: "drop" };
+ }
+ if (event.traceSeed !== undefined) {
+ return resolveTracePolicyDecision(isSampledTrace(event.traceSeed), content);
+ }
+ if (event.parentTraceContext !== undefined) {
+ return resolveTracePolicyDecision(isSampledTrace(event.parentTraceContext), content);
+ }
+ if (event.agentName === undefined) {
+ return policy === undefined ? resolveTracePolicyDecision(true, content) : { action: "drop" };
+ }
+ // The tool loop can evaluate the same policy before this first-session
+ // preparation path; the persisted decision removes that window on replay.
+ return resolveTracePolicy(policy, {
+ agentName: event.agentName,
+ audience,
+ channel: {
+ kind: normalizeInstrumentationChannelKind(event.channelKind ?? event.channelType),
+ },
+ environment,
+ principalType: "unknown",
+ });
+}
diff --git a/packages/eve/src/tracing/eve/telemetry-budget.test.ts b/packages/eve/src/tracing/eve/telemetry-budget.test.ts
index 0eb8de393..a9d498990 100644
--- a/packages/eve/src/tracing/eve/telemetry-budget.test.ts
+++ b/packages/eve/src/tracing/eve/telemetry-budget.test.ts
@@ -3,7 +3,7 @@ import {
boundedPrincipalId,
telemetryByteLength,
truncateTelemetryText,
-} from "#tracing/telemetry-budget.js";
+} from "#compiled/@eve/tracing/otel.js";
describe("boundedPrincipalId", () => {
it.each(["x".repeat(1024), String.fromCodePoint(0x1f600).repeat(256)])(
diff --git a/packages/eve/src/tracing/eve/test-data/remote-local-agent-trace.otlp.json b/packages/eve/src/tracing/eve/test-data/remote-local-agent-trace.otlp.json
index d8c9a0766..e61754306 100644
--- a/packages/eve/src/tracing/eve/test-data/remote-local-agent-trace.otlp.json
+++ b/packages/eve/src/tracing/eve/test-data/remote-local-agent-trace.otlp.json
@@ -45,51 +45,45 @@
"endTimeUnixNano": "1700000000001000000",
"attributes": [
{
- "key": "agent.framework.name",
- "value": {
- "stringValue": "eve"
- }
- },
- {
- "key": "agent.framework.version",
+ "key": "agent.run.id",
"value": {
- "stringValue": "test"
+ "stringValue": "remote-session"
}
},
{
- "key": "agent.name",
+ "key": "agent.trace.schema.version",
"value": {
- "stringValue": "remote"
+ "intValue": 4
}
},
{
- "key": "agent.channel.audience",
+ "key": "gen_ai.conversation.id",
"value": {
- "stringValue": "public"
+ "stringValue": "caller-root"
}
},
{
- "key": "agent.principal.current.id",
+ "key": "agent.framework.name",
"value": {
- "stringValue": "current-user"
+ "stringValue": "eve"
}
},
{
- "key": "agent.principal.current.type",
+ "key": "agent.framework.version",
"value": {
- "stringValue": "service"
+ "stringValue": "test"
}
},
{
- "key": "agent.principal.initiator.id",
+ "key": "operation.name",
"value": {
- "stringValue": "initiator-user"
+ "stringValue": "invoke_agent"
}
},
{
- "key": "agent.principal.initiator.type",
+ "key": "resource.name",
"value": {
- "stringValue": "user"
+ "stringValue": "invoke_agent remote"
}
},
{
@@ -105,87 +99,93 @@
}
},
{
- "key": "agent.parent_call.id",
+ "key": "agent.name",
"value": {
- "stringValue": "remote-call"
+ "stringValue": "remote"
}
},
{
- "key": "agent.parent_run.id",
+ "key": "gen_ai.agent.name",
"value": {
- "stringValue": "caller-root"
+ "stringValue": "remote"
}
},
{
- "key": "agent.run.type",
+ "key": "gen_ai.operation.name",
"value": {
- "stringValue": "subagent"
+ "stringValue": "invoke_agent"
}
},
{
- "key": "agent.trace.content.input",
+ "key": "agent.turn.id",
"value": {
- "boolValue": true
+ "stringValue": "turn_0"
}
},
{
- "key": "agent.trace.content.output",
+ "key": "agent.turn.sequence",
"value": {
- "boolValue": true
+ "intValue": 0
}
},
{
- "key": "agent.turn.id",
+ "key": "agent.run.type",
"value": {
- "stringValue": "turn_0"
+ "stringValue": "subagent"
}
},
{
- "key": "agent.turn.sequence",
+ "key": "agent.parent_call.id",
"value": {
- "intValue": 0
+ "stringValue": "remote-call"
}
},
{
- "key": "gen_ai.agent.name",
+ "key": "agent.parent_run.id",
"value": {
- "stringValue": "remote"
+ "stringValue": "caller-root"
}
},
{
- "key": "gen_ai.operation.name",
+ "key": "agent.trace.content.input",
"value": {
- "stringValue": "invoke_agent"
+ "boolValue": true
}
},
{
- "key": "operation.name",
+ "key": "agent.trace.content.output",
"value": {
- "stringValue": "invoke_agent"
+ "boolValue": true
}
},
{
- "key": "resource.name",
+ "key": "agent.channel.audience",
"value": {
- "stringValue": "invoke_agent remote"
+ "stringValue": "public"
}
},
{
- "key": "agent.run.id",
+ "key": "agent.principal.current.type",
"value": {
- "stringValue": "remote-session"
+ "stringValue": "service"
}
},
{
- "key": "agent.trace.schema.version",
+ "key": "agent.principal.current.id",
"value": {
- "intValue": 4
+ "stringValue": "current-user"
}
},
{
- "key": "gen_ai.conversation.id",
+ "key": "agent.principal.initiator.type",
"value": {
- "stringValue": "caller-root"
+ "stringValue": "user"
+ }
+ },
+ {
+ "key": "agent.principal.initiator.id",
+ "value": {
+ "stringValue": "initiator-user"
}
},
{
@@ -248,51 +248,45 @@
"endTimeUnixNano": "1700000000001000000",
"attributes": [
{
- "key": "agent.framework.name",
- "value": {
- "stringValue": "eve"
- }
- },
- {
- "key": "agent.framework.version",
+ "key": "agent.run.id",
"value": {
- "stringValue": "test"
+ "stringValue": "remote-session"
}
},
{
- "key": "agent.name",
+ "key": "agent.trace.schema.version",
"value": {
- "stringValue": "remote"
+ "intValue": 4
}
},
{
- "key": "agent.channel.audience",
+ "key": "gen_ai.conversation.id",
"value": {
- "stringValue": "public"
+ "stringValue": "caller-root"
}
},
{
- "key": "agent.principal.current.id",
+ "key": "agent.framework.name",
"value": {
- "stringValue": "current-user"
+ "stringValue": "eve"
}
},
{
- "key": "agent.principal.current.type",
+ "key": "agent.framework.version",
"value": {
- "stringValue": "service"
+ "stringValue": "test"
}
},
{
- "key": "agent.principal.initiator.id",
+ "key": "operation.name",
"value": {
- "stringValue": "initiator-user"
+ "stringValue": "invoke_agent"
}
},
{
- "key": "agent.principal.initiator.type",
+ "key": "resource.name",
"value": {
- "stringValue": "user"
+ "stringValue": "invoke_agent remote"
}
},
{
@@ -308,87 +302,93 @@
}
},
{
- "key": "agent.parent_call.id",
+ "key": "agent.name",
"value": {
- "stringValue": "remote-call"
+ "stringValue": "remote"
}
},
{
- "key": "agent.parent_run.id",
+ "key": "gen_ai.agent.name",
"value": {
- "stringValue": "caller-root"
+ "stringValue": "remote"
}
},
{
- "key": "agent.run.type",
+ "key": "gen_ai.operation.name",
"value": {
- "stringValue": "subagent"
+ "stringValue": "invoke_agent"
}
},
{
- "key": "agent.trace.content.input",
+ "key": "agent.turn.id",
"value": {
- "boolValue": true
+ "stringValue": "turn_1"
}
},
{
- "key": "agent.trace.content.output",
+ "key": "agent.turn.sequence",
"value": {
- "boolValue": true
+ "intValue": 1
}
},
{
- "key": "agent.turn.id",
+ "key": "agent.run.type",
"value": {
- "stringValue": "turn_1"
+ "stringValue": "subagent"
}
},
{
- "key": "agent.turn.sequence",
+ "key": "agent.parent_call.id",
"value": {
- "intValue": 1
+ "stringValue": "remote-call"
}
},
{
- "key": "gen_ai.agent.name",
+ "key": "agent.parent_run.id",
"value": {
- "stringValue": "remote"
+ "stringValue": "caller-root"
}
},
{
- "key": "gen_ai.operation.name",
+ "key": "agent.trace.content.input",
"value": {
- "stringValue": "invoke_agent"
+ "boolValue": true
}
},
{
- "key": "operation.name",
+ "key": "agent.trace.content.output",
"value": {
- "stringValue": "invoke_agent"
+ "boolValue": true
}
},
{
- "key": "resource.name",
+ "key": "agent.channel.audience",
"value": {
- "stringValue": "invoke_agent remote"
+ "stringValue": "public"
}
},
{
- "key": "agent.run.id",
+ "key": "agent.principal.current.type",
"value": {
- "stringValue": "remote-session"
+ "stringValue": "service"
}
},
{
- "key": "agent.trace.schema.version",
+ "key": "agent.principal.current.id",
"value": {
- "intValue": 4
+ "stringValue": "current-user"
}
},
{
- "key": "gen_ai.conversation.id",
+ "key": "agent.principal.initiator.type",
"value": {
- "stringValue": "caller-root"
+ "stringValue": "user"
+ }
+ },
+ {
+ "key": "agent.principal.initiator.id",
+ "value": {
+ "stringValue": "initiator-user"
}
},
{
@@ -437,135 +437,135 @@
"endTimeUnixNano": "1700000000001000000",
"attributes": [
{
- "key": "agent.framework.name",
+ "key": "agent.run.id",
"value": {
- "stringValue": "eve"
+ "stringValue": "local-session"
}
},
{
- "key": "agent.framework.version",
+ "key": "agent.trace.schema.version",
"value": {
- "stringValue": "test"
+ "intValue": 4
}
},
{
- "key": "agent.name",
+ "key": "gen_ai.conversation.id",
"value": {
- "stringValue": "local"
+ "stringValue": "caller-root"
}
},
{
- "key": "agent.channel.audience",
+ "key": "agent.framework.name",
"value": {
- "stringValue": "public"
+ "stringValue": "eve"
}
},
{
- "key": "agent.principal.current.id",
+ "key": "agent.framework.version",
"value": {
- "stringValue": "current-user"
+ "stringValue": "test"
}
},
{
- "key": "agent.principal.current.type",
+ "key": "operation.name",
"value": {
- "stringValue": "service"
+ "stringValue": "invoke_agent"
}
},
{
- "key": "agent.principal.initiator.id",
+ "key": "resource.name",
"value": {
- "stringValue": "initiator-user"
+ "stringValue": "invoke_agent local"
}
},
{
- "key": "agent.principal.initiator.type",
+ "key": "agent.name",
"value": {
- "stringValue": "user"
+ "stringValue": "local"
}
},
{
- "key": "agent.parent_call.id",
+ "key": "gen_ai.agent.name",
"value": {
- "stringValue": "local-call"
+ "stringValue": "local"
}
},
{
- "key": "agent.parent_run.id",
+ "key": "gen_ai.operation.name",
"value": {
- "stringValue": "remote-session"
+ "stringValue": "invoke_agent"
}
},
{
- "key": "agent.run.type",
+ "key": "agent.turn.id",
"value": {
- "stringValue": "subagent"
+ "stringValue": "turn_0"
}
},
{
- "key": "agent.trace.content.input",
+ "key": "agent.turn.sequence",
"value": {
- "boolValue": true
+ "intValue": 0
}
},
{
- "key": "agent.trace.content.output",
+ "key": "agent.run.type",
"value": {
- "boolValue": true
+ "stringValue": "subagent"
}
},
{
- "key": "agent.turn.id",
+ "key": "agent.parent_call.id",
"value": {
- "stringValue": "turn_0"
+ "stringValue": "local-call"
}
},
{
- "key": "agent.turn.sequence",
+ "key": "agent.parent_run.id",
"value": {
- "intValue": 0
+ "stringValue": "remote-session"
}
},
{
- "key": "gen_ai.agent.name",
+ "key": "agent.trace.content.input",
"value": {
- "stringValue": "local"
+ "boolValue": true
}
},
{
- "key": "gen_ai.operation.name",
+ "key": "agent.trace.content.output",
"value": {
- "stringValue": "invoke_agent"
+ "boolValue": true
}
},
{
- "key": "operation.name",
+ "key": "agent.channel.audience",
"value": {
- "stringValue": "invoke_agent"
+ "stringValue": "public"
}
},
{
- "key": "resource.name",
+ "key": "agent.principal.current.type",
"value": {
- "stringValue": "invoke_agent local"
+ "stringValue": "service"
}
},
{
- "key": "agent.run.id",
+ "key": "agent.principal.current.id",
"value": {
- "stringValue": "local-session"
+ "stringValue": "current-user"
}
},
{
- "key": "agent.trace.schema.version",
+ "key": "agent.principal.initiator.type",
"value": {
- "intValue": 4
+ "stringValue": "user"
}
},
{
- "key": "gen_ai.conversation.id",
+ "key": "agent.principal.initiator.id",
"value": {
- "stringValue": "caller-root"
+ "stringValue": "initiator-user"
}
},
{
diff --git a/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.test.ts b/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.test.ts
index 6b766492e..de037728c 100644
--- a/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.test.ts
+++ b/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.test.ts
@@ -1,6 +1,6 @@
import { afterEach, describe, expect, it, vi } from "vitest";
-import { vercelRuntimeSpanExporter } from "#tracing/eve/vercel-runtime-span-exporter.js";
+import { vercelRuntimeSpanExporter } from "./vercel-runtime-span-exporter.js";
const { serializeRequest } = vi.hoisted(() => ({
serializeRequest: vi.fn(() => new TextEncoder().encode('{"resourceSpans":[]}')),
@@ -17,7 +17,7 @@ afterEach(() => {
serializeRequest.mockClear();
});
-describe("vercelRuntimeSpanExporter", () => {
+describe("Vercel runtime span export", () => {
it("reports through the current Vercel request context", () => {
const reportSpans = vi.fn();
(globalThis as Record<symbol, unknown>)[REQUEST_CONTEXT] = {
diff --git a/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.ts b/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.ts
index 49d1519ba..c9f27707c 100644
--- a/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.ts
+++ b/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.ts
@@ -1,6 +1,6 @@
import { JsonTraceSerializer } from "#compiled/@opentelemetry/otlp-transformer/index.js";
import { TraceFlags } from "#compiled/@opentelemetry/api/index.js";
-import type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js";
+import type { SpanExporter, SpanProcessor } from "./otel-configuration.js";
import { createLogger, formatError } from "#internal/logging.js";
@@ -29,7 +29,9 @@ export function vercelRuntimeSpanExporter(): SpanExporter {
}
try {
- const serialized = JsonTraceSerializer.serializeRequest([...spans]);
+ const serialized = JsonTraceSerializer.serializeRequest([...spans] as Parameters<
+ typeof JsonTraceSerializer.serializeRequest
+ >[0]);
if (serialized === undefined) throw new Error("Failed to serialize spans.");
telemetry.reportSpans(JSON.parse(new TextDecoder().decode(serialized)) as unknown);
resultCallback({ code: 0 });
diff --git a/packages/eve/src/tracing/local-instrumentation-runtime-conflict.scenario.test.ts b/packages/eve/src/tracing/local-instrumentation-runtime-conflict.scenario.test.ts
deleted file mode 100644
index 73cf2b6b0..000000000
--- a/packages/eve/src/tracing/local-instrumentation-runtime-conflict.scenario.test.ts
+++ /dev/null
@@ -1,30 +0,0 @@
-import { mkdtemp, rm } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-
-import { trace } from "@opentelemetry/api";
-import { BasicTracerProvider } from "@opentelemetry/sdk-trace-base";
-import { afterEach, describe, expect, it } from "vitest";
-
-import { installLocalInstrumentationRuntime } from "#tracing/local/instrumentation-runtime.js";
-
-let appRoot: string | undefined;
-
-afterEach(async () => {
- if (appRoot !== undefined) await rm(appRoot, { force: true, recursive: true });
-});
-
-describe("local instrumentation runtime ownership", () => {
- it("fails loudly when another global tracer provider already exists", async () => {
- appRoot = await mkdtemp(join(tmpdir(), "eve-local-traces-conflict-"));
- expect(trace.setGlobalTracerProvider(new BasicTracerProvider())).toBe(true);
-
- expect(() =>
- installLocalInstrumentationRuntime({
- appRoot: appRoot!,
- frameworkVersion: "test",
- serviceName: "test-agent",
- }),
- ).toThrow(/another runtime already owns the global tracer provider/u);
- });
-});
diff --git a/packages/eve/src/tracing/local/agent-trace-retention.test.ts b/packages/eve/src/tracing/local/agent-trace-retention.test.ts
index 7d7dd9f5b..213777e0c 100644
--- a/packages/eve/src/tracing/local/agent-trace-retention.test.ts
+++ b/packages/eve/src/tracing/local/agent-trace-retention.test.ts
@@ -4,8 +4,8 @@ import { serializeContext } from "#context/serialize.js";
import {
ContextAgentTraceStateStore,
pruneAgentTraceState,
-} from "#tracing/agent-trace-context-store.js";
-import { AGENT_TRACE_CONTEXT_KEY } from "#tracing/agent-trace-context-codec.js";
+} from "#tracing/eve/agent-trace-context-store.js";
+import { AGENT_TRACE_CONTEXT_KEY } from "#tracing/eve/agent-trace-context-store.js";
import { AgentTraceSpanProcessor } from "#tracing/local/agent-trace-span-processor.js";
const anchor = {
@@ -13,12 +13,9 @@ const anchor = {
callId: "call",
kind: "tool-call" as const,
name: "workflow",
- parent: { spanId: "1".repeat(16), traceFlags: 1, traceId: "2".repeat(32) },
+ context: { spanId: "3".repeat(16), traceFlags: 1, traceId: "2".repeat(32) },
rootSessionId: "session",
- traceSessionId: "session",
sessionId: "session",
- spanId: "3".repeat(16),
- startTimeMs: 1,
stepIndex: 0,
turnId: "turn",
};
@@ -28,7 +25,7 @@ describe("trace retention by live work", () => {
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
const context = new ContextContainer();
contextStorage.run(context, () =>
- new ContextAgentTraceStateStore().setActionAnchor("key", anchor),
+ new ContextAgentTraceStateStore().set("anchor", "key", anchor),
);
const before = serializeContext(context);
expect(() =>
@@ -49,7 +46,7 @@ describe("trace retention by live work", () => {
contextStorage.run(context, () => {
const store = new ContextAgentTraceStateStore();
for (let index = 0; index < 1000; index++) {
- store.setActionAnchor(`key-${index}`, {
+ store.set("anchor", `key-${index}`, {
...anchor,
callId: `call-${index}`,
turnId: `turn-${index}`,
@@ -57,7 +54,7 @@ describe("trace retention by live work", () => {
pruneAgentTraceState(context, "session", undefined);
}
});
- expect(serializeContext(context)[AGENT_TRACE_CONTEXT_KEY]).toMatchObject({ actionAnchors: {} });
+ expect(serializeContext(context)[AGENT_TRACE_CONTEXT_KEY]).toMatchObject({ entries: {} });
});
it("unpins completed activation traces only after the writer flushes", async () => {
diff --git a/packages/eve/src/tracing/local/agent-trace-span-processor.ts b/packages/eve/src/tracing/local/agent-trace-span-processor.ts
index 4642f93c7..5286ed521 100644
--- a/packages/eve/src/tracing/local/agent-trace-span-processor.ts
+++ b/packages/eve/src/tracing/local/agent-trace-span-processor.ts
@@ -1,5 +1,5 @@
import type { SpanProcessor } from "#compiled/@vercel/otel/index.js";
-import { isAgentActivationSpan, isDirectToolCallSpan } from "#tracing/agent-span-contract.js";
+import { isAgentActivationSpan, isDirectToolCallSpan } from "#tracing/local/inspection.js";
const REMEMBERED_TRACE_LIMIT = 2048;
diff --git a/packages/eve/src/tracing/local/inspection.ts b/packages/eve/src/tracing/local/inspection.ts
new file mode 100644
index 000000000..24c033a5e
--- /dev/null
+++ b/packages/eve/src/tracing/local/inspection.ts
@@ -0,0 +1,33 @@
+interface SpanRecord {
+ readonly name: string;
+ readonly attributes: Readonly<Record<string, unknown>>;
+}
+
+/** Marks the `execute_tool` span of a tool call made outside any conversation. */
+export const DIRECT_TOOL_CALL_ATTRIBUTE = "eve.tool.invocation";
+export const DIRECT_TOOL_CALL_VALUE = "direct";
+
+/**
+ * A direct tool call has no turn, so its own span is what starts and ends
+ * its claim on the trace, the way an activation span does for a turn.
+ */
+export function isDirectToolCallSpan(span: Pick<SpanRecord, "attributes">): boolean {
+ return span.attributes[DIRECT_TOOL_CALL_ATTRIBUTE] === DIRECT_TOOL_CALL_VALUE;
+}
+
+export function isAgentActivationSpan(span: SpanRecord): boolean {
+ return (
+ span.name === "agent.turn" ||
+ (span.attributes["gen_ai.operation.name"] === "invoke_agent" &&
+ span.attributes["agent.invocation.role"] !== "caller" &&
+ typeof span.attributes["agent.turn.id"] === "string")
+ );
+}
+
+export function agentTurnIdentity(span: SpanRecord): string | undefined {
+ const sessionId = span.attributes["agent.run.id"];
+ const turnId = span.attributes["agent.turn.id"];
+ return typeof turnId === "string"
+ ? `${typeof sessionId === "string" ? sessionId : ""}\0${turnId}`
+ : undefined;
+}
diff --git a/packages/eve/src/tracing/local/trace-reader.ts b/packages/eve/src/tracing/local/trace-reader.ts
index 16dbc8e99..f8a46d475 100644
--- a/packages/eve/src/tracing/local/trace-reader.ts
+++ b/packages/eve/src/tracing/local/trace-reader.ts
@@ -15,7 +15,7 @@ import {
resolveLocalTraceSchemaDirectory,
resolveLocalTraceSegmentsDirectory,
} from "#tracing/local/trace-span-processor.js";
-import { isAgentActivationSpan } from "#tracing/agent-span-contract.js";
+import { isAgentActivationSpan } from "#tracing/local/inspection.js";
const TRACE_ID_PATTERN = /^[0-9a-f]{32}$/u;
const SPAN_FILE_PATTERN = /^[0-9a-f]{16}\.otlp\.json$/u;
diff --git a/packages/eve/src/tracing/local/trace-summary.ts b/packages/eve/src/tracing/local/trace-summary.ts
index f1beb1dc8..a6ff7fba4 100644
--- a/packages/eve/src/tracing/local/trace-summary.ts
+++ b/packages/eve/src/tracing/local/trace-summary.ts
@@ -1,5 +1,5 @@
import type { LocalTraceSpan } from "#tracing/local/trace-reader.js";
-import { AGENT_USAGE_ATTRIBUTES } from "#tracing/agent-span-contract.js";
+import { USAGE_FIELDS as AGENT_USAGE_ATTRIBUTES } from "#compiled/@eve/tracing/otel.js";
import {
localTraceOperations,
traceModelName,
diff --git a/packages/eve/src/tracing/local/traces.test.ts b/packages/eve/src/tracing/local/traces.test.ts
index 9dae51387..cc267f6e8 100644
--- a/packages/eve/src/tracing/local/traces.test.ts
+++ b/packages/eve/src/tracing/local/traces.test.ts
@@ -7,7 +7,7 @@ import {
createLocalTracesProcessor,
resolveLocalTracesExportPolicy,
} from "#tracing/local/traces.js";
-import { resolveTracePolicy } from "#tracing/sampled-trace.js";
+import { resolveTracePolicy } from "#shared/trace-policy.js";
import { localTraces } from "#public/instrumentation/otel.js";
vi.mock("#tracing/local/trace-span-processor.js", () => ({
@@ -60,14 +60,6 @@ describe("createLocalTracesProcessor", () => {
await expect(spool.releaseConversation("session-one")).resolves.toBe(false);
});
- it("is a span processor, so it composes wherever one goes", () => {
- const spool = createLocalTracesProcessor({ appRoot: "/tmp/eve-local-traces-test" });
- expect(typeof spool.onStart).toBe("function");
- expect(typeof spool.onEnd).toBe("function");
- expect(typeof spool.forceFlush).toBe("function");
- expect(typeof spool.shutdown).toBe("function");
- });
-
it("is inert outside a development worker", async () => {
vi.stubEnv("EVE_DEV_WORKER_APP_ROOT", undefined);
const [processor] = localTraces().spanProcessors;
diff --git a/packages/eve/src/tracing/sampled-trace.ts b/packages/eve/src/tracing/sampled-trace.ts
deleted file mode 100644
index 86a634f6d..000000000
--- a/packages/eve/src/tracing/sampled-trace.ts
+++ /dev/null
@@ -1,9 +0,0 @@
-export { resolveTracePolicy, resolveTracePolicyDecision } from "#shared/trace-policy.js";
-
-// W3C sampled flag; written as a literal so this module stays out of the
-// vendored OTel chunk, which the workflow driver bundle cannot include.
-const TRACE_FLAGS_SAMPLED = 0x01;
-
-export function isSampledTrace(context: { readonly traceFlags: number }): boolean {
- return (context.traceFlags & TRACE_FLAGS_SAMPLED) === TRACE_FLAGS_SAMPLED;
-}
diff --git a/packages/eve/src/tracing/suppress-tracing.ts b/packages/eve/src/tracing/suppress-tracing.ts
deleted file mode 100644
index b21a13abb..000000000
--- a/packages/eve/src/tracing/suppress-tracing.ts
+++ /dev/null
@@ -1,8 +0,0 @@
-import { createContextKey, type Context } from "#compiled/@opentelemetry/api/index.js";
-
-// Standard OpenTelemetry SDK suppression key, kept behind an eve-owned wrapper.
-const SUPPRESS_TRACING_KEY = createContextKey("OpenTelemetry SDK Context Key SUPPRESS_TRACING");
-
-export function suppressTracing(context: Context): Context {
- return context.setValue(SUPPRESS_TRACING_KEY, true);
-}
diff --git a/packages/eve/src/tracing/telemetry-budget.ts b/packages/eve/src/tracing/telemetry-budget.ts
deleted file mode 100644
index 2d858c659..000000000
--- a/packages/eve/src/tracing/telemetry-budget.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-const TELEMETRY_PRINCIPAL_ID_BYTES = 1024;
-
-export function boundedPrincipalId(value: unknown): string | undefined {
- return typeof value === "string" &&
- value.length > 0 &&
- value.length <= TELEMETRY_PRINCIPAL_ID_BYTES &&
- telemetryByteLength(value) <= TELEMETRY_PRINCIPAL_ID_BYTES
- ? value
- : undefined;
-}
-
-const encoder = new TextEncoder();
-const decoder = new TextDecoder();
-const MARKER = "... [truncated]";
-
-export function telemetryByteLength(text: string): number {
- return encoder.encode(text).length;
-}
-
-export function truncateTelemetryText(text: string, maxBytes: number): string {
- const prefix = text.slice(0, maxBytes + 1);
- const bytes = encoder.encode(prefix);
- if (prefix.length === text.length && bytes.length <= maxBytes) return text;
- const marker = MARKER.slice(0, maxBytes);
- let end = Math.max(0, maxBytes - marker.length);
- while (end > 0 && (bytes[end]! & 0xc0) === 0x80) end -= 1;
- return decoder.decode(bytes.subarray(0, end)) + marker;
-}
diff --git a/packages/eve/src/tracing/test-data/remote-local-agent-trace.otlp.json b/packages/eve/src/tracing/test-data/remote-local-agent-trace.otlp.json
new file mode 100644
index 000000000..d8c9a0766
--- /dev/null
+++ b/packages/eve/src/tracing/test-data/remote-local-agent-trace.otlp.json
@@ -0,0 +1,600 @@
+{
+ "resourceSpans": [
+ {
+ "resource": {
+ "attributes": [
+ {
+ "key": "service.name",
+ "value": {
+ "stringValue": "eve-trace-contract"
+ }
+ },
+ {
+ "key": "telemetry.sdk.language",
+ "value": {
+ "stringValue": "nodejs"
+ }
+ },
+ {
+ "key": "telemetry.sdk.name",
+ "value": {
+ "stringValue": "opentelemetry"
+ }
+ },
+ {
+ "key": "telemetry.sdk.version",
+ "value": {
+ "stringValue": "2.6.1"
+ }
+ }
+ ],
+ "droppedAttributesCount": 0
+ },
+ "scopeSpans": [
+ {
+ "scope": {
+ "name": "eve.agent"
+ },
+ "spans": [
+ {
+ "traceId": "b31314b8db6855d6662fc41a0ea318e2",
+ "spanId": "b31314b8db6855d6",
+ "name": "invoke_agent remote",
+ "kind": 1,
+ "startTimeUnixNano": "1700000000000000000",
+ "endTimeUnixNano": "1700000000001000000",
+ "attributes": [
+ {
+ "key": "agent.framework.name",
+ "value": {
+ "stringValue": "eve"
+ }
+ },
+ {
+ "key": "agent.framework.version",
+ "value": {
+ "stringValue": "test"
+ }
+ },
+ {
+ "key": "agent.name",
+ "value": {
+ "stringValue": "remote"
+ }
+ },
+ {
+ "key": "agent.channel.audience",
+ "value": {
+ "stringValue": "public"
+ }
+ },
+ {
+ "key": "agent.principal.current.id",
+ "value": {
+ "stringValue": "current-user"
+ }
+ },
+ {
+ "key": "agent.principal.current.type",
+ "value": {
+ "stringValue": "service"
+ }
+ },
+ {
+ "key": "agent.principal.initiator.id",
+ "value": {
+ "stringValue": "initiator-user"
+ }
+ },
+ {
+ "key": "agent.principal.initiator.type",
+ "value": {
+ "stringValue": "user"
+ }
+ },
+ {
+ "key": "agent.channel.kind",
+ "value": {
+ "stringValue": "http"
+ }
+ },
+ {
+ "key": "agent.session.origin",
+ "value": {
+ "stringValue": "channel"
+ }
+ },
+ {
+ "key": "agent.parent_call.id",
+ "value": {
+ "stringValue": "remote-call"
+ }
+ },
+ {
+ "key": "agent.parent_run.id",
+ "value": {
+ "stringValue": "caller-root"
+ }
+ },
+ {
+ "key": "agent.run.type",
+ "value": {
+ "stringValue": "subagent"
+ }
+ },
+ {
+ "key": "agent.trace.content.input",
+ "value": {
+ "boolValue": true
+ }
+ },
+ {
+ "key": "agent.trace.content.output",
+ "value": {
+ "boolValue": true
+ }
+ },
+ {
+ "key": "agent.turn.id",
+ "value": {
+ "stringValue": "turn_0"
+ }
+ },
+ {
+ "key": "agent.turn.sequence",
+ "value": {
+ "intValue": 0
+ }
+ },
+ {
+ "key": "gen_ai.agent.name",
+ "value": {
+ "stringValue": "remote"
+ }
+ },
+ {
+ "key": "gen_ai.operation.name",
+ "value": {
+ "stringValue": "invoke_agent"
+ }
+ },
+ {
+ "key": "operation.name",
+ "value": {
+ "stringValue": "invoke_agent"
+ }
+ },
+ {
+ "key": "resource.name",
+ "value": {
+ "stringValue": "invoke_agent remote"
+ }
+ },
+ {
+ "key": "agent.run.id",
+ "value": {
+ "stringValue": "remote-session"
+ }
+ },
+ {
+ "key": "agent.trace.schema.version",
+ "value": {
+ "intValue": 4
+ }
+ },
+ {
+ "key": "gen_ai.conversation.id",
+ "value": {
+ "stringValue": "caller-root"
+ }
+ },
+ {
+ "key": "vercel.session_id",
+ "value": {
+ "stringValue": "remote-session"
+ }
+ },
+ {
+ "key": "agent.turn.outcome",
+ "value": {
+ "stringValue": "completed"
+ }
+ }
+ ],
+ "droppedAttributesCount": 0,
+ "events": [
+ {
+ "attributes": [],
+ "name": "turn.started",
+ "timeUnixNano": "1700000000000000000",
+ "droppedAttributesCount": 0
+ },
+ {
+ "attributes": [],
+ "name": "turn.completed",
+ "timeUnixNano": "1700000000000000000",
+ "droppedAttributesCount": 0
+ }
+ ],
+ "droppedEventsCount": 0,
+ "status": {
+ "code": 0
+ },
+ "links": [
+ {
+ "attributes": [
+ {
+ "key": "eve.link.type",
+ "value": {
+ "stringValue": "agent.dispatch"
+ }
+ }
+ ],
+ "spanId": "aaaaaaaaaaaaaaaa",
+ "traceId": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+ "droppedAttributesCount": 0,
+ "flags": 769
+ }
+ ],
+ "droppedLinksCount": 0,
+ "flags": 257
+ },
+ {
+ "traceId": "86d795248ef3bc2325f68a0cc80f27d7",
+ "spanId": "86d795248ef3bc23",
+ "name": "invoke_agent remote",
+ "kind": 1,
+ "startTimeUnixNano": "1700000000000000000",
+ "endTimeUnixNano": "1700000000001000000",
+ "attributes": [
+ {
+ "key": "agent.framework.name",
+ "value": {
+ "stringValue": "eve"
+ }
+ },
+ {
+ "key": "agent.framework.version",
+ "value": {
+ "stringValue": "test"
+ }
+ },
+ {
+ "key": "agent.name",
+ "value": {
+ "stringValue": "remote"
+ }
+ },
+ {
+ "key": "agent.channel.audience",
+ "value": {
+ "stringValue": "public"
+ }
+ },
+ {
+ "key": "agent.principal.current.id",
+ "value": {
+ "stringValue": "current-user"
+ }
+ },
+ {
+ "key": "agent.principal.current.type",
+ "value": {
+ "stringValue": "service"
+ }
+ },
+ {
+ "key": "agent.principal.initiator.id",
+ "value": {
+ "stringValue": "initiator-user"
+ }
+ },
+ {
+ "key": "agent.principal.initiator.type",
+ "value": {
+ "stringValue": "user"
+ }
+ },
+ {
+ "key": "agent.channel.kind",
+ "value": {
+ "stringValue": "http"
+ }
+ },
+ {
+ "key": "agent.session.origin",
+ "value": {
+ "stringValue": "channel"
+ }
+ },
+ {
+ "key": "agent.parent_call.id",
+ "value": {
+ "stringValue": "remote-call"
+ }
+ },
+ {
+ "key": "agent.parent_run.id",
+ "value": {
+ "stringValue": "caller-root"
+ }
+ },
+ {
+ "key": "agent.run.type",
+ "value": {
+ "stringValue": "subagent"
+ }
+ },
+ {
+ "key": "agent.trace.content.input",
+ "value": {
+ "boolValue": true
+ }
+ },
+ {
+ "key": "agent.trace.content.output",
+ "value": {
+ "boolValue": true
+ }
+ },
+ {
+ "key": "agent.turn.id",
+ "value": {
+ "stringValue": "turn_1"
+ }
+ },
+ {
+ "key": "agent.turn.sequence",
+ "value": {
+ "intValue": 1
+ }
+ },
+ {
+ "key": "gen_ai.agent.name",
+ "value": {
+ "stringValue": "remote"
+ }
+ },
+ {
+ "key": "gen_ai.operation.name",
+ "value": {
+ "stringValue": "invoke_agent"
+ }
+ },
+ {
+ "key": "operation.name",
+ "value": {
+ "stringValue": "invoke_agent"
+ }
+ },
+ {
+ "key": "resource.name",
+ "value": {
+ "stringValue": "invoke_agent remote"
+ }
+ },
+ {
+ "key": "agent.run.id",
+ "value": {
+ "stringValue": "remote-session"
+ }
+ },
+ {
+ "key": "agent.trace.schema.version",
+ "value": {
+ "intValue": 4
+ }
+ },
+ {
+ "key": "gen_ai.conversation.id",
+ "value": {
+ "stringValue": "caller-root"
+ }
+ },
+ {
+ "key": "vercel.session_id",
+ "value": {
+ "stringValue": "remote-session"
+ }
+ },
+ {
+ "key": "agent.turn.outcome",
+ "value": {
+ "stringValue": "completed"
+ }
+ }
+ ],
+ "droppedAttributesCount": 0,
+ "events": [
+ {
+ "attributes": [],
+ "name": "turn.started",
+ "timeUnixNano": "1700000000000000000",
+ "droppedAttributesCount": 0
+ },
+ {
+ "attributes": [],
+ "name": "turn.completed",
+ "timeUnixNano": "1700000000000000000",
+ "droppedAttributesCount": 0
+ }
+ ],
+ "droppedEventsCount": 0,
+ "status": {
+ "code": 0
+ },
+ "links": [],
+ "droppedLinksCount": 0,
+ "flags": 257
+ },
+ {
+ "traceId": "b31314b8db6855d6662fc41a0ea318e2",
+ "spanId": "d8f3a7d23b288e65",
+ "parentSpanId": "b31314b8db6855d6",
+ "name": "invoke_agent local",
+ "kind": 1,
+ "startTimeUnixNano": "1700000000000000000",
+ "endTimeUnixNano": "1700000000001000000",
+ "attributes": [
+ {
+ "key": "agent.framework.name",
+ "value": {
+ "stringValue": "eve"
+ }
+ },
+ {
+ "key": "agent.framework.version",
+ "value": {
+ "stringValue": "test"
+ }
+ },
+ {
+ "key": "agent.name",
+ "value": {
+ "stringValue": "local"
+ }
+ },
+ {
+ "key": "agent.channel.audience",
+ "value": {
+ "stringValue": "public"
+ }
+ },
+ {
+ "key": "agent.principal.current.id",
+ "value": {
+ "stringValue": "current-user"
+ }
+ },
+ {
+ "key": "agent.principal.current.type",
+ "value": {
+ "stringValue": "service"
+ }
+ },
+ {
+ "key": "agent.principal.initiator.id",
+ "value": {
+ "stringValue": "initiator-user"
+ }
+ },
+ {
+ "key": "agent.principal.initiator.type",
+ "value": {
+ "stringValue": "user"
+ }
+ },
+ {
+ "key": "agent.parent_call.id",
+ "value": {
+ "stringValue": "local-call"
+ }
+ },
+ {
+ "key": "agent.parent_run.id",
+ "value": {
+ "stringValue": "remote-session"
+ }
+ },
+ {
+ "key": "agent.run.type",
+ "value": {
+ "stringValue": "subagent"
+ }
+ },
+ {
+ "key": "agent.trace.content.input",
+ "value": {
+ "boolValue": true
+ }
+ },
+ {
+ "key": "agent.trace.content.output",
+ "value": {
+ "boolValue": true
+ }
+ },
+ {
+ "key": "agent.turn.id",
+ "value": {
+ "stringValue": "turn_0"
+ }
+ },
+ {
+ "key": "agent.turn.sequence",
+ "value": {
+ "intValue": 0
+ }
+ },
+ {
+ "key": "gen_ai.agent.name",
+ "value": {
+ "stringValue": "local"
+ }
+ },
+ {
+ "key": "gen_ai.operation.name",
+ "value": {
+ "stringValue": "invoke_agent"
+ }
+ },
+ {
+ "key": "operation.name",
+ "value": {
+ "stringValue": "invoke_agent"
+ }
+ },
+ {
+ "key": "resource.name",
+ "value": {
+ "stringValue": "invoke_agent local"
+ }
+ },
+ {
+ "key": "agent.run.id",
+ "value": {
+ "stringValue": "local-session"
+ }
+ },
+ {
+ "key": "agent.trace.schema.version",
+ "value": {
+ "intValue": 4
+ }
+ },
+ {
+ "key": "gen_ai.conversation.id",
+ "value": {
+ "stringValue": "caller-root"
+ }
+ },
+ {
+ "key": "vercel.session_id",
+ "value": {
+ "stringValue": "remote-session"
+ }
+ }
+ ],
+ "droppedAttributesCount": 0,
+ "events": [
+ {
+ "attributes": [],
+ "name": "turn.started",
+ "timeUnixNano": "1700000000000000000",
+ "droppedAttributesCount": 0
+ }
+ ],
+ "droppedEventsCount": 0,
+ "status": {
+ "code": 0
+ },
+ "links": [],
+ "droppedLinksCount": 0,
+ "flags": 257
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}