The Open Framework for Building Agents
refactor(eve): integrate the shared agent tracing engine (#4164)
Chad Hietala76f68542dbfa
diff --git a/.changeset/clean-agent-tracing.md b/.changeset/clean-agent-tracing.mdnew file mode 100644index 000000000..a7a5ab6de--- /dev/null+++ b/.changeset/clean-agent-tracing.md@@ -0,0 +1,5 @@+---+"eve": patch+---++eve's agent spans now come from the shared tracing library, and durable turns recover from one checkpoint per turn. Each tool call is still one `execute_tool` span, a local subagent's first turn still nests under its caller, and trace schema version 4 is unchanged.diff --git a/packages/eve/scripts/build-rolldown.mjs b/packages/eve/scripts/build-rolldown.mjsindex 875492253..bb7725480 100644--- a/packages/eve/scripts/build-rolldown.mjs+++ b/packages/eve/scripts/build-rolldown.mjs@@ -162,7 +162,6 @@ const EXCLUDED_DIRECTORIES = new Set([join("internal", "testing")]);*/const EXTERNAL_PACKAGES = new Set(["@nuxt/kit",- "@opentelemetry/api","@sveltejs/kit","ai","braintrust",@@ -259,6 +258,20 @@ function createVendoredConnectPlugin() {};}+function createVendoredTelemetryPlugin() {+ const packages = new Set([+ "@opentelemetry/api",+ "@opentelemetry/otlp-transformer",+ "@vercel/otel",+ ]);+ return {+ name: "eve:vendored-telemetry",+ resolveId(source) {+ return packages.has(source) ? { id: `#compiled/${source}/index.js`, external: true } : null;+ },+ };+}+async function collectSourceFiles(directory, relativeRoot = "") {const entries = await readdir(directory, { withFileTypes: true });const sourceFiles = [];@@ -315,6 +328,7 @@ await buildWithNitroRolldown({plugins: [createVendoredZodPlugin(),createVendoredConnectPlugin(),+ createVendoredTelemetryPlugin(),createStripUnusedRolldownRuntimeImportPlugin(),createDynamicToolTransformPlugin(),createWorkflowMetadataTransformPlugin(),diff --git a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts b/packages/eve/src/cli/dev/tui/traces/trace-conversation.tsindex e7e5aa0ac..aa62360c8 100644--- a/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts+++ b/packages/eve/src/cli/dev/tui/traces/trace-conversation.ts@@ -10,7 +10,7 @@ import { formatElapsed } from "#cli/format-elapsed.js";import { clipVisible, stripTerminalControls, visibleLength } from "#cli/ui/terminal-text.js";import type { LocalTrace, LocalTraceSpan } from "#tracing/local/trace-reader.js";import { compareLocalTraceSpans, isAgentTurnSpan } from "#tracing/local/trace-reader.js";-import { agentTurnIdentity } from "#tracing/agent-span-contract.js";+import { agentTurnIdentity } from "#tracing/local/inspection.js";import { localTraceSpanCostUsd } from "#tracing/local/trace-summary.js";import { traceStringAttribute } from "#tracing/local/trace-operations.js";diff --git a/packages/eve/src/eve-channel/index.ts b/packages/eve/src/eve-channel/index.tsindex f4997fd74..180abea59 100644--- a/packages/eve/src/eve-channel/index.ts+++ b/packages/eve/src/eve-channel/index.ts@@ -1,6 +1,7 @@import { InvalidToolStubTargetError } from "#tool-stubs/validate-targets.js";import { readStubFailure, readMatchedStubRules } from "#execution/tool-stubs/steps.js";import { getRun } from "#internal/workflow/runtime.js";+import { proxyStreamHeaders } from "#eve-channel/stream-headers.js";import { handleExpiredLegacyAuthorization } from "#execution/legacy-session/authorization.js";import { EVE_ROUTE_PREFIX } from "#protocol/routes.js";import type { SessionAuthContext, SessionParent, SessionTraceContext } from "#channel/types.js";@@ -18,13 +19,7 @@ import {readRemoteAgentStreamHeadersResolver,readRouteSessionCreator,} from "#internal/nitro/routes/channel-route-context.js";-import {- EVE_SESSION_ID_HEADER,- EVE_STREAM_CONTROL_VERSION_QUERY,- EVE_STREAM_FORMAT_HEADER,- EVE_STREAM_TAIL_INDEX_HEADER,- EVE_STREAM_VERSION_HEADER,-} from "#protocol/message.js";+import { EVE_SESSION_ID_HEADER, EVE_STREAM_CONTROL_VERSION_QUERY } from "#protocol/message.js";import { legacyTaskInputRoute } from "#execution/legacy-remote-agent/protocol.js";import {EVE_CALLBACK_ROUTE_PATTERN,@@ -52,7 +47,7 @@ import {readForwardedAudienceBaggage,readForwardedParentSessionBaggage,} from "#protocol/baggage.js";-import { readConversationBaggage } from "#tracing/eve/conversation-context.js";+import { agentDelegation } from "#tracing/eve/delegation.js";import {FAIL_CLOSED_FORWARDED_TRACE_ASSERTION,formatTraceContentCeiling,@@ -336,40 +331,49 @@ export function eveChannel(input: EveChannelInput): EveChannel {let handle: Awaited<ReturnType<typeof createSession>>;try {- handle = await createSession({- audienceAuth: auth,- toolStubs:- body.stubs === undefined- ? undefined- : {- rules: body.stubs,- token: crypto.randomUUID(),+ handle = await agentDelegation.receive(+ {+ headers: req.headers,+ delegated: body.callback !== undefined,+ trusted: forwarded.accepted,+ parent,+ callerTrace: parsedParentTraceContext,+ parentTraceContext,+ },+ ({ conversationId, parentTraceContext: inheritedTrace }) => {+ return createSession({+ audienceAuth: auth,+ toolStubs:+ body.stubs === undefined+ ? undefined+ : {+ rules: body.stubs,+ token: crypto.randomUUID(),+ },+ auth: messageResult.auth,+ capabilities: body.capabilities ?? { requestInput: true },+ callback: body.callback,+ legacyRemoteAgentCaller: body.legacyRemoteAgentCaller,+ continuationToken: operationToken,+ initiatorAuth: forwarded.accepted ? forwarded.initiatorAuth : undefined,+ input: attachClientContext(+ {+ message: body.message,+ context: messageResult.context,+ outputSchema: body.outputSchema,},- auth: messageResult.auth,- capabilities: body.capabilities ?? { requestInput: true },- callback: body.callback,- legacyRemoteAgentCaller: body.legacyRemoteAgentCaller,- continuationToken: operationToken,- initiatorAuth: forwarded.accepted ? forwarded.initiatorAuth : undefined,- input: attachClientContext(- {- message: body.message,- context: messageResult.context,- outputSchema: body.outputSchema,- },- body.context,- ),- conversationId:- body.callback === undefined- ? undefined- : readConversationBaggage(req.headers.get("baggage")),- parent,- parentTraceContext,- // A remote agent's spans form their own session in this- // deployment; its lineage still names the caller's root.- traceRoot: parent === undefined ? undefined : { kind: "own" },- title: messageResult.title,- });+ body.context,+ ),+ conversationId,+ parent,+ parentTraceContext: inheritedTrace,+ // A remote agent's spans form their own session in this+ // deployment; its lineage still names the caller's root.+ traceRoot: parent === undefined ? undefined : { kind: "own" },+ title: messageResult.title,+ });+ },+ );} catch (error) {if (error instanceof InvalidToolStubTargetError) {return Response.json({ error: error.message, ok: false }, { status: 400 });@@ -723,19 +727,7 @@ export function eveChannel(input: EveChannelInput): EveChannel {redirect: "manual",signal: req.signal,});- const responseHeaders = new Headers();- for (const name of [- "cache-control",- "content-type",- "x-accel-buffering",- EVE_SESSION_ID_HEADER,- EVE_STREAM_FORMAT_HEADER,- EVE_STREAM_TAIL_INDEX_HEADER,- EVE_STREAM_VERSION_HEADER,- ]) {- const value = upstream.headers.get(name);- if (value !== null) responseHeaders.set(name, value);- }+ const responseHeaders = proxyStreamHeaders(upstream.headers);return new Response(upstream.body, {headers: responseHeaders,status: upstream.status,diff --git a/packages/eve/src/eve-channel/stream-headers.ts b/packages/eve/src/eve-channel/stream-headers.tsnew file mode 100644index 000000000..a008211a2--- /dev/null+++ b/packages/eve/src/eve-channel/stream-headers.ts@@ -0,0 +1,19 @@+import {+ EVE_SESSION_ID_HEADER,+ EVE_STREAM_FORMAT_HEADER,+ EVE_STREAM_TAIL_INDEX_HEADER,+ EVE_STREAM_VERSION_HEADER,+} from "#protocol/message.js";++export function proxyStreamHeaders(headers: Headers): Headers {+ const allowed = [+ "cache-control",+ "content-type",+ "x-accel-buffering",+ EVE_SESSION_ID_HEADER,+ EVE_STREAM_FORMAT_HEADER,+ EVE_STREAM_TAIL_INDEX_HEADER,+ EVE_STREAM_VERSION_HEADER,+ ];+ return new Headers([...headers].filter(([name]) => allowed.includes(name)));+}diff --git a/packages/eve/src/execution/agent-sessions/remote.test.ts b/packages/eve/src/execution/agent-sessions/remote.test.tsindex 0b06c18d9..d8a882e2e 100644--- a/packages/eve/src/execution/agent-sessions/remote.test.ts+++ b/packages/eve/src/execution/agent-sessions/remote.test.ts@@ -166,6 +166,64 @@ describe("resolveRemoteAgentStreamHeaders", () => {});describe("startRemoteAgentSession", () => {+ it("uses the library transport handoff for a worker dispatch", async () => {+ const { AsyncLocalStorageContextManager } = await import("@opentelemetry/context-async-hooks");+ const { context } = await import("@opentelemetry/api");+ const { agentDelegation } = await import("#tracing/eve/delegation.js");+ const manager = new AsyncLocalStorageContextManager().enable();+ context.setGlobalContextManager(manager);+ const fetchMock = vi+ .fn()+ .mockResolvedValue(+ Response.json(+ { ok: true, protocolVersion: 2, sessionId: "remote-session", status: "accepted" },+ { status: 202 },+ ),+ );+ vi.stubGlobal("fetch", fetchMock);+ try {+ await agentDelegation.resume(+ {+ context: {+ trace: {+ conversationId: "conversation",+ originAudience: "public",+ parentTraceContext: {+ traceId: "a".repeat(32),+ spanId: "b".repeat(16),+ traceFlags: 1,+ decision: { action: "record", recordInputs: false, recordOutputs: false },+ },+ },+ parent: {+ sessionId: "parent",+ rootSessionId: "parent",+ callId: "call",+ turn: { id: "turn", sequence: 0 },+ },+ },+ agentName: "research",+ },+ () =>+ startRemoteAgentSession({+ action: createAction(),+ callbackBaseUrl: "https://caller.example.com",+ remote: createRemoteAgent(),+ session: { continuationToken: "eve:parent-token" },+ }),+ );+ const options = fetchMock.mock.calls[0]?.[1];+ expect(JSON.parse(new Headers(options.headers).get("x-agent-tracing")!)).toMatchObject({+ conversationId: "conversation",+ parentRunId: "parent",+ parentCallId: "call",+ });+ expect(options.redirect).toBe("error");+ } finally {+ context.disable();+ manager.disable();+ }+ });afterEach(() => {vi.unstubAllGlobals();vi.unstubAllEnvs();diff --git a/packages/eve/src/execution/agent-sessions/remote.ts b/packages/eve/src/execution/agent-sessions/remote.tsindex 3411a3fc1..51d2811e9 100644--- a/packages/eve/src/execution/agent-sessions/remote.ts+++ b/packages/eve/src/execution/agent-sessions/remote.ts@@ -44,6 +44,7 @@ import {} from "#protocol/baggage.js";import { decisionToTraceContentCeiling } from "#shared/forwarded-trace-policy.js";import { writeConversationBaggage } from "#tracing/eve/conversation-context.js";+import { agentDelegation } from "#tracing/eve/delegation.js";const CreateSessionResponseSchema = z.object({ok: z.literal(true),@@ -151,7 +152,7 @@ export async function startRemoteAgentSession(input: {"baggage",writeForwardedParentSessionBaggage(conversationBaggage, input.parent?.lineage),);- const response = await fetch(createRemoteAgentSessionUrl(input.remote), {+ const response = await agentDelegation.fetch(createRemoteAgentSessionUrl(input.remote), {body: JSON.stringify(requestBody),headers: {"content-type": "application/json",diff --git a/packages/eve/src/execution/agent-sessions/steps.ts b/packages/eve/src/execution/agent-sessions/steps.tsindex f4e293dda..34db0b8e7 100644--- a/packages/eve/src/execution/agent-sessions/steps.ts+++ b/packages/eve/src/execution/agent-sessions/steps.ts@@ -39,6 +39,7 @@ import {import { buildSubagentRunInput } from "#subagents/tool.js";import { resolveConversationId } from "#shared/conversation-identity.js";import { stubToolPath } from "#tool-stubs/target.js";+import { agentDelegation } from "#tracing/eve/delegation.js";const log = createLogger("execution.agent-sessions");@@ -115,10 +116,11 @@ export async function openAgentSessionStep(isRootSession: context.parent.rootSessionId === context.parent.sessionId,});const start = { auth: input.auth, bundle, context, key: input.key, replyTo: input.replyTo };- if (target.kind === "remote") {- return await startRemoteSession({ ...start, target });- }- return await startLocalSession({ ...start, target });+ const execute = () =>+ target.kind === "remote"+ ? startRemoteSession({ ...start, target })+ : startLocalSession({ ...start, target });+ return await agentDelegation.resume({ context, agentName: action.name }, execute);}/**diff --git a/packages/eve/src/execution/cancelled-model-call-batch.ts b/packages/eve/src/execution/cancelled-model-call-batch.tsindex a3bb98960..2a099935d 100644--- a/packages/eve/src/execution/cancelled-model-call-batch.ts+++ b/packages/eve/src/execution/cancelled-model-call-batch.ts@@ -11,7 +11,7 @@ import type { HarnessSession, StepInput, StepResult } from "#harness/types.js";import { appliedSession, saveSessionProjection } from "#harness/session-machine/current.js";import { takeDeferredMessage } from "#harness/hitl/index.js";import { preserveSerializedInstrumentationState } from "#instrumentation/state.js";-import { preserveSerializedAgentTraceState } from "#tracing/agent-trace-context-store.js";+import { preserveSerializedAgentTraceState } from "#tracing/eve/agent-trace-context-store.js";const log = createLogger("execution.cancelled-turn");diff --git a/packages/eve/src/execution/eve-workflow-attributes.ts b/packages/eve/src/execution/eve-workflow-attributes.tsindex b29622e02..f00a82c3a 100644--- a/packages/eve/src/execution/eve-workflow-attributes.ts+++ b/packages/eve/src/execution/eve-workflow-attributes.ts@@ -53,7 +53,7 @@ import {ConversationContextKey,normalizeConversationContext,} from "#shared/conversation-context.js";-import { isSampledTrace } from "#tracing/sampled-trace.js";+import { isSampledTrace } from "#shared/trace-policy.js";import { resolveForwardedTraceSeed } from "#shared/forwarded-trace-policy.js";/** eve version recorded on every run eve starts; handoff successors and legacy importers re-record it at boot. */diff --git a/packages/eve/src/execution/node-step.test.ts b/packages/eve/src/execution/node-step.test.tsindex 4336b5d77..eea51697e 100644--- a/packages/eve/src/execution/node-step.test.ts+++ b/packages/eve/src/execution/node-step.test.ts@@ -1,3 +1,4 @@+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";import { ToolLoopAgent } from "ai";import { afterEach, describe, expect, it, vi } from "vitest";import type { Runtime } from "#channel/types.js";@@ -330,6 +331,7 @@ describe("createExecutionNodeStep", () => {setupMockAgentForToolExecution("regular-tool", { question: "Run the tool." });const forceFlush = vi.fn(async () => undefined);const runtime: InstrumentationRuntime = {+ idGenerator: new AgentSpanIdGenerator(),forceFlush,hooks: createInstrumentationHooks([]),otelSettings: undefined,diff --git a/packages/eve/src/execution/session/turn-step.test.ts b/packages/eve/src/execution/session/turn-step.test.tsindex 666f1329f..d76c79c2d 100644--- a/packages/eve/src/execution/session/turn-step.test.ts+++ b/packages/eve/src/execution/session/turn-step.test.ts@@ -1,3 +1,4 @@+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";import { createTestSessionState } from "#internal/testing/session-state.js";import { jsonSchema, type ModelMessage } from "ai";import { MockLanguageModelV4 } from "ai/test";@@ -2229,6 +2230,7 @@ describe("turnStep", () => {});instrumentationRuntimeOverride.current = {forceFlush: async () => undefined,+ idGenerator: new AgentSpanIdGenerator(),hooks: {capturesContent: false,publish: async (event) => {diff --git a/packages/eve/src/execution/session/turn-step.ts b/packages/eve/src/execution/session/turn-step.tsindex 111ade392..8d5e32b9d 100644--- a/packages/eve/src/execution/session/turn-step.ts+++ b/packages/eve/src/execution/session/turn-step.ts@@ -32,7 +32,7 @@ import { BundleKey, ChannelKey } from "#runtime/sessions/runtime-context-keys.jsimport { deserializeContext, serializeContext } from "#context/serialize.js";import { bindSessionInstrumentation } from "#instrumentation/runtime.js";import { RuntimeActionSettlementTimesKey } from "#harness/runtime-action-settlement-state.js";-import * as agentTraceState from "#tracing/agent-trace-context-store.js";+import * as agentTraceState from "#tracing/eve/agent-trace-context-store.js";import { matchAuthorizationCallbacks } from "#execution/authorization-callback-match.js";import { isTurnCancellation, throwIfTurnAborted } from "#harness/turn-cancellation.js";import { setChannelContext } from "#execution/channel-context.js";diff --git a/packages/eve/src/execution/workflow-runtime.test.ts b/packages/eve/src/execution/workflow-runtime.test.tsindex 0d50b6d30..b1a03e2e5 100644--- a/packages/eve/src/execution/workflow-runtime.test.ts+++ b/packages/eve/src/execution/workflow-runtime.test.ts@@ -27,7 +27,7 @@ import {sessionHandoffMarkerToken,} from "#execution/session-inbox/address.js";import { registerInstrumentationRuntime } from "#instrumentation/runtime.js";-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";import type { RuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js";import { getCompiledRuntimeAgentBundle } from "#runtime/sessions/compiled-agent-cache.js";import { markAgentTraceContext } from "#tracing/eve/agent-trace-context.js";diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex 740899ce4..43fdc92eb 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -1,3 +1,4 @@+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";import { context as otelContext, trace } from "#compiled/@opentelemetry/api/index.js";import {type FilePart,@@ -195,6 +196,7 @@ function declareTelemetry(? undefined: {forceFlush: async () => undefined,+ idGenerator: new AgentSpanIdGenerator(),hooks: createInstrumentationHooks([]),otelSettings: {...config,@@ -226,6 +228,7 @@ function bindHookInstrumentation(return bindInstrumentationRuntime({forceFlush: async () => undefined,+ idGenerator: new AgentSpanIdGenerator(),hooks,otelSettings: useDeclaredRuntime ? declaredRuntime?.otelSettings : undefined,runtimeContextResolvers: useDeclaredRuntime@@ -10176,6 +10179,7 @@ describe("createToolLoopHarness", () => {};const authoredSpan = trace.wrapSpanContext(authoredTrace);const getTracerSpy = vi.spyOn(trace, "getTracer").mockReturnValue({+ startActiveSpan: vi.fn(),startSpan: vi.fn(() => authoredSpan),} as ReturnType<typeof trace.getTracer>);setupMockAgent({diff --git a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.tsindex 80c8006f9..9c93920f0 100644--- a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts+++ b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts@@ -3,14 +3,12 @@ import type { Telemetry } from "ai";import type {InstrumentationAttemptScope,InstrumentationStepAttemptStartedEvent,- InstrumentationContentPart,InstrumentationContextRunner,InstrumentationHooks,InstrumentationModelCallCompletedEvent,InstrumentationModelCallStartedEvent,InstrumentationOperationRef,InstrumentationToolCallStartedEvent,- InstrumentationUsage,} from "#instrumentation/lifecycle.js";import {attemptIdempotencyKey,@@ -18,6 +16,7 @@ import {toolCallIdempotencyKey,} from "#instrumentation/lifecycle.js";import { structuralProviderMetadata } from "#instrumentation/content.js";+import { modelContent, modelUsage } from "#compiled/@eve/tracing/ai-sdk.js";type TelemetryEvent<TKey extends keyof Telemetry> = Parameters<NonNullable<Telemetry[TKey]>>[0];@@ -224,78 +223,19 @@ function toModelCallCompleted(source: TelemetryEvent<"onLanguageModelCallEnd">,): InstrumentationModelCallCompletedEvent {return Object.freeze({- content: state.capturesOutputs ? toContentParts(source.content) : undefined,+ content: state.capturesOutputs+ ? Object.freeze(modelContent(source.content).map((part) => Object.freeze(part)))+ : undefined,finishReason: source.finishReason,idempotencyKey,responseModelId: source.modelId,responseId: source.responseId,scope: state.scope,type: "model.call.completed",- usage: toUsage(source.usage),- });-}--function toUsage(usage: TelemetryEvent<"onLanguageModelCallEnd">["usage"]): InstrumentationUsage {- return Object.freeze({- inputTokenDetails: Object.freeze({- cacheReadTokens: usage.inputTokenDetails?.cacheReadTokens,- cacheWriteTokens: usage.inputTokenDetails?.cacheWriteTokens,- }),- inputTokens: usage.inputTokens,- outputTokens: usage.outputTokens,+ usage: Object.freeze(modelUsage(source.usage)),});}-/** Drops kinds eve does not record; see {@link InstrumentationContentPart}. */-function toContentParts(- content: TelemetryEvent<"onLanguageModelCallEnd">["content"],-): readonly InstrumentationContentPart[] {- const parts: InstrumentationContentPart[] = [];- for (const part of content) {- switch (part.type) {- case "text":- case "reasoning":- parts.push(Object.freeze({ text: part.text, type: part.type }));- break;- case "tool-call":- parts.push(- Object.freeze({- callId: part.toolCallId,- input: part.input,- toolName: part.toolName,- type: "tool-call",- }),- );- break;- case "tool-result":- parts.push(- Object.freeze({- callId: part.toolCallId,- input: part.input,- output: part.output,- toolName: part.toolName,- type: "tool-result",- }),- );- break;- case "tool-error":- parts.push(- Object.freeze({- callId: part.toolCallId,- error: part.error,- input: part.input,- toolName: part.toolName,- type: "tool-error",- }),- );- break;- default:- break;- }- }- return Object.freeze(parts);-}-function toToolCallStarted(state: AttemptState,idempotencyKey: string,diff --git a/packages/eve/src/instrumentation/channel-delivery.test.ts b/packages/eve/src/instrumentation/channel-delivery.test.tsindex e4ae24eb5..77514f8d2 100644--- a/packages/eve/src/instrumentation/channel-delivery.test.ts+++ b/packages/eve/src/instrumentation/channel-delivery.test.ts@@ -1,3 +1,4 @@+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";import { describe, expect, it, vi } from "vitest";import { ContextContainer, contextStorage } from "#context/container.js";@@ -34,6 +35,7 @@ function bindHooks(return bindInstrumentationRuntime({forceFlush: async () => undefined,+ idGenerator: new AgentSpanIdGenerator(),hooks,otelSettings: undefined,runInContext: (_operation, execute) => execute(),diff --git a/packages/eve/src/instrumentation/memory.ts b/packages/eve/src/instrumentation/memory.tsindex e1f56883f..f6aed1333 100644--- a/packages/eve/src/instrumentation/memory.ts+++ b/packages/eve/src/instrumentation/memory.ts@@ -59,10 +59,7 @@ export type InstrumentationMemoryOperationTerminalEvent =| InstrumentationMemoryOperationCompletedEvent| InstrumentationMemoryOperationFailedEvent;-export interface InstrumentationMemoryExecutionOperation {- readonly idempotencyKey: string;- readonly sessionId: string;- readonly turnId?: string;+export interface InstrumentationMemoryExecutionOperation extends InstrumentationMemoryOperationEvent {readonly type: "memory.operation";}@@ -98,9 +95,7 @@ export function createMemoryInstrumentation(input: {try {const result = await input.runInContext({- idempotencyKey: operation.idempotencyKey,- sessionId: input.sessionId,- turnId: operation.turnId,+ ...event,type: "memory.operation",},execute,diff --git a/packages/eve/src/instrumentation/principal-summary.ts b/packages/eve/src/instrumentation/principal-summary.tsindex 6cba63e4e..e04dd2cc6 100644--- a/packages/eve/src/instrumentation/principal-summary.ts+++ b/packages/eve/src/instrumentation/principal-summary.ts@@ -11,7 +11,7 @@ import {} from "#shared/instrumentation-content.js";import type { InstrumentationDecision } from "#shared/instrumentation-decision.js";import { resolveTracePolicyDecision } from "#shared/trace-policy.js";-import { boundedPrincipalId } from "#tracing/telemetry-budget.js";+import { boundedPrincipalId } from "#compiled/@eve/tracing/otel.js";export function summarizeInstrumentationPrincipal(principal: SessionAuthContext | null | undefined,diff --git a/packages/eve/src/instrumentation/providers-otel-policy.test.ts b/packages/eve/src/instrumentation/providers-otel-policy.test.tsindex aff40d504..1f934121e 100644--- a/packages/eve/src/instrumentation/providers-otel-policy.test.ts+++ b/packages/eve/src/instrumentation/providers-otel-policy.test.ts@@ -8,20 +8,16 @@ import {import { defineInstrumentation } from "#public/instrumentation/index.js";import { otel, type TracePolicyDecision } from "#public/instrumentation/otel.js";+const REGISTRY_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers");vi.mock("#tracing/eve/otel-registration.js", () => ({- registerOtelPipeline: () => ({- forceFlush: async () => undefined,- idGenerator: {- deriveSpanId: () => "2".repeat(16),- generateTraceId: () => "1".repeat(32),- withSpanId: (_spanId: string, run: () => unknown) => run(),- withTraceId: (_traceId: string, run: () => unknown) => run(),- },- shutdown: async () => undefined,+ registerOtel: () => ({+ provider: { getTracer: () => ({}) },+ idGenerator: { deriveSpanId: () => "2".repeat(16), generateTraceId: () => "1".repeat(32) },+ samplesTrace: () => true,+ forceFlush: async () => {},+ shutdown: async () => {},}),}));--const REGISTRY_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers");const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime");const traceContext = (audience: "public" | "private" | "unknown") => ({diff --git a/packages/eve/src/instrumentation/runtime.test.ts b/packages/eve/src/instrumentation/runtime.test.tsindex 795ca3b01..d28134475 100644--- a/packages/eve/src/instrumentation/runtime.test.ts+++ b/packages/eve/src/instrumentation/runtime.test.ts@@ -26,8 +26,8 @@ import {type InstrumentationRuntime,type InstrumentationStepScope,} from "#instrumentation/runtime.js";-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";-import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";+import { ContextAgentTraceStateStore } from "#tracing/eve/agent-trace-context-store.js";import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";import { readForwardedAudienceBaggage, writeForwardedAudienceBaggage } from "#protocol/baggage.js";import { ConversationContextKey } from "#shared/conversation-context.js";@@ -45,6 +45,7 @@ function createRuntime(): InstrumentationRuntime {return {forceFlush: async () => undefined,+ idGenerator: new AgentSpanIdGenerator(),hooks,otelSettings: {recordInputs: true,@@ -354,6 +355,7 @@ describe("initializeSessionInstrumentation", () => {});it("does not widen an unsampled parent with a record decision", () => {+ registerInstrumentationRuntime(createRuntime({ capturesContent: true, publish: vi.fn() }));const ctx = createContext("public");ctx.set(ParentTraceContextKey, {decision: { action: "record", recordInputs: true, recordOutputs: true },@@ -999,7 +1001,7 @@ describe("bindSessionInstrumentation", () => {);const ctx = createContext();contextStorage.run(ctx, () => {- new ContextAgentTraceStateStore().setSession("session-1", {+ new ContextAgentTraceStateStore().set("session", "session-1", {context: { spanId: "1".repeat(16), traceFlags: 1, traceId: "2".repeat(32) },decision: { action: "record", recordInputs: false, recordOutputs: true },rootSessionId: "session-1",diff --git a/packages/eve/src/instrumentation/runtime.ts b/packages/eve/src/instrumentation/runtime.tsindex aa337665c..03b9939c9 100644--- a/packages/eve/src/instrumentation/runtime.ts+++ b/packages/eve/src/instrumentation/runtime.ts@@ -43,7 +43,7 @@ import {} from "#instrumentation/prepare-trace-context.js";import type { RuntimeTraceContext } from "#protocol/message.js";import type { OtelHarnessSettings, RuntimeContextResolver } from "#tracing/eve/otel-declaration.js";-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";import { contextStorage, type ContextContainer } from "#context/container.js";import {createMemoryInstrumentation,@@ -51,15 +51,15 @@ import {} from "#instrumentation/memory.js";import { ConversationIdKey, ParentSessionKey } from "#context/keys.js";import type { ConversationContext } from "#shared/conversation-context.js";-import { withErrorContent } from "#tracing/error-content-context.js";-import type { AgentSamplingOperation } from "#tracing/agent-span-contract.js";+import { withCapture } from "#compiled/@eve/tracing/otel.js";+import type { SamplingOperation as AgentSamplingOperation } from "#tracing/eve/otel-registration.js";import {isSampledTrace,resolveTracePolicy,resolveTracePolicyDecision,-} from "#tracing/sampled-trace.js";+} from "#shared/trace-policy.js";import { readInstrumentationSessionContext } from "#instrumentation/session-context.js";-import { readSessionTraceDecision } from "#tracing/agent-trace-context-store.js";+import { readSessionTraceDecision } from "#tracing/eve/agent-trace-context-store.js";import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";import { applyLiveDeliveryAudienceCeiling } from "#shared/forwarded-trace-policy.js";import { readConversationId } from "#shared/conversation-identity.js";@@ -498,7 +498,11 @@ export function bindInstrumentationRuntime(});try {return await otelContext.with(- withErrorContent(parentContext ?? otelContext.active(), content.recordOutputs),+ withCapture(parentContext ?? otelContext.active(), {+ emit: true,+ recordInputs: content.recordInputs,+ recordOutputs: content.recordOutputs,+ }),run,);} finally {diff --git a/packages/eve/src/instrumentation/session-init.test.ts b/packages/eve/src/instrumentation/session-init.test.tsindex b4a51c89c..1c378350c 100644--- a/packages/eve/src/instrumentation/session-init.test.ts+++ b/packages/eve/src/instrumentation/session-init.test.ts@@ -10,7 +10,7 @@ import {import { initializeSessionInstrumentation } from "#instrumentation/session-init.js";import { registerInstrumentationRuntime } from "#instrumentation/runtime-global.js";import type { InstrumentationRuntime } from "#instrumentation/runtime.js";-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";afterEach(() => {diff --git a/packages/eve/src/instrumentation/session-init.ts b/packages/eve/src/instrumentation/session-init.tsindex 6ec447ba7..ce2500a2f 100644--- a/packages/eve/src/instrumentation/session-init.ts+++ b/packages/eve/src/instrumentation/session-init.ts@@ -29,8 +29,8 @@ import {isSampledTrace,resolveTracePolicy,resolveTracePolicyDecision,-} from "#tracing/sampled-trace.js";-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";+} from "#shared/trace-policy.js";+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";import { getInstrumentationRuntime } from "#instrumentation/runtime-global.js";import type { InstrumentationRuntime } from "#instrumentation/runtime.js";diff --git a/packages/eve/src/internal/logging.test.ts b/packages/eve/src/internal/logging.test.tsindex d6298b308..e0d373f2f 100644--- a/packages/eve/src/internal/logging.test.ts+++ b/packages/eve/src/internal/logging.test.ts@@ -6,7 +6,7 @@ import {trace,type Span,} from "#compiled/@opentelemetry/api/index.js";-import { withErrorContent } from "#tracing/error-content-context.js";+import { withErrorContent } from "#compiled/@eve/tracing/otel.js";import {createErrorId,@@ -36,6 +36,11 @@ describe("span error content policy", () => {function mockSpan() {return {+ setAttributes: vi.fn(),+ addLink: vi.fn(),+ addLinks: vi.fn(),+ updateName: vi.fn(),+ isRecording: vi.fn(() => true),addEvent: vi.fn(),end: vi.fn(),recordException: vi.fn(),diff --git a/packages/eve/src/internal/logging.ts b/packages/eve/src/internal/logging.tsindex 2f8dc4fd9..1ef855487 100644--- a/packages/eve/src/internal/logging.ts+++ b/packages/eve/src/internal/logging.ts@@ -10,8 +10,8 @@ import { inspect } from "node:util";import { isNonEmptyString, isObject } from "#shared/guards.js";import type { JsonObject, JsonValue } from "#shared/json.js";-import { capturesErrorContent } from "#tracing/error-content-context.js";-import { truncateTelemetryText } from "#tracing/telemetry-budget.js";+import { currentCapture } from "#compiled/@eve/tracing/index.js";+import { truncateTelemetryText } from "#compiled/@eve/tracing/otel.js";const MAX_INSPECT_STRING_LENGTH = 8 * 1024;const MAX_DETAIL_BYTES = 16 * 1024;@@ -230,7 +230,7 @@ function truncateForDisplay(value: string, maxChars = 160): string {* non-active span should be annotated.*/export function recordErrorOnSpan(span: Span, error: unknown): void {- if (!capturesErrorContent()) {+ if (currentCapture()?.recordOutputs === false) {span.setStatus({ code: SpanStatusCode.ERROR });return;}@@ -365,7 +365,7 @@ function recordOnActiveSpan(message: string, fields?: LogFields): void {}const error = fields?.error;- if (!capturesErrorContent()) {+ if (currentCapture()?.recordOutputs === false) {if (error instanceof Error || isFormattedError(error)) {span.setStatus({ code: SpanStatusCode.ERROR });}diff --git a/packages/eve/src/internal/nitro/routes/channel-dispatch.test.ts b/packages/eve/src/internal/nitro/routes/channel-dispatch.test.tsindex 2d55abd30..b8d783465 100644--- a/packages/eve/src/internal/nitro/routes/channel-dispatch.test.ts+++ b/packages/eve/src/internal/nitro/routes/channel-dispatch.test.ts@@ -1,3 +1,4 @@+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";import type { H3Event } from "nitro";import {context as apiContext,@@ -94,6 +95,7 @@ function createDeferred<T>() {function registerChannelRequestTracing(enabled: boolean): void {registerInstrumentationRuntime({forceFlush: async () => undefined,+ idGenerator: new AgentSpanIdGenerator(),hooks: createInstrumentationHooks([]),otelSettings: {functionId: undefined,diff --git a/packages/eve/src/internal/nitro/routes/channel-dispatch.ts b/packages/eve/src/internal/nitro/routes/channel-dispatch.tsindex 959040936..cef93717d 100644--- a/packages/eve/src/internal/nitro/routes/channel-dispatch.ts+++ b/packages/eve/src/internal/nitro/routes/channel-dispatch.ts@@ -65,7 +65,7 @@ export async function dispatchChannelRequest(// Correlation does not require an eve-owned request span. Preserve any// active platform request or function span before route resolution.const requestTraceContext =- span?.spanContext() ?? otelTrace.getSpan(otelContext.active())?.spanContext();+ span?.reference ?? otelTrace.getSpan(otelContext.active())?.spanContext();const bundle = await resolveNitroChannelRuntimeBundle(config);const matchedChannel = bundle.channels.find(@@ -85,11 +85,11 @@ export async function dispatchChannelRequest(// (`channel:<name>`) over the raw adapter kind — behaviorless authored// channels keep adapter kind `"http"`, so the adapter alone would report// `"http"` where the rest of the trace reports `channel:<name>`.- span?.setAttribute("eve.channel.name", matchedChannel.name);+ span?.channel({ channelName: matchedChannel.name });const channelKind =getChannelInstrumentationKind(matchedChannel.definition) ?? matchedChannel.adapter?.kind;if (channelKind !== undefined) {- span?.setAttribute("eve.channel.kind", channelKind);+ span?.channel({ channelKind });}const routeArgs = await buildRouteArgs(diff --git a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.tsindex 12bd57bc3..6a2909d88 100644--- a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts+++ b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts@@ -1,102 +1,71 @@import {- type Attributes,context,propagation,- type Span,+ trace,SpanKind,SpanStatusCode,type TextMapGetter,- trace,} from "#compiled/@opentelemetry/api/index.js";import { getInstrumentationRuntime } from "#instrumentation/runtime.js";+import { withErrorContent } from "#compiled/@eve/tracing/otel.js";import { markAgentTraceContext } from "#tracing/eve/agent-trace-context.js";-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";-import { AGENT_SPAN_NAMES } from "#tracing/agent-span-contract.js";-import { withErrorContent } from "#tracing/error-content-context.js";-/**- * Stable tracer name for every inbound eve channel HTTP request. Kept- * low-cardinality and independent of the agent so dashboards can group- * request spans across deployments.- */-const TRACER_NAME = "eve.channel";--/**- * Reads OTel context from an inbound request's `Headers`. Header lookups- * are case-insensitive, so this is a thin adapter over `Headers.get`.- */+export interface ChannelRequestTrace {+ readonly reference: import("@opentelemetry/api").SpanContext;+ channel(input: { channelName?: string; channelKind?: string }): void;+}const headersGetter: TextMapGetter<Headers> = {- get(carrier, key) {- return carrier.get(key) ?? undefined;- },- keys(carrier) {- return [...carrier.keys()];- },+ get: (headers, key) => headers.get(key) ?? undefined,+ keys: (headers) => [...headers.keys()],};-/** Inputs for {@link traceChannelRequest}. */-interface TraceChannelRequestInput {- readonly request: Request;- readonly routeKey: string;-}--/**- * Wraps one inbound channel HTTP request in an OTel `SERVER` span.- *- * The span is named for the low-cardinality registered `routeKey`- * (`"POST /eve/v1/session/:sessionId"`), never the concrete URL, while the- * `http.route` attribute carries only the path template and- * `http.request.method` carries the method. It starts from context extracted- * off the incoming request headers so an upstream `traceparent` becomes its- * parent and nested channel operations become its descendants.- *- * The handler runs inside the span's active context. When it returns, the- * response status is recorded and a `>= 500` status marks the span as an- * error. A thrown handler marks failure status and is rethrown; request- * spans never record exception content. The span always ends in `finally`, without- * waiting for `event.waitUntil()` work or streamed response bodies.- *- * Emitting these spans is opt-in: unless authored instrumentation enables it- * via `traceChannelRequests: true`, the handler runs with no span (`undefined`)- * and performs no header extraction. The channel dispatcher can still use an- * already-active platform span for the activation's cross-trace link.- *- * This is observability-only: it never changes the response and performs no- * synchronous span export in the request path, adding only minimal in-process- * tracing overhead.- */+/** The request lifetime ends at handler return, not response-body consumption. */export async function traceChannelRequest<T extends Response>(- input: TraceChannelRequestInput,- handler: (span: Span | undefined) => Promise<T>,+ input: { readonly request: Request; readonly routeKey: string },+ handler: (operation: ChannelRequestTrace | undefined) => Promise<T>,): Promise<T> {- if (getInstrumentationRuntime()?.otelSettings?.traceChannelRequests !== true) {- return await handler(undefined);- }-+ if (getInstrumentationRuntime()?.otelSettings?.traceChannelRequests !== true)+ return handler(undefined);const { request, routeKey } = input;- const parentContext = propagation.extract(context.active(), request.headers, headersGetter);- const spanName = AGENT_SPAN_NAMES.channelRequest;- const span = trace.getTracer(TRACER_NAME).startSpan(- spanName,+ const parent = propagation.extract(context.active(), request.headers, headersGetter);+ const separator = routeKey.indexOf(" ");+ let url: URL | undefined;+ try {+ url = new URL(request.url);+ } catch {}+ const span = trace.getTracer("eve.channel").startSpan(+ "agent.channel.request",{+ kind: SpanKind.SERVER,attributes: {- ...baseAttributes(request, routeKey),- ...(spanName === "agent.channel.request" ? agentSpanNamingAttributes(spanName) : undefined),+ "operation.name": "agent.channel.request",+ "resource.name": "agent.channel.request",+ "http.request.method": request.method,+ "http.route": separator === -1 ? routeKey : routeKey.slice(separator + 1),+ "url.scheme": url?.protocol.replace(/:$/, ""),+ "server.address": url?.hostname,},- kind: SpanKind.SERVER,},- parentContext,+ parent,);- const activeContext = markAgentTraceContext(- withErrorContent(trace.setSpan(parentContext, span), false),- );-+ const operation: ChannelRequestTrace = {+ reference: span.spanContext(),+ channel(input) {+ if (input.channelName !== undefined)+ span.setAttribute("agent.channel.name", input.channelName);+ if (input.channelName !== undefined) span.setAttribute("eve.channel.name", input.channelName);+ if (input.channelKind !== undefined)+ span.setAttribute("agent.channel.kind", input.channelKind);+ if (input.channelKind !== undefined) span.setAttribute("eve.channel.kind", input.channelKind);+ },+ };try {- const response = await context.with(activeContext, () => handler(span));+ const response = await context.with(+ markAgentTraceContext(withErrorContent(trace.setSpan(parent, span), false)),+ () => handler(operation),+ );span.setAttribute("http.response.status_code", response.status);- if (response.status >= 500) {- span.setStatus({ code: SpanStatusCode.ERROR });- }+ if (response.status >= 500) span.setStatus({ code: SpanStatusCode.ERROR });return response;} catch (error) {span.setStatus({ code: SpanStatusCode.ERROR });@@ -105,43 +74,3 @@ export async function traceChannelRequest<T extends Response>(span.end();}}--/**- * Standard, non-sensitive request attributes known before the channel is- * resolved. `eve.channel.name` / `eve.channel.kind` are attached later by- * the caller once the matching channel is known. Never records session- * ids, tokens, auth or cookie headers, bodies, or query parameters.- */-function baseAttributes(request: Request, routeKey: string): Attributes {- const attributes: Attributes = {- "http.request.method": request.method,- "http.route": routeTemplate(routeKey),- };-- const url = parseRequestUrl(request.url);- if (url !== undefined) {- attributes["url.scheme"] = url.protocol.replace(/:$/, "");- attributes["server.address"] = url.hostname;- }-- return attributes;-}--/**- * Extracts the path template from a `"METHOD /path/:param"` route key. The- * OTel HTTP convention keeps `http.route` free of the method so route- * facets align with other server spans; the method lives in- * `http.request.method` and the full key remains the span name.- */-function routeTemplate(routeKey: string): string {- const separator = routeKey.indexOf(" ");- return separator === -1 ? routeKey : routeKey.slice(separator + 1);-}--function parseRequestUrl(rawUrl: string): URL | undefined {- try {- return new URL(rawUrl);- } catch {- return undefined;- }-}diff --git a/packages/eve/src/internal/testing/in-memory-agent-trace-state-store.ts b/packages/eve/src/internal/testing/in-memory-agent-trace-state-store.tsindex a259acc9f..a5651baf0 100644--- a/packages/eve/src/internal/testing/in-memory-agent-trace-state-store.ts+++ b/packages/eve/src/internal/testing/in-memory-agent-trace-state-store.ts@@ -1,98 +1,30 @@-import type {- AgentActionTraceState,- AgentSessionTraceState,- AgentTraceStateStore,- AgentTurnTraceState,-} from "#tracing/agent-trace-state.js";+import type { AgentTraceStateStore, AgentTraceValues } from "#tracing/eve/agent-trace-state.js";export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {- readonly #actionAnchors = new Map<string, AgentActionTraceState>();- readonly #actions = new Map<string, AgentActionTraceState>();- readonly #sessions = new Map<string, AgentSessionTraceState>();- readonly #turns = new Map<string, AgentTurnTraceState>();-- deleteAction(idempotencyKey: string): void {- this.#actions.delete(idempotencyKey);- }-- deleteActionAnchors(sessionId: string): void {- for (const [key, state] of this.#actionAnchors) {- if (state.sessionId === sessionId) this.#actionAnchors.delete(key);- }- }-- deleteActions(sessionId: string, turnId?: string): void {- for (const [key, state] of this.#actions) {- if (state.sessionId === sessionId && (turnId === undefined || state.turnId === turnId)) {- this.#actions.delete(key);- }- }- }-- deleteSession(sessionId: string): void {- this.#sessions.delete(sessionId);+ readonly #entries = new Map<string, AgentTraceValues[keyof AgentTraceValues]>();+ get<K extends keyof AgentTraceValues>(kind: K, key: string): AgentTraceValues[K] | undefined {+ return this.#entries.get(JSON.stringify([kind, key])) as AgentTraceValues[K] | undefined;}-- deleteTurn(sessionId: string, turnId: string): void {- this.#turns.delete(turnKey(sessionId, turnId));+ set<K extends keyof AgentTraceValues>(kind: K, key: string, value: AgentTraceValues[K]): void {+ this.#entries.set(JSON.stringify([kind, key]), value);}-- findAction(sessionId: string, callId: string): AgentActionTraceState | undefined {- return [...this.#actions.values()].find(- (state) => state.sessionId === sessionId && state.callId === callId,- );- }-- findActionAnchor(- sessionId: string,- turnId: string,- callId: string,- ): AgentActionTraceState | undefined {- return [...this.#actionAnchors.values()].find(- (state) =>- state.sessionId === sessionId && state.turnId === turnId && state.callId === callId,- );- }-- getAction(idempotencyKey: string): AgentActionTraceState | undefined {- return this.#actions.get(idempotencyKey);- }-- getSession(sessionId: string): AgentSessionTraceState | undefined {- return this.#sessions.get(sessionId);+ delete(kind: keyof AgentTraceValues, key: string): void {+ this.#entries.delete(JSON.stringify([kind, key]));}-- getTurn(sessionId: string, turnId: string): AgentTurnTraceState | undefined {- return this.#turns.get(turnKey(sessionId, turnId));- }-- setAction(idempotencyKey: string, state: AgentActionTraceState): void {- this.#actions.set(idempotencyKey, state);- }-- setActionAnchor(idempotencyKey: string, state: AgentActionTraceState): void {- this.#actionAnchors.set(idempotencyKey, state);- }-- setSession(sessionId: string, state: AgentSessionTraceState): void {- this.#sessions.set(sessionId, state);- }-- setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void {- this.#turns.set(turnKey(sessionId, turnId), state);- }-- updateTurn(- sessionId: string,- turnId: string,- update: (state: AgentTurnTraceState) => AgentTurnTraceState,+ update<K extends keyof AgentTraceValues>(+ kind: K,+ key: string,+ update: (value: AgentTraceValues[K]) => AgentTraceValues[K],): void {- const key = turnKey(sessionId, turnId);- const state = this.#turns.get(key);- if (state !== undefined) this.#turns.set(key, update(state));+ const value = this.get(kind, key);+ if (value !== undefined) this.set(kind, key, update(value));+ }+ entries<K extends keyof AgentTraceValues>(kind: K): [string, AgentTraceValues[K]][] {+ return [...this.#entries].flatMap(([key, value]) => {+ const [storedKind, id] = JSON.parse(key);+ return storedKind === kind+ ? [[id, value as AgentTraceValues[K]] as [string, AgentTraceValues[K]]]+ : [];+ });}-}--function turnKey(sessionId: string, turnId: string): string {- return `${sessionId}:${turnId}`;}diff --git a/packages/eve/src/public/channels/eve.test.ts b/packages/eve/src/public/channels/eve.test.tsindex 26b75cbf6..87760e430 100644--- a/packages/eve/src/public/channels/eve.test.ts+++ b/packages/eve/src/public/channels/eve.test.ts@@ -1010,6 +1010,77 @@ describe("eveChannel — onMessage", () => {});describe("eveChannel — create session idempotency", () => {+ it.each([true, false])(+ "accepts library handoff only with trusted forwarded provenance (%s)",+ async (trusted) => {+ const { AsyncLocalStorageContextManager } =+ await import("@opentelemetry/context-async-hooks");+ const { context } = await import("@opentelemetry/api");+ const manager = new AsyncLocalStorageContextManager().enable();+ context.setGlobalContextManager(manager);+ try {+ const handler = createEveCreateHandler({+ auth: () => ACCEPTED_AUTH,+ trustedForwarders: () => trusted,+ });+ const request = createJsonMessageRequest({+ callback: {+ callId: "call-1",+ subagentName: "research",+ token: "tok123",+ url: "https://caller.example.com/eve/v1/callback/tok123",+ },+ message: "hi",+ protocolVersion: REMOTE_AGENT_PROTOCOL_VERSION,+ ...(trusted+ ? {+ forwardedPrincipal: {+ current: {+ attributes: {},+ authenticator: "api-key",+ principalId: "user",+ principalType: "user",+ },+ },+ }+ : {}),+ });+ request.headers.set("traceparent", `00-${"a".repeat(32)}-${"b".repeat(16)}-01`);+ request.headers.set(+ "baggage",+ writeForwardedParentSessionBaggage(undefined, {+ callId: "call-1",+ rootSessionId: "root",+ sessionId: "parent",+ turn: { id: "turn", sequence: 0 },+ })!,+ );+ request.headers.set(+ "x-agent-tracing",+ JSON.stringify({+ caller: { traceId: "a".repeat(32), spanId: "b".repeat(16), traceFlags: 1 },+ conversationId: "handoff-conversation",+ parentRunId: "parent",+ parentCallId: "call-1",+ agentName: "research",+ capture: { emit: true, recordInputs: false, recordOutputs: false },+ }),+ );+ expect((await handler.fetch(request)).status).toBe(202);+ const options = handler.createSession.mock.calls[0]?.[0];+ expect(options?.conversationId).toBe(trusted ? "handoff-conversation" : undefined);+ if (trusted)+ expect(options?.parentTraceContext?.decision).toEqual({+ action: "record",+ recordInputs: false,+ recordOutputs: false,+ });+ } finally {+ context.disable();+ manager.disable();+ }+ },+ );it("creates once for an operation id and uses it as the continuation token", async () => {const handler = createEveCreateHandler({ auth: () => ACCEPTED_AUTH });diff --git a/packages/eve/src/runtime/connections/mcp-tracing.test.ts b/packages/eve/src/runtime/connections/mcp-tracing.test.tsindex 11d5185dc..034e56143 100644--- a/packages/eve/src/runtime/connections/mcp-tracing.test.ts+++ b/packages/eve/src/runtime/connections/mcp-tracing.test.ts@@ -23,10 +23,7 @@ import {withMcpToolCallSpan,withMcpToolsListSpan,} from "#runtime/connections/mcp-tracing.js";-import {- withAgentToolContentPolicy,- withAgentToolSpanContext,-} from "#tracing/agent-tool-span-context.js";+import { otelTelemetry } from "#compiled/@eve/tracing/index.js";describe("MCP trace propagation", () => {let exporter: InMemorySpanExporter;@@ -51,11 +48,20 @@ describe("MCP trace propagation", () => {it("injects configured propagation into params._meta and annotates the tool span", async () => {const setAttributes = vi.fn();- const toolContext = withAgentToolSpanContext(ROOT_CONTEXT, {- recordInputs: false,- recordOutputs: false,- setAttributes,- });+ const capture = { emit: true, recordInputs: false, recordOutputs: false };+ const telemetry = otelTelemetry({ provider });+ let toolContext = ROOT_CONTEXT;+ telemetry.run(+ {+ type: "tool",+ reference: { traceId: "1".repeat(32), spanId: "2".repeat(16), traceFlags: 1 },+ capture,+ mcp: { update: setAttributes, error() {}, arguments() {}, result() {} },+ },+ () => {+ toolContext = runtimeContext.active();+ },+ );const fetcher = vi.fn(async (_request: Parameters<typeof fetch>[0], _init?: Parameters<typeof fetch>[1]) =>new Response(null, { headers: { "mcp-session-id": "session-1" } }),@@ -102,17 +108,14 @@ describe("MCP trace propagation", () => {expect(requestBody.params._meta).not.toHaveProperty("recordOutputs");expect(setAttributes).toHaveBeenCalledWith(expect.objectContaining({- "eve.connection.name": "linear",- "gen_ai.operation.name": "execute_tool",- "gen_ai.tool.name": "get_issue",- "jsonrpc.request.id": "42",- "mcp.method.name": "tools/call",- "mcp.protocol.version": "2025-11-25",- "network.protocol.name": "http",- "network.transport": "tcp",+ connectionName: "linear",+ toolName: "get_issue",+ requestId: "42",+ method: "tools/call",+ protocolVersion: "2025-11-25",}),);- expect(setAttributes).toHaveBeenCalledWith({ "mcp.session.id": "session-1" });+ expect(setAttributes).toHaveBeenCalledWith({ sessionId: "session-1" });});it("leaves oversized JSON-RPC requests untouched", async () => {@@ -174,10 +177,17 @@ describe("MCP trace propagation", () => {it("propagates the fallback CLIENT span context for tools/call", async () => {const parent = runtimeTrace.getTracer("test.mcp").startSpan("parent");- const parentContext = withAgentToolContentPolicy(runtimeTrace.setSpan(ROOT_CONTEXT, parent), {- recordInputs: true,- recordOutputs: true,- });+ let parentContext = ROOT_CONTEXT;+ otelTelemetry({ provider }).run(+ {+ type: "activation",+ reference: parent.spanContext(),+ capture: { emit: true, recordInputs: true, recordOutputs: true },+ },+ () => {+ parentContext = runtimeContext.active();+ },+ );const fetcher = vi.fn(async (_request: Parameters<typeof fetch>[0], _init?: Parameters<typeof fetch>[1]) =>new Response(null, { headers: { "mcp-session-id": "session-1" } }),diff --git a/packages/eve/src/runtime/connections/mcp-tracing.ts b/packages/eve/src/runtime/connections/mcp-tracing.tsindex d7df9a7ad..32f20d28f 100644--- a/packages/eve/src/runtime/connections/mcp-tracing.ts+++ b/packages/eve/src/runtime/connections/mcp-tracing.ts@@ -1,380 +1,34 @@-import {- context as otelContext,- createContextKey,- propagation,- SpanKind,- trace,- type Attributes,- type Context,- type Span,- type TextMapSetter,-} from "#compiled/@opentelemetry/api/index.js";--import { contentAttribute } from "#tracing/agent-otel-content.js";-import { recordAgentSpanError } from "#tracing/agent-span-error.js";-import {- agentToolContentPolicy,- agentToolSpanContext,- annotateAgentToolSpan,- recordAgentToolSpanError,- withAgentToolSpanContext,-} from "#tracing/agent-tool-span-context.js";-import { truncateTelemetryText } from "#tracing/telemetry-budget.js";+import { createMcpTraceFetch as traceFetch } from "#tracing/eve/mcp.js";+import { createMcpTracing } from "#tracing/eve/mcp.js";import { replaceBaggageMember } from "#protocol/baggage.js";-import { isObject } from "#shared/guards.js";--const MAX_MCP_TRACE_REQUEST_BYTES = 1024 * 1024;-const MAX_MCP_TRACE_CONTEXT_BYTES = 8192;-const utf8Encoder = new TextEncoder();-const EVE_TRACE_BAGGAGE_KEYS = [- "eve.audience",- "eve.conversation.id",- "eve.parent_session",-] as const;-const traceContextSetter: TextMapSetter<Record<string, string>> = {- set(carrier, key, value) {- carrier[key] = value;- },-};-const MCP_METHOD_NAME_CONTEXT_KEY = createContextKey("eve.mcp.method-name");--export function createMcpTraceFetch(input: {- readonly connectionName: string;- readonly fetcher?: typeof fetch;- readonly getActiveContext?: () => Context;- readonly getProtocolVersion: () => string | undefined;- readonly injectContext?: McpTraceContextInjector;-}): typeof fetch {- const fetcher = input.fetcher ?? ((request, init) => globalThis.fetch(request, init));- const getActiveContext = input.getActiveContext ?? (() => otelContext.active());- const injectContext = input.injectContext ?? injectOpenTelemetryTraceContext;-- return async (request, init) => {- const message = readJsonRpcRequest(init?.body);- if (message === undefined) return await fetcher(request, init);-- const activeContext = getActiveContext();- const propagated = injectMcpTraceContext(message, activeContext, injectContext);- const annotateRequest =- message.method === "tools/call" || mcpMethodName(activeContext) === message.method;- if (annotateRequest) {- annotateAgentToolSpan(- mcpRequestAttributes({- connectionName: input.connectionName,- message,- params: isObject(message.params) ? message.params : {},- protocolVersion:- headerValue(init?.headers, "mcp-protocol-version") ?? input.getProtocolVersion(),- }),- activeContext,- );- }-- const response = await fetcher(- request,- propagated === undefined ? init : { ...init, body: JSON.stringify(propagated) },- );- if (annotateRequest) {- const sessionId = response.headers.get("mcp-session-id");- if (sessionId !== null) {- annotateAgentToolSpan(- { "mcp.session.id": truncateTelemetryText(sessionId, 512) },- activeContext,- );- }- }- return response;- };-}-function injectMcpTraceContext(- message: JsonRpcRequest,- activeContext: Context,- injectContext: McpTraceContextInjector = injectOpenTelemetryTraceContext,-): JsonRpcRequest | undefined {- if (!withinTraceRequestLimit(JSON.stringify(message))) return undefined;-- if (message.params !== undefined && !isObject(message.params)) return undefined;- const propagated: Record<string, string> = {};- injectContext(activeContext, propagated);- if (!withinTraceContextLimit(propagated)) return undefined;- stripEveTraceBaggage(propagated);-- const params = isObject(message.params) ? message.params : {};- const existingMeta = isObject(params["_meta"]) ? params["_meta"] : {};- return {- ...message,- params: {- ...params,- _meta: { ...existingMeta, ...propagated },+export function createMcpTraceFetch(input: Parameters<typeof traceFetch>[0]): typeof fetch {+ return traceFetch({+ ...input,+ filterCarrier(carrier) {+ let baggage = carrier.baggage;+ for (const key of ["eve.audience", "eve.conversation.id", "eve.parent_session"])+ baggage = replaceBaggageMember(baggage, key, undefined);+ if (baggage === undefined) delete carrier.baggage;+ else carrier.baggage = baggage;},- };-}--export async function withMcpToolsListSpan<T>(input: {- readonly connectionName: string;- readonly execute: () => Promise<T>;- readonly protocolVersion?: string;-}): Promise<T> {- const parent = otelContext.active();- const attributes = mcpRequestAttributes({- connectionName: input.connectionName,- method: "tools/list",- protocolVersion: input.protocolVersion,});- const span = trace.getTracer("eve.mcp").startSpan(- "tools/list",- {- attributes,- kind: SpanKind.CLIENT,- },- parent,- );- const spanContext = withMcpMethodName(- withAgentToolSpanContext(trace.setSpan(parent, span), {- ...agentToolContentPolicy(parent),- recordError: (error, errorType) => recordAgentSpanError(span, error, errorType),- setAttributes: (spanAttributes) => setSpanAttributes(span, spanAttributes),- }),- "tools/list",- );- try {- return await otelContext.with(spanContext, async () => {- try {- return await input.execute();- } catch (error) {- const code = jsonRpcErrorCode(error);- if (code !== undefined) {- annotateAgentToolSpan({ "rpc.response.status_code": code }, spanContext);- }- recordAgentToolSpanError(error, errorType(error, code), spanContext);- throw error;- }- });- } finally {- span.end();- }}-export async function withMcpToolCallSpan<T>(input: {- readonly arguments: unknown;- readonly connectionName: string;- readonly execute: () => Promise<T>;- readonly protocolVersion?: string;- readonly toolName: string;+export function withMcpToolsListSpan<T>(input: {+ connectionName: string;+ protocolVersion?: string;+ execute: () => Promise<T>;}): Promise<T> {- const parent = otelContext.active();- const existing = agentToolSpanContext(parent);- const policy = agentToolContentPolicy(parent);- if (existing?.setAttributes !== undefined) {- annotateAgentToolSpan(mcpToolCallAttributes(input), parent);- return await runMcpToolCall(input, parent, undefined, policy);- }-- const span = trace.getTracer("eve.mcp").startSpan(- `tools/call ${truncateTelemetryText(input.toolName, 128)}`,- {- attributes: mcpToolCallAttributes(input),- kind: SpanKind.CLIENT,- },- parent,- );- const spanContext = withMcpMethodName(- withAgentToolSpanContext(trace.setSpan(parent, span), {- ...policy,- recordError: (error, errorType) => recordAgentSpanError(span, error, errorType),- setAttributes: (attributes) => setSpanAttributes(span, attributes),- }),- "tools/call",- );-- try {- return await otelContext.with(spanContext, () =>- runMcpToolCall(input, spanContext, span, policy),- );- } finally {- span.end();- }-}--function runMcpToolCall<T>(- input: {- readonly arguments: unknown;- readonly execute: () => Promise<T>;- },- context: Context,- fallbackSpan: Span | undefined,- policy: ReturnType<typeof agentToolContentPolicy>,-): Promise<T> {- if (fallbackSpan !== undefined && policy.recordInputs) {- const argumentsAttribute = contentAttribute(input.arguments);- if (argumentsAttribute !== undefined) {- fallbackSpan.setAttribute("gen_ai.tool.call.arguments", argumentsAttribute);- }- }-- return input- .execute()- .then((result) => {- if (isToolErrorResult(result)) {- recordAgentToolSpanError(undefined, "tool_error", context);- }- if (fallbackSpan !== undefined && policy.recordOutputs) {- const resultAttribute = contentAttribute(result);- if (resultAttribute !== undefined) {- fallbackSpan.setAttribute("gen_ai.tool.call.result", resultAttribute);- }- }- return result;- })- .catch((error: unknown) => {- const code = jsonRpcErrorCode(error);- if (code !== undefined) {- annotateAgentToolSpan({ "rpc.response.status_code": code }, context);- }- if (fallbackSpan !== undefined) {- recordAgentToolSpanError(error, errorType(error, code), context);- }- throw error;- });-}--function mcpToolCallAttributes(input: {- readonly connectionName: string;- readonly protocolVersion?: string;- readonly toolName: string;-}): Attributes {- return mcpRequestAttributes({- connectionName: input.connectionName,- method: "tools/call",- protocolVersion: input.protocolVersion,- toolName: input.toolName,- });-}--function mcpRequestAttributes(input: {- readonly connectionName: string;- readonly message?: JsonRpcRequest;- readonly method?: string;- readonly params?: Record<string, unknown>;- readonly protocolVersion?: string;- readonly toolName?: string;-}): Attributes {- const method = input.method ?? input.message?.method;- const params = input.params ?? (isObject(input.message?.params) ? input.message.params : {});- const attributes: Attributes = {- "eve.connection.name": input.connectionName,- "mcp.method.name": method ?? "unknown",- "network.protocol.name": "http",- "network.transport": "tcp",- };- const toolName =- input.toolName ?? (typeof params["name"] === "string" ? params["name"] : undefined);- if (method === "tools/call") {- attributes["gen_ai.operation.name"] = "execute_tool";- if (toolName !== undefined) attributes["gen_ai.tool.name"] = toolName;- }- const requestId = input.message?.id;- if (typeof requestId === "number" || typeof requestId === "string") {- attributes["jsonrpc.request.id"] = String(requestId);- }- const meta = params["_meta"];- const protocolVersion =- input.protocolVersion ??- (isObject(meta) && typeof meta["io.modelcontextprotocol/protocolVersion"] === "string"- ? meta["io.modelcontextprotocol/protocolVersion"]- : undefined);- if (protocolVersion !== undefined) {- attributes["mcp.protocol.version"] = protocolVersion;- }- return attributes;-}--interface JsonRpcRequest extends Record<string, unknown> {- readonly method: string;- readonly params?: unknown;-}--type McpTraceContextInjector = (context: Context, carrier: Record<string, string>) => void;--function injectOpenTelemetryTraceContext(context: Context, carrier: Record<string, string>): void {- propagation.inject(context, carrier, traceContextSetter);+ return createMcpTracing().list(input);}-function stripEveTraceBaggage(carrier: Record<string, string>): void {- let baggage = carrier["baggage"];- if (baggage === undefined) return;- for (const key of EVE_TRACE_BAGGAGE_KEYS) {- baggage = replaceBaggageMember(baggage, key, undefined);- }- if (baggage === undefined) delete carrier["baggage"];- else carrier["baggage"] = baggage;-}--function readJsonRpcRequest(body: RequestInit["body"]): JsonRpcRequest | undefined {- if (typeof body !== "string" || !withinTraceRequestLimit(body)) return undefined;- try {- const value: unknown = JSON.parse(body);- return isObject(value) && typeof value["method"] === "string"- ? (value as JsonRpcRequest)- : undefined;- } catch {- return undefined;- }-}--function withinTraceRequestLimit(value: string): boolean {- return (- value.length <= MAX_MCP_TRACE_REQUEST_BYTES &&- utf8Encoder.encode(value).byteLength <= MAX_MCP_TRACE_REQUEST_BYTES- );-}--function withinTraceContextLimit(carrier: Readonly<Record<string, string>>): boolean {- let bytes = 0;- for (const [key, value] of Object.entries(carrier)) {- if (key.length + value.length > MAX_MCP_TRACE_CONTEXT_BYTES) return false;- bytes += utf8Encoder.encode(key).byteLength + utf8Encoder.encode(value).byteLength + 2;- if (bytes > MAX_MCP_TRACE_CONTEXT_BYTES) return false;- }- return bytes > 0;-}--function headerValue(headers: RequestInit["headers"], name: string): string | undefined {- if (headers === undefined) return undefined;- try {- return new Headers(headers).get(name) ?? undefined;- } catch {- return undefined;- }-}--function isToolErrorResult(value: unknown): boolean {- return isObject(value) && value["isError"] === true;-}--function jsonRpcErrorCode(error: unknown): string | undefined {- if (!isObject(error)) return undefined;- const code = error["code"];- return typeof code === "number" || typeof code === "string" ? String(code) : undefined;-}--function errorType(error: unknown, code: string | undefined): string | undefined {- if (code !== undefined) return code;- return error instanceof Error ? error.name || "Error" : undefined;-}--function setSpanAttributes(span: Span, attributes: Attributes): void {- for (const [name, value] of Object.entries(attributes)) {- if (value !== undefined) span.setAttribute(name, value);- }-}--function withMcpMethodName(context: Context, methodName: string): Context {- return context.setValue(MCP_METHOD_NAME_CONTEXT_KEY, methodName);-}--function mcpMethodName(context: Context): string | undefined {- const methodName = context.getValue(MCP_METHOD_NAME_CONTEXT_KEY);- return typeof methodName === "string" ? methodName : undefined;+export function withMcpToolCallSpan<T>(input: {+ connectionName: string;+ protocolVersion?: string;+ toolName: string;+ arguments: unknown;+ execute: () => Promise<T>;+}): Promise<T> {+ return createMcpTracing().call(input);}diff --git a/packages/eve/src/tracing/sampled-trace.test.ts b/packages/eve/src/shared/trace-policy.test.tssimilarity index 99%rename from packages/eve/src/tracing/sampled-trace.test.tsrename to packages/eve/src/shared/trace-policy.test.tsindex e48709178..d5642c503 100644--- a/packages/eve/src/tracing/sampled-trace.test.ts+++ b/packages/eve/src/shared/trace-policy.test.ts@@ -1,6 +1,6 @@import { afterEach, describe, expect, it, vi } from "vitest";-import { resolveTracePolicy, resolveTracePolicyDecision } from "#tracing/sampled-trace.js";+import { resolveTracePolicy, resolveTracePolicyDecision } from "#shared/trace-policy.js";const contentContext = (audience: "public" | "private" | "unknown",diff --git a/packages/eve/src/shared/trace-policy.ts b/packages/eve/src/shared/trace-policy.tsindex 1d554180c..3610cd440 100644--- a/packages/eve/src/shared/trace-policy.ts+++ b/packages/eve/src/shared/trace-policy.ts@@ -1,5 +1,10 @@import { shouldCaptureInstrumentationContent } from "#shared/instrumentation-content.js";import type { ConversationContext } from "#shared/conversation-context.js";++// Keep sampled-flag checks free of OTel imports for workflow driver bundles.+export function isSampledTrace(context: { readonly traceFlags: number }): boolean {+ return (context.traceFlags & 1) !== 0;+}import {DROP_INSTRUMENTATION,type InstrumentationDecision,diff --git a/packages/eve/src/tracing/agent-approval-instrumentation.ts b/packages/eve/src/tracing/agent-approval-instrumentation.tsdeleted file mode 100644index c3ef5df8b..000000000--- a/packages/eve/src/tracing/agent-approval-instrumentation.ts+++ /dev/null@@ -1,193 +0,0 @@-import {- ROOT_CONTEXT,- type SpanContext,- type Tracer,- trace,-} from "#compiled/@opentelemetry/api/index.js";--import type {- InstrumentationHandlerContext,- InstrumentationInputRequestedEvent,- InstrumentationInputResolvedEvent,- InstrumentationProviderDefinition,-} from "#instrumentation/lifecycle.js";-import type { JsonValue } from "#shared/json.js";-import { contentAttribute } from "#tracing/agent-otel-content.js";-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";-import { agentTraceIdentityAttributes, traceSessionIdOf } from "#tracing/agent-otel-attributes.js";-import { decodeTraceSessionId } from "#tracing/agent-trace-context-codec.js";-import { AGENT_SPAN_NAMES } from "#tracing/agent-span-contract.js";-import { recordAgentSpanError as recordError } from "#tracing/agent-span-error.js";-import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";-import type { AgentToolContext } from "#tracing/agent-tool-instrumentation.js";-import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";-import { normalizeChannelAudience, type ChannelAudience } from "#shared/channel-audience.js";--interface AgentApprovalSpanState {- readonly traceSessionId: string;- readonly actionCallId: string;- readonly actionName: string;- readonly attemptIndex: number;- readonly channelAudience: ChannelAudience;- readonly parent: SpanContext;- readonly requestAttribute?: string;- readonly requestId: string;- readonly rootSessionId: string;- readonly sessionId: string;- readonly startTimeMs: number;- readonly stepIndex: number;- readonly turnId: string;-}--/** Builds durable approval wait spans under their originating runtime action. */-export function createAgentApprovalInstrumentation(input: {- readonly actionContextFor: (- sessionId: string,- turnId: string,- callId: string,- ) => Promise<AgentToolContext | undefined>;- readonly frameworkVersion: string;- readonly idGenerator: AgentSpanIdGenerator;- readonly tracer: Tracer;-}): Pick<- NonNullable<InstrumentationProviderDefinition["events"]>,- "input.requested" | "input.resolved"-> {- const onRequested = async (- event: InstrumentationInputRequestedEvent,- ctx: InstrumentationHandlerContext,- ): Promise<void> => {- if (event.kind !== "tool-approval" || readState(ctx.state.get()) !== undefined) return;- const parent = await input.actionContextFor(- event.scope.sessionId,- event.scope.turnId,- event.action.callId,- );- if (parent === undefined) return;- const state: Record<string, JsonValue> = {- actionCallId: event.action.callId,- actionName: event.action.name,- attemptIndex: event.scope.attemptIndex,- channelAudience: normalizeChannelAudience(event.scope.channelAudience),- parent: {- spanId: parent.spanContext.spanId,- traceFlags: parent.spanContext.traceFlags,- traceId: parent.spanContext.traceId,- },- requestId: event.requestId,- rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,- traceSessionId: traceSessionIdOf(event.scope),- sessionId: event.scope.sessionId,- startTimeMs: Date.now(),- stepIndex: event.scope.stepIndex,- turnId: event.scope.turnId,- };- const requestAttribute = contentAttribute(event.request);- if (requestAttribute !== undefined) state["requestAttribute"] = requestAttribute;- ctx.state.set(state);- };-- const onResolved = (- event: InstrumentationInputResolvedEvent,- ctx: InstrumentationHandlerContext,- ): void => {- const state = readState(ctx.state.get());- if (state === undefined) return;- const span = input.idGenerator.withSpanId(- input.idGenerator.deriveSpanId(`approval:${event.idempotencyKey}`),- () =>- input.tracer.startSpan(- AGENT_SPAN_NAMES.approval,- {- attributes: {- "gen_ai.tool.call.id": state.actionCallId,- "agent.action.name": state.actionName,- "agent.approval.kind": "tool-approval",- "agent.approval.outcome": event.outcome,- "agent.approval.request_id": state.requestId,- "agent.framework.name": "eve",- "agent.framework.version": input.frameworkVersion,- "agent.step.attempt": state.attemptIndex,- "agent.step.index": state.stepIndex,- "agent.turn.id": state.turnId,- "gen_ai.operation.name": "workflow",- ...agentSpanNamingAttributes("agent.approval", "workflow"),- ...agentTraceIdentityAttributes({- rootSessionId: state.rootSessionId,- traceSessionId: state.traceSessionId,- sessionId: state.sessionId,- }),- },- startTime: state.startTimeMs,- },- withChannelAudience(- trace.setSpan(- ROOT_CONTEXT,- trace.wrapSpanContext({ ...state.parent, isRemote: false }),- ),- state.channelAudience,- ),- ),- );- if (state.requestAttribute !== undefined) {- span.setAttribute("agent.approval.request", state.requestAttribute);- }- if (event.response !== undefined) {- const response = contentAttribute(event.response);- if (response !== undefined) span.setAttribute("agent.approval.response", response);- }- if (event.outcome === "failed") recordError(span, event.error);- span.end();- };-- return {- "input.requested": onRequested,- "input.resolved": onResolved,- };-}--function readState(value: unknown): AgentApprovalSpanState | undefined {- if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;- const state = value as Record<string, unknown>;- const parent = state["parent"];- if (typeof parent !== "object" || parent === null || Array.isArray(parent)) return undefined;- const parentRecord = parent as Record<string, unknown>;- if (- typeof state["actionCallId"] !== "string" ||- typeof state["actionName"] !== "string" ||- typeof state["attemptIndex"] !== "number" ||- typeof parentRecord["spanId"] !== "string" ||- typeof parentRecord["traceFlags"] !== "number" ||- typeof parentRecord["traceId"] !== "string" ||- typeof state["requestId"] !== "string" ||- typeof state["rootSessionId"] !== "string" ||- typeof state["sessionId"] !== "string" ||- typeof state["startTimeMs"] !== "number" ||- typeof state["stepIndex"] !== "number" ||- typeof state["turnId"] !== "string"- ) {- return undefined;- }- const requestAttribute = state["requestAttribute"];- if (requestAttribute !== undefined && typeof requestAttribute !== "string") return undefined;- return {- actionCallId: state["actionCallId"],- actionName: state["actionName"],- attemptIndex: state["attemptIndex"],- channelAudience: normalizeChannelAudience(state["channelAudience"]),- parent: {- isRemote: false,- spanId: parentRecord["spanId"],- traceFlags: parentRecord["traceFlags"],- traceId: parentRecord["traceId"],- },- requestAttribute,- requestId: state["requestId"],- rootSessionId: state["rootSessionId"],- traceSessionId: decodeTraceSessionId(state),- sessionId: state["sessionId"],- startTimeMs: state["startTimeMs"],- stepIndex: state["stepIndex"],- turnId: state["turnId"],- };-}diff --git a/packages/eve/src/tracing/agent-memory-instrumentation.ts b/packages/eve/src/tracing/agent-memory-instrumentation.tsdeleted file mode 100644index 5a31a500a..000000000--- a/packages/eve/src/tracing/agent-memory-instrumentation.ts+++ /dev/null@@ -1,195 +0,0 @@-import {- ROOT_CONTEXT,- SpanKind,- context,- trace,- type Context,- type Span,- type Tracer,-} from "#compiled/@opentelemetry/api/index.js";--import { contextStorage } from "#context/container.js";-import { SessionTraceSeedKey } from "#context/keys.js";-import type { InstrumentationProviderDefinition } from "#instrumentation/lifecycle.js";-import type {- InstrumentationMemoryExecutionOperation,- InstrumentationMemoryOperationEvent,- InstrumentationMemoryOperationStartedEvent,- InstrumentationMemoryOperationTerminalEvent,-} from "#instrumentation/memory.js";-import { normalizeChannelAudience } from "#shared/channel-audience.js";-import {- applyLiveDeliveryAudienceCeiling,- resolveForwardedTraceSeed,-} from "#shared/forwarded-trace-policy.js";-import { genAiMemoryRecordsAttribute } from "#tracing/agent-otel-content.js";-import { agentTraceIdentityAttributes, traceSessionIdOf } from "#tracing/agent-otel-attributes.js";-import { isAgentTraceContext, markAgentTraceContext } from "#tracing/eve/agent-trace-context.js";-import { recordAgentSpanError } from "#tracing/agent-span-error.js";-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";-import type { AgentTraceStateStore } from "#tracing/agent-trace-state.js";-import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";-import { withErrorContent } from "#tracing/error-content-context.js";-import { isSampledTrace } from "#tracing/sampled-trace.js";-import { suppressTracing } from "#tracing/suppress-tracing.js";-import { withOperationConversation } from "#tracing/eve/conversation-context.js";-import type { ConversationEnvironment } from "#shared/conversation-context.js";--type SpanState = { readonly context: Context; readonly span: Span };--interface AgentMemoryInstrumentation {- readonly events: Pick<- NonNullable<InstrumentationProviderDefinition["events"]>,- "memory.operation.completed" | "memory.operation.failed" | "memory.operation.started"- >;- runInContext<T>(- operation: InstrumentationMemoryExecutionOperation,- execute: () => PromiseLike<T>,- ): Promise<T>;-}--export function createAgentMemoryInstrumentation(input: {- readonly environment: ConversationEnvironment;- readonly recordOutputs?: boolean;- readonly stateStore: AgentTraceStateStore;- readonly tracer: Tracer;-}): AgentMemoryInstrumentation {- const recordOutputs = input.recordOutputs ?? false;- const spans = new Map<string, SpanState>();-- const parentContext = async (- event: InstrumentationMemoryOperationEvent,- ): Promise<Context | undefined> => {- const active = context.active();- if (isAgentTraceContext(active)) return active;-- const session = await input.stateStore.getSession(event.sessionId);- if (event.turnId !== undefined) {- const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);- if (turn !== undefined) {- return withChannelAudience(contextFromSpanContext(turn.context), session?.channelAudience);- }- }- return session === undefined- ? undefined- : withChannelAudience(contextFromSpanContext(session.context), session.channelAudience);- };-- const onStarted = async (event: InstrumentationMemoryOperationStartedEvent): Promise<void> => {- if (spans.has(event.idempotencyKey)) return;- const parent = await parentContext(event);- const parentSpan = parent === undefined ? undefined : trace.getSpan(parent)?.spanContext();- if (parent === undefined || parentSpan === undefined || !isSampledTrace(parentSpan)) return;- const span = input.tracer.startSpan(- event.operationName,- {- attributes: memorySpanAttributes(event),- kind: SpanKind.CLIENT,- },- parent,- );- spans.set(event.idempotencyKey, {- context: trace.setSpan(parent, span),- span,- });- };-- const onTerminal = (event: InstrumentationMemoryOperationTerminalEvent): void => {- const state = spans.get(event.idempotencyKey);- if (state === undefined) return;- spans.delete(event.idempotencyKey);- updateMemorySpan(state.span, event);- state.span.end();- };-- return {- events: {- "memory.operation.completed": onTerminal,- "memory.operation.failed": onTerminal,- "memory.operation.started": onStarted,- },- async runInContext(operation, execute) {- const session = await input.stateStore.getSession(operation.sessionId);- let parent = spans.get(operation.idempotencyKey)?.context;- if (parent === undefined && operation.turnId !== undefined) {- const turn = await input.stateStore.getTurn(operation.sessionId, operation.turnId);- if (turn !== undefined) {- parent = withChannelAudience(- contextFromSpanContext(turn.context),- session?.channelAudience,- );- if (!isSampledTrace(turn.context)) parent = suppressTracing(parent);- }- }- if (parent === undefined && session !== undefined) {- parent = withChannelAudience(- contextFromSpanContext(session.context),- session.channelAudience,- );- if (!isSampledTrace(session.context)) parent = suppressTracing(parent);- }- const seed = resolveForwardedTraceSeed(contextStorage.getStore()?.get(SessionTraceSeedKey));- const decision = seed?.decision ?? session?.decision;- const effective =- decision === undefined- ? undefined- : applyLiveDeliveryAudienceCeiling(- decision,- normalizeChannelAudience(session?.channelAudience),- seed?.forwardedTracePolicy,- input.environment,- );- return parent === undefined- ? await execute()- : await context.with(- markAgentTraceContext(- withErrorContent(- withOperationConversation(parent, {- rootSessionId: session?.rootSessionId,- sessionId: operation.sessionId,- }),- recordOutputs && effective?.action === "record" && effective.recordOutputs,- ),- ),- execute,- );- },- };-}--function memorySpanAttributes(- event: InstrumentationMemoryOperationEvent,-): Record<string, string | number> {- const attributes: Record<string, string | number> = {- "agent.memory.phase": event.phase,- "agent.memory.slot": event.slot,- "gen_ai.memory.store.id": event.storeId,- "gen_ai.operation.name": event.operationName,- ...agentSpanNamingAttributes(event.operationName, event.operationName),- ...agentTraceIdentityAttributes({- rootSessionId: event.rootSessionId,- traceSessionId: traceSessionIdOf(event),- sessionId: event.sessionId,- }),- };- if (event.turnId !== undefined) attributes["agent.turn.id"] = event.turnId;- return attributes;-}--function updateMemorySpan(span: Span, event: InstrumentationMemoryOperationTerminalEvent): void {- if (event.type === "memory.operation.failed") {- recordAgentSpanError(span, event.error);- } else {- if (event.recordCount !== undefined) {- span.setAttribute("gen_ai.memory.record.count", event.recordCount);- }- const records = genAiMemoryRecordsAttribute(event.outputRecords ?? []);- if (records !== undefined && event.outputRecords !== undefined) {- span.setAttribute("gen_ai.memory.records", records);- }- }-}--function contextFromSpanContext(spanContext: Parameters<typeof trace.wrapSpanContext>[0]): Context {- return trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(spanContext));-}diff --git a/packages/eve/src/tracing/agent-otel-attributes.test.ts b/packages/eve/src/tracing/agent-otel-attributes.test.tsdeleted file mode 100644index 77ed5e893..000000000--- a/packages/eve/src/tracing/agent-otel-attributes.test.ts+++ /dev/null@@ -1,74 +0,0 @@-import { afterEach, describe, expect, it, vi } from "vitest";--import { ContextContainer, contextStorage } from "#context/container.js";-import { ConversationIdKey, TraceRootKey } from "#context/keys.js";-import { agentTraceIdentityAttributes } from "#tracing/agent-otel-attributes.js";--afterEach(() => {- vi.unstubAllEnvs();-});--describe("agentTraceIdentityAttributes", () => {- it("uses the lineage fallback when a remote entry has no conversation in context", () => {- vi.stubEnv("VERCEL_ENV", "preview");- const context = new ContextContainer();- context.set(TraceRootKey, { kind: "own" });- contextStorage.run(context, () => {- expect(- agentTraceIdentityAttributes({- rootSessionId: "caller-root",- sessionId: "remote-session",- traceSessionId: "remote-session",- }),- ).toMatchObject({- "gen_ai.conversation.id": "caller-root",- "vercel.session_id": "remote-session",- "agent.run.id": "remote-session",- });- });- });- const input = {- rootSessionId: "root-session",- sessionId: "session-1",- traceSessionId: "root-session",- };- it("uses the explicit trace session when active context belongs to another session", () => {- vi.stubEnv("VERCEL_ENV", "preview");- const context = new ContextContainer();- context.set(TraceRootKey, { kind: "inherited", sessionId: "unrelated-session" });- contextStorage.run(context, () => {- expect(- agentTraceIdentityAttributes({- rootSessionId: "caller-root",- sessionId: "remote",- traceSessionId: "remote",- })["vercel.session_id"],- ).toBe("remote");- });- });-- it("uses the GenAI conversation identifier outside Vercel", () => {- vi.stubEnv("VERCEL_ENV", undefined);-- expect(agentTraceIdentityAttributes(input)).toEqual({- "agent.run.id": "session-1",- "agent.trace.schema.version": 4,- "gen_ai.conversation.id": "root-session",- });- });-- it("adds the Vercel session identifier on Vercel", () => {- vi.stubEnv("VERCEL_ENV", "preview");- const context = new ContextContainer();- context.set(ConversationIdKey, "caller-conversation");-- contextStorage.run(context, () => {- expect(agentTraceIdentityAttributes(input)).toEqual({- "agent.run.id": "session-1",- "agent.trace.schema.version": 4,- "gen_ai.conversation.id": "caller-conversation",- "vercel.session_id": "root-session",- });- });- });-});diff --git a/packages/eve/src/tracing/agent-otel-attributes.ts b/packages/eve/src/tracing/agent-otel-attributes.tsdeleted file mode 100644index dfa77f366..000000000--- a/packages/eve/src/tracing/agent-otel-attributes.ts+++ /dev/null@@ -1,27 +0,0 @@-import { operationConversationId } from "#tracing/eve/conversation-context.js";-import { AGENT_TRACE_SCHEMA_VERSION } from "#tracing/agent-span-contract.js";--export function traceSessionIdOf(scope: {- readonly traceSessionId?: string;- readonly rootSessionId?: string;- readonly sessionId: string;-}): string {- return scope.traceSessionId ?? scope.rootSessionId ?? scope.sessionId;-}--export function agentTraceIdentityAttributes(input: {- readonly rootSessionId: string;- readonly sessionId: string;- readonly traceSessionId: string;-}): Record<string, string | number> {- const conversationId = operationConversationId(input);- const attributes: Record<string, string | number> = {- "agent.run.id": input.sessionId,- "agent.trace.schema.version": AGENT_TRACE_SCHEMA_VERSION,- "gen_ai.conversation.id": conversationId,- };- if (process.env.VERCEL_ENV !== undefined) {- attributes["vercel.session_id"] = input.traceSessionId;- }- return attributes;-}diff --git a/packages/eve/src/tracing/agent-otel-content.test.ts b/packages/eve/src/tracing/agent-otel-content.test.tsdeleted file mode 100644index 74dd467a0..000000000--- a/packages/eve/src/tracing/agent-otel-content.test.ts+++ /dev/null@@ -1,297 +0,0 @@-import { describe, expect, it, vi } from "vitest";--import type { FrameworkMessageKind } from "#harness/messages.js";-import {- CONTENT_ATTRIBUTE_LIMIT,- genAiInputMessagesAttribute,- genAiOutputMessagesAttribute,- genAiSystemInstructionsAttribute,- genAiToolDefinitionsAttribute,- toolResultsContentAttribute,-} from "#tracing/agent-otel-content.js";--const FRAMEWORK_MESSAGE_KINDS = [- "context.instruction",- "context.state",- "context.compaction",- "memory.load",- "execution.continuation",- "execution.retry",-] as const satisfies readonly FrameworkMessageKind[];--describe("GenAI message attributes", () => {- it.each([- ["Buffer", () => Buffer.alloc(64, 97)],- ["Uint8Array", () => new Uint8Array(64).fill(97)],- ["base64", () => "a".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)],- ["data URL", () => `data:image/png;base64,${"a".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)}`],- ] as const)("omits %s attachment payloads", (_, createData) => {- const data = createData();- const entries = Object.entries;- let binaryEnumerations = 0;- const spy = vi.spyOn(Object, "entries").mockImplementation((value) => {- if (value === data) binaryEnumerations += 1;- return entries(value);- });-- let attribute: string | undefined;- try {- attribute = genAiInputMessagesAttribute([- {- role: "user",- content: [- { type: "text", text: "Summarize file." },- { type: "file", mediaType: "application/octet-stream", filename: "test.bin", data },- ],- },- ]);- } finally {- spy.mockRestore();- }-- expect(binaryEnumerations).toBe(0);- expect(attribute).toBe(- '[{"parts":[{"content":"Summarize file.","type":"text"},{"type":"file"}],"role":"user"}]',- );- });-- it("preserves every user-message kind in the GenAI input attribute", () => {- expect(- genAiInputMessagesAttribute([- { content: "A real user message.", kind: "user", role: "user" },- {- content: "Continue the interrupted turn.",- kind: "execution.continuation",- role: "user",- },- ]),- ).toBe(- '[{"kind":"user","parts":[{"content":"A real user message.","type":"text"}],"role":"user"},{"kind":"execution.continuation","parts":[{"content":"Continue the interrupted turn.","type":"text"}],"role":"user"}]',- );- });-- it.each(FRAMEWORK_MESSAGE_KINDS)("preserves %s in the GenAI input attribute", (kind) => {- const attribute = genAiInputMessagesAttribute([- { content: "Framework message.", kind, role: "user" },- ]);-- expect(JSON.parse(attribute!)).toEqual([- {- kind,- parts: [{ content: "Framework message.", type: "text" }],- role: "user",- },- ]);- });-- it.each([- ["only", [{ content: "x".repeat(CONTENT_ATTRIBUTE_LIMIT + 1), role: "user" }], undefined],- [- "newest",- [- { content: "older message", role: "assistant" },- {- content: "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2),- kind: "context.state",- role: "user",- },- ],- "context.state",- ],- ] as const)("keeps a truncated %s message when it alone exceeds the cap", (_, messages, kind) => {- const attribute = genAiInputMessagesAttribute(messages);-- expect(attribute).toBeDefined();- expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);- const parsed = JSON.parse(attribute!) as Array<Record<string, unknown>>;- expect(parsed).toHaveLength(1);- expect(parsed[0]).toMatchObject({- parts: [{ content: expect.stringMatching(/… \[truncated\]$/u), type: "text" }],- role: "user",- });- expect(parsed[0]?.kind).toBe(kind);- expect(attribute).not.toContain("older message");- });-- it("formats model input, output, and system instructions for inspectors", () => {- expect(- genAiInputMessagesAttribute([- { content: "hello", kind: "user", role: "user" },- {- content: [- {- input: { message: "echo" },- toolCallId: "call-1",- toolName: "delegate",- type: "tool-call",- },- ],- role: "assistant",- },- ]),- ).toBe(- '[{"kind":"user","parts":[{"content":"hello","type":"text"}],"role":"user"},{"parts":[{"arguments":{"message":"echo"},"id":"call-1","name":"delegate","type":"tool_call"}],"role":"assistant"}]',- );- expect(genAiSystemInstructionsAttribute("Be concise.")).toBe(- '[{"content":"Be concise.","type":"text"}]',- );- expect(- genAiOutputMessagesAttribute(- [- { text: "Working.", type: "text" },- {- callId: "call-1",- input: { message: "echo" },- toolName: "delegate",- type: "tool-call",- },- ],- "tool-calls",- ),- ).toBe(- '[{"finish_reason":"tool_call","parts":[{"content":"Working.","type":"text"},{"arguments":{"message":"echo"},"id":"call-1","name":"delegate","type":"tool_call"}],"role":"assistant"}]',- );- });-- it.each([- ["near the limit", "line of instructions\n".repeat(1500)],- ["over the limit", "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)],- ["escape-heavy", "\\\n".repeat(9000)],- ])("keeps %s system instructions as valid capped JSON", (_, instructions) => {- const attribute = genAiSystemInstructionsAttribute(instructions);-- expect(attribute).toBeDefined();- expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);- const parsed = JSON.parse(attribute!) as Array<Record<string, unknown>>;- expect(parsed).toHaveLength(1);- expect(parsed[0]).toMatchObject({ type: "text" });- expect(parsed[0]?.content).toEqual(expect.stringContaining("… [truncated]"));- });-- it("marks a message truncated when oversized non-text parts are omitted", () => {- const attribute = genAiInputMessagesAttribute([- {- content: [- { text: "partial assistant answer", type: "text" },- {- input: { payload: "x".repeat(CONTENT_ATTRIBUTE_LIMIT) },- toolCallId: "call-1",- toolName: "delegate",- type: "tool-call",- },- ],- role: "assistant",- },- ]);-- expect(attribute).toBeDefined();- expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);- const parsed = JSON.parse(attribute!) as Array<Record<string, unknown>>;- expect(parsed).toEqual([- {- parts: [{ content: "partial assistant answer… [truncated]", type: "text" }],- role: "assistant",- },- ]);- });-});--describe("genAiToolDefinitionsAttribute", () => {- it("maps AI SDK input schemas to GenAI parameters", () => {- expect(- genAiToolDefinitionsAttribute([- {- description: "Get the current weather.",- inputSchema: {- properties: { city: { type: "string" } },- required: ["city"],- type: "object",- },- name: "get_weather",- },- ]),- ).toBe(- '[{"name":"get_weather","description":"Get the current weather.","parameters":{"properties":{"city":{"type":"string"}},"required":["city"],"type":"object"}}]',- );- });-- it("drops trailing definitions to keep oversized content valid JSON", () => {- const attribute = genAiToolDefinitionsAttribute([- { inputSchema: { type: "object" }, name: "first" },- {- description: "x".repeat(CONTENT_ATTRIBUTE_LIMIT),- inputSchema: { type: "object" },- name: "oversized",- },- { inputSchema: { type: "object" }, name: "last" },- ]);-- expect(attribute).toBeDefined();- expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);- expect(JSON.parse(attribute!)).toEqual([{ name: "first", parameters: { type: "object" } }]);- });-- it("returns undefined when no tool definitions are available", () => {- expect(genAiToolDefinitionsAttribute([])).toBeUndefined();- expect(genAiToolDefinitionsAttribute(undefined)).toBeUndefined();- });-});--describe("toolResultsContentAttribute", () => {- it("returns the full payload when it fits", () => {- const json = toolResultsContentAttribute([- { input: { query: "weather" }, output: { results: ["sunny"] }, toolName: "web_search" },- ]);- expect(json).toBe(- '[{"input":{"query":"weather"},"output":{"results":["sunny"]},"toolName":"web_search"}]',- );- });-- it("returns undefined for no results", () => {- expect(toolResultsContentAttribute([])).toBeUndefined();- });-- it("keeps oversized payloads valid JSON by capping entry text", () => {- const json = toolResultsContentAttribute([- {- input: { query: "weather" },- output: { excerpts: "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2) },- toolName: "web_search",- },- ]);- expect(json).toBeDefined();- expect(json!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);- const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;- expect(parsed[0]!.toolName).toBe("web_search");- expect(parsed[0]!.input).toBe('{"query":"weather"}');- expect(parsed[0]!.output).toContain("… [truncated]");- });-- it("preserves the error key through truncation", () => {- const json = toolResultsContentAttribute([- {- error: `quota exceeded ${"y".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)}`,- input: { query: "again" },- toolName: "web_search",- },- ]);- const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;- expect(parsed[0]!.error).toContain("quota exceeded");- expect(parsed[0]!.error).toContain("… [truncated]");- expect(parsed[0]!.output).toBeUndefined();- });-- it("splits the budget across many oversized entries", () => {- const entries = Array.from({ length: 8 }, (_, index) => ({- input: { index },- output: "z".repeat(CONTENT_ATTRIBUTE_LIMIT),- toolName: `tool_${index}`,- }));- const json = toolResultsContentAttribute(entries);- expect(json).toBeDefined();- expect(json!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);- const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;- expect(parsed).toHaveLength(8);- expect(parsed[7]!.toolName).toBe("tool_7");- });-});diff --git a/packages/eve/src/tracing/agent-otel-content.ts b/packages/eve/src/tracing/agent-otel-content.tsdeleted file mode 100644index 4810380c1..000000000--- a/packages/eve/src/tracing/agent-otel-content.ts+++ /dev/null@@ -1,282 +0,0 @@-import { isUserMessageKind } from "#harness/messages.js";--/**- * Serializes model and tool payloads into span content attributes for the- * local trace viewer: prompt messages, the system prompt, responses, and- * tool arguments/results. Everything is capped so a giant payload cannot- * bloat a span segment past the reader's per-file limit, and message- * payloads stay valid JSON through every truncation path.- */--/** Content attributes are capped so a giant payload cannot bloat a span segment. */-export const CONTENT_ATTRIBUTE_LIMIT = 32 * 1024;--/** Serializes one payload value as capped JSON. */-export function contentAttribute(value: unknown): string | undefined {- if (value === undefined) return undefined;- let json: string | undefined;- try {- json = JSON.stringify(value);- } catch {- return undefined;- }- if (json === undefined) return undefined;- return textContentAttribute(json);-}--/** Serializes model messages using the OpenTelemetry GenAI message schema. */-export function genAiInputMessagesAttribute(messages: unknown): string | undefined {- if (!Array.isArray(messages)) return undefined;- const formatted = messages.flatMap((message) => {- if (!isRecord(message) || message.role === "system" || typeof message.role !== "string") {- return [];- }- const parts = semanticParts(message.content);- if (parts.length === 0) return [];- const kind =- message.role === "user" && isUserMessageKind(message.kind) ? message.kind : undefined;- return [- kind === undefined ? { parts, role: message.role } : { kind, parts, role: message.role },- ];- });- for (let start = 0; start < formatted.length; start += 1) {- const json = semanticJsonAttribute(formatted.slice(start));- if (json !== undefined) return json;- }- return truncateSingleSemanticMessage(formatted) ?? semanticJsonAttribute([]);-}--/** Serializes the system prompt using the OpenTelemetry GenAI instruction schema. */-export function genAiSystemInstructionsAttribute(instructions: unknown): string | undefined {- const text = systemPromptText(instructions);- if (text === undefined) return undefined;- return fitSemanticText(text, (content) => [{ content, type: "text" }]);-}--/** Serializes tool definitions, dropping trailing entries to keep the JSON whole. */-export function genAiToolDefinitionsAttribute(tools: unknown): string | undefined {- if (!Array.isArray(tools)) return undefined;- const definitions = tools.flatMap((tool): Record<string, unknown>[] => {- if (!isRecord(tool) || typeof tool.name !== "string") return [];- const definition: Record<string, unknown> = { name: tool.name };- if (typeof tool.description === "string") definition.description = tool.description;- if (tool.inputSchema !== undefined) definition.parameters = tool.inputSchema;- return [definition];- });- if (definitions.length === 0) return undefined;-- const serialized: string[] = [];- let length = 2;- for (const definition of definitions) {- const json = stringifyContent(definition);- if (json === undefined) break;- const nextLength = length + (serialized.length === 0 ? 0 : 1) + json.length;- if (nextLength > CONTENT_ATTRIBUTE_LIMIT) break;- serialized.push(json);- length = nextLength;- }- return `[${serialized.join(",")}]`;-}--/** Serializes one model response using the OpenTelemetry GenAI message schema. */-export function genAiOutputMessagesAttribute(- content: readonly unknown[],- finishReason: string,-): string | undefined {- const parts = semanticParts(content);- return semanticJsonAttribute([- {- finish_reason: finishReason === "tool-calls" ? "tool_call" : finishReason,- parts,- role: "assistant",- },- ]);-}--/** Serializes memory records using the OpenTelemetry GenAI memory-records schema. */-export function genAiMemoryRecordsAttribute(records: readonly unknown[]): string | undefined {- return semanticJsonAttribute(records);-}--/**- * Serializes provider-executed tool results, keeping the attribute valid- * JSON under the cap: when the full payload is too big, each entry's input- * and output collapse to capped text at progressively smaller budgets- * instead of cutting the JSON mid-string.- */-export function toolResultsContentAttribute(- results: readonly Record<string, unknown>[],-): string | undefined {- if (results.length === 0) return undefined;- const full = stringifyContent(results);- if (full !== undefined && full.length <= CONTENT_ATTRIBUTE_LIMIT) return full;- for (let cap = CONTENT_ATTRIBUTE_LIMIT; cap >= 0; cap = cap >= 256 ? Math.floor(cap / 2) : -1) {- const json = stringifyContent(results.map((entry) => cappedToolResult(entry, cap)));- if (json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT) return json;- }- return undefined;-}--function cappedToolResult(entry: Record<string, unknown>, cap: number): Record<string, unknown> {- const out: Record<string, unknown> = { toolName: entry.toolName };- for (const key of ["input", "output", "error"]) {- if (!(key in entry)) continue;- const value = entry[key];- const text = typeof value === "string" ? value : (stringifyContent(value) ?? "");- out[key] = text.length <= cap ? text : `${text.slice(0, cap)}… [truncated]`;- }- return out;-}--/** Normalizes the AI SDK's `instructions` prompt to plain text. */-function systemPromptText(instructions: unknown): string | undefined {- if (typeof instructions === "string") return instructions.length === 0 ? undefined : instructions;- if (!isRecord(instructions) && !Array.isArray(instructions)) return undefined;- const messages = Array.isArray(instructions) ? instructions : [instructions];- const texts: string[] = [];- for (const message of messages) {- if (!isRecord(message)) continue;- if (typeof message.content === "string") texts.push(message.content);- else if (Array.isArray(message.content))- for (const part of message.content)- if (isRecord(part) && part.type === "text" && typeof part.text === "string")- texts.push(part.text);- }- const joined = texts.join("\n\n").trim();- return joined.length === 0 ? undefined : joined;-}--/** Caps plain text, marking the cut. */-export function textContentAttribute(text: string): string | undefined {- if (text.length === 0) return undefined;- return text.length <= CONTENT_ATTRIBUTE_LIMIT- ? text- : `${text.slice(0, CONTENT_ATTRIBUTE_LIMIT)}… [truncated]`;-}--function stringifyContent(value: unknown): string | undefined {- try {- return JSON.stringify(value);- } catch {- return undefined;- }-}--function semanticJsonAttribute(value: unknown): string | undefined {- const json = stringifyContent(value);- return json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT ? json : undefined;-}--function truncateSingleSemanticMessage(- messages: readonly Record<string, unknown>[],-): string | undefined {- const message = messages.at(-1);- if (message === undefined || typeof message.role !== "string") return undefined;- const text = semanticMessageText(message.parts);- const base =- typeof message.kind === "string"- ? { kind: message.kind, role: message.role }- : { role: message.role };- return fitSemanticText(- text,- (content) => [{ ...base, parts: [{ content, type: "text" }] }],- true,- );-}--function fitSemanticText(- text: string,- wrap: (content: string) => unknown,- alwaysMark = false,-): string | undefined {- if (!alwaysMark && text.length <= CONTENT_ATTRIBUTE_LIMIT) {- const full = semanticJsonAttribute(wrap(text));- if (full !== undefined) return full;- }- for (let length = Math.min(text.length, CONTENT_ATTRIBUTE_LIMIT); length > 0; length -= 256) {- const json = semanticJsonAttribute(wrap(`${text.slice(0, length)}… [truncated]`));- if (json !== undefined) return json;- }- return semanticJsonAttribute(wrap("… [truncated]"));-}--function semanticMessageText(parts: unknown): string {- if (!Array.isArray(parts)) return "";- let text = "";- for (const part of parts) {- if (!isRecord(part) || part.type !== "text" || typeof part.content !== "string") continue;- const separator = text.length === 0 ? "" : "\n";- const remaining = CONTENT_ATTRIBUTE_LIMIT + 1 - text.length - separator.length;- if (remaining <= 0) break;- text += separator + part.content.slice(0, remaining);- }- return text;-}--function semanticParts(content: unknown): Record<string, unknown>[] {- if (typeof content === "string") return [{ content, type: "text" }];- if (!Array.isArray(content)) return [];- const parts: Record<string, unknown>[] = [];- for (const part of content) {- const formatted = semanticPart(part);- if (formatted !== undefined) parts.push(formatted);- }- return parts;-}--function semanticPart(value: unknown): Record<string, unknown> | undefined {- if (!isRecord(value) || typeof value.type !== "string") return undefined;- switch (value.type) {- case "text":- case "reasoning": {- const content = typeof value.text === "string" ? value.text : value.content;- return typeof content === "string" ? { content, type: value.type } : undefined;- }- case "tool-call":- return {- arguments: value.input,- id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,- name: value.toolName,- type: "tool_call",- };- case "tool-result":- return {- id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,- response: toolResponse(value.output),- type: "tool_call_response",- };- case "tool-error":- return {- id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,- response: value.error instanceof Error ? value.error.message : value.error,- type: "tool_call_response",- };- default:- return { type: value.type };- }-}--function toolResponse(output: unknown): unknown {- if (!isRecord(output)) return output;- if (- (output.type === "text" ||- output.type === "error-text" ||- output.type === "json" ||- output.type === "error-json") &&- "value" in output- ) {- return output.value;- }- if (output.type === "execution-denied") {- return { denied: true, reason: output.reason };- }- return output;-}--function stringValue(value: unknown): string | undefined {- return typeof value === "string" ? value : undefined;-}--function isRecord(value: unknown): value is Record<string, unknown> {- return typeof value === "object" && value !== null && !Array.isArray(value);-}diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsdeleted file mode 100644index 98ee7c459..000000000--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ /dev/null@@ -1,706 +0,0 @@-import {- ROOT_CONTEXT,- SpanKind,- context,- type Context,- type Link,- type Span,- type SpanContext,- type Tracer,- trace,-} from "#compiled/@opentelemetry/api/index.js";--import { contextStorage } from "#context/container.js";-import { SessionTraceSeedKey } from "#context/keys.js";-import { withoutInstrumentationContent } from "#instrumentation/content.js";-import { instrumentationEventForTraceDecision } from "#instrumentation/content-policy.js";-import type { AgentTraceStateStore, AgentTurnTraceState } from "#tracing/agent-trace-state.js";-import {- contentAttribute,- genAiInputMessagesAttribute,- genAiOutputMessagesAttribute,- genAiSystemInstructionsAttribute,- genAiToolDefinitionsAttribute,- textContentAttribute,- toolResultsContentAttribute,-} from "#tracing/agent-otel-content.js";-import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";-import { createAgentApprovalInstrumentation } from "#tracing/agent-approval-instrumentation.js";-import { createAgentChannelDeliveryInstrumentation } from "#tracing/eve/agent-channel-delivery-instrumentation.js";-import { createAgentToolInstrumentation } from "#tracing/agent-tool-instrumentation.js";-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";-import { markAgentTraceContext } from "#tracing/eve/agent-trace-context.js";-import { agentTraceIdentityAttributes, traceSessionIdOf } from "#tracing/agent-otel-attributes.js";-import * as runtimeAttributes from "#tracing/agent-otel-runtime-context.js";-import { createAgentMemoryInstrumentation } from "#tracing/agent-memory-instrumentation.js";-import {- readGatewayCost,- setAgentInvocationUsage,- setAgentUsage,- setGenAiUsage,-} from "#tracing/agent-otel-usage.js";-import { createAgentOtelSessionContext } from "#tracing/agent-otel-session-context.js";-import {- rememberTurnInputMessages,- rememberTurnOutputMessages,-} from "#tracing/agent-otel-turn-state.js";-import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";-import { isSampledTrace, resolveTracePolicyDecision } from "#tracing/sampled-trace.js";-import {- applyLiveDeliveryAudienceCeiling,- resolveForwardedTraceSeed,-} from "#shared/forwarded-trace-policy.js";-import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";-import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";-import { suppressTracing } from "#tracing/suppress-tracing.js";-import { normalizeChannelAudience, type ChannelAudience } from "#shared/channel-audience.js";-import type {- InstrumentationEvent,- InstrumentationStepAttemptMetadataEvent,- InstrumentationAttemptScope,- InstrumentationStepAttemptStartedEvent,- InstrumentationStepAttemptTerminalEvent,- InstrumentationContextRunner,- InstrumentationModelCallTerminalEvent,- InstrumentationModelCallStartedEvent,- InstrumentationProviderDefinition,- InstrumentationSessionStartedEvent,- InstrumentationTraceSeed,- InstrumentationSessionTransitionEvent,- InstrumentationTurnStartedEvent,- InstrumentationTurnTerminalEvent,-} from "#instrumentation/lifecycle.js";-import { attemptIdempotencyKey } from "#instrumentation/lifecycle.js";-import {- AGENT_SPAN_NAMES,- agentInvocationSpanName,- modelSpanName,- type AgentSamplingOperation,-} from "#tracing/agent-span-contract.js";-import { withErrorContent } from "#tracing/error-content-context.js";-import { withAgentToolContentPolicy } from "#tracing/agent-tool-span-context.js";-import { withOperationConversation } from "#tracing/eve/conversation-context.js";-import { recordAgentSpanError as recordError } from "#tracing/agent-span-error.js";-import { resolveInstrumentationEnvironment } from "#internal/application/dev-environment.js";-import type { ConversationEnvironment } from "#shared/conversation-context.js";--type SpanState = { readonly context: Context; readonly span: Span };--export interface AgentOtelInstrumentationInput {- readonly environment?: ConversationEnvironment;- /** Whether any destination records model and tool inputs. */- readonly recordInputs?: boolean;- /** Whether any destination records model and tool outputs. */- readonly recordOutputs?: boolean;- readonly frameworkVersion: string;- readonly idGenerator: AgentSpanIdGenerator;- readonly samplesTrace?: (traceId: string, operation?: AgentSamplingOperation) => boolean;- readonly stateStore: AgentTraceStateStore;- readonly tracer: Tracer;- readonly tracePolicy?: TraceCapturePolicy;-}--/** OTel event definition and its trusted framework context runner. */-interface AgentOtelInstrumentation {- readonly hook: InstrumentationProviderDefinition;- readonly prepareSessionTrace: (- event: InstrumentationSessionStartedEvent,- ) => Promise<InstrumentationTraceSeed>;- readonly prepareTurnTrace: (- event: InstrumentationTurnStartedEvent,- ) => Promise<InstrumentationTraceSeed>;- readonly runInContext: InstrumentationContextRunner;-}--/** Creates OTel instrumentation for eve's structural and GenAI spans. */-export function createAgentOtelInstrumentation(- input: AgentOtelInstrumentationInput,-): AgentOtelInstrumentation {- const environment = input.environment ?? resolveInstrumentationEnvironment();- const recordInputs = input.recordInputs ?? false;- const recordOutputs = input.recordOutputs ?? false;- const executionContexts = new WeakMap<InstrumentationAttemptScope, Map<string, Context>>();- const attemptScopes = new Map<string, InstrumentationAttemptScope>();- // A lost serverless worker retries the whole turn step from entry.- const steps = new WeakMap<InstrumentationAttemptScope, SpanState>();- const modelSpans = new WeakMap<InstrumentationAttemptScope, Map<string, SpanState>>();- const tools = createAgentToolInstrumentation({- frameworkVersion: input.frameworkVersion,- idGenerator: input.idGenerator,- recordInputs,- recordOutputs,- resolveTraceContext: async (event) => {- const turn = await input.stateStore.getTurn(event.scope.sessionId, event.scope.turnId);- return turn?.context;- },- stateStore: input.stateStore,- resolveFallback: (event) => {- const scope = attemptScopes.get(event.scope.attemptId) ?? event.scope;- const step = steps.get(scope);- return step === undefined- ? undefined- : { context: step.context, spanContext: step.span.spanContext() };- },- tracer: input.tracer,- });- const approvals = createAgentApprovalInstrumentation({- actionContextFor: tools.dispatchContextFor,- frameworkVersion: input.frameworkVersion,- idGenerator: input.idGenerator,- tracer: input.tracer,- });- const memory = createAgentMemoryInstrumentation({ ...input, environment });- const { prepareSessionTrace, prepareTurnTrace } = createAgentOtelSessionContext({- ...input,- environment,- });-- const projectEvent = async (event: InstrumentationEvent): Promise<InstrumentationEvent> => {- const session = await input.stateStore.getSession(sessionIdForEvent(event));- const audience = audienceForEvent(event, session?.channelAudience);- const eventSeed = "traceSeed" in event ? event.traceSeed : undefined;- const contextSeed = contextStorage.getStore()?.get(SessionTraceSeedKey);- const contextTraceState = resolveForwardedTraceSeed(contextSeed);- const eventTraceState = resolveForwardedTraceSeed(- eventSeed,- contextTraceState?.forwardedTracePolicy,- );- const decisionForTrace = (trace: { readonly traceFlags: number } | undefined) =>- trace === undefined- ? undefined- : resolveTracePolicyDecision(isSampledTrace(trace), { audience, environment });- const decision =- eventTraceState?.decision ??- contextTraceState?.decision ??- readInstrumentationDecision(session?.decision) ??- decisionForTrace(eventSeed) ??- decisionForTrace(contextSeed) ??- decisionForTrace(session?.context);- if (decision === undefined) return withoutInstrumentationContent(event);- const normalizedEvent =- eventTraceState === undefined || !("traceSeed" in event) || event.traceSeed === undefined- ? event- : {- ...event,- traceSeed: {- ...event.traceSeed,- decision: eventTraceState.decision,- traceFlags: eventTraceState.traceFlags,- },- };- return instrumentationEventForTraceDecision(- normalizedEvent,- applyLiveDeliveryAudienceCeiling(- decision.action === "drop"- ? decision- : {- action: "record",- recordInputs: recordInputs && decision.recordInputs,- recordOutputs: recordOutputs && decision.recordOutputs,- },- audience,- eventTraceState?.forwardedTracePolicy ?? contextTraceState?.forwardedTracePolicy,- environment,- ),- { audience, environment },- { applyAudienceCeiling: false },- );- };-- const onSessionStarted = async (event: InstrumentationSessionStartedEvent): Promise<void> => {- await prepareSessionTrace(event);- };-- const onTurnStarted = async (event: InstrumentationTurnStartedEvent): Promise<void> => {- await prepareTurnTrace(event);- };-- const onStepStarted = async (event: InstrumentationStepAttemptStartedEvent): Promise<void> => {- const turn = await input.stateStore.getTurn(event.scope.sessionId, event.scope.turnId);- if (turn === undefined || !isSampledTrace(turn.context)) return;- const session = await input.stateStore.getSession(event.scope.sessionId);- const turnContext = withChannelAudience(- contextFromSpanContext(turn.context),- event.scope.channelAudience,- );- const activeSpanContext = trace.getSpan(context.active())?.spanContext();- const stepSpan = input.idGenerator.withSpanId(- input.idGenerator.deriveSpanId(attemptIdempotencyKey(event.scope)),- () =>- input.tracer.startSpan(- AGENT_SPAN_NAMES.step,- {- attributes: runtimeAttributes.agentStepAttributes({- event,- frameworkVersion: input.frameworkVersion,- session,- turn,- }),- links:- activeSpanContext === undefined || activeSpanContext.traceId === turn.context.traceId- ? undefined- : [- {- attributes: { "eve.link.type": "workflow.delivery" },- context: activeSpanContext,- },- ],- },- turnContext,- ),- );- stepSpan.addEvent("step.started");- const stepContext = trace.setSpan(turnContext, stepSpan);- steps.set(event.scope, { context: stepContext, span: stepSpan });- attemptScopes.set(event.scope.attemptId, event.scope);- };-- const onStepTerminal = async (event: InstrumentationStepAttemptTerminalEvent): Promise<void> => {- const scope = attemptScopes.get(event.scope.attemptId) ?? event.scope;- executionContexts.delete(scope);- drainOpenSpans({ ...event, scope });- tools.drain(- event.scope.attemptId,- event.type === "step.attempt.failed" ? { error: event.error } : undefined,- );- if (event.type === "step.attempt.failed") {- await tools.failForAttempt(scope, event.error);- }- attemptScopes.delete(event.scope.attemptId);- const attempt = steps.get(scope);- if (attempt === undefined) return;- attempt.span.addEvent(- event.type === "step.attempt.completed" ? "step.completed" : "step.failed",- );- if (event.type === "step.attempt.failed") {- recordError(attempt.span, event.error);- }- attempt.span.end();- steps.delete(scope);- };-- const onTurnTerminal = async (event: InstrumentationTurnTerminalEvent): Promise<void> => {- await input.stateStore.updateTurn(event.sessionId, event.turnId, (turn) => ({- ...turn,- terminal:- event.type === "turn.failed"- ? { error: event.error, type: event.type }- : { type: event.type },- }));- };-- const onSessionTransition = async (- event: InstrumentationSessionTransitionEvent,- ): Promise<void> => {- if (event.type === "session.failed" && event.turnId !== undefined) {- await input.stateStore.updateTurn(event.sessionId, event.turnId, (turn) => ({- ...turn,- terminal: turn.terminal ?? { error: event.error, type: "turn.failed" },- }));- }- if (event.turnId !== undefined) {- const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);- if (turn !== undefined) {- const session = await input.stateStore.getSession(event.sessionId);- if (isSampledTrace(turn.context)) {- const agentName = session?.agentName ?? turn.subagentName;- const parentContext = withChannelAudience(- turn.caller === undefined || turn.caller.isRemote === true- ? ROOT_CONTEXT- : contextFromSpanContext(turn.caller),- session?.channelAudience,- );- const startSpan = () =>- input.tracer.startSpan(- agentInvocationSpanName(agentName),- {- attributes: runtimeAttributes.agentActivationAttributes({- agentName,- frameworkVersion: input.frameworkVersion,- session,- sessionId: event.sessionId,- turnId: event.turnId!,- turn,- }),- kind: SpanKind.INTERNAL,- links: agentActivationLinks(turn),- root: turn.caller === undefined || turn.caller.isRemote === true,- startTime: turn.startTimeMs,- },- parentContext,- );- const span = input.idGenerator.withSpanId(turn.context.spanId, () =>- input.idGenerator.withTraceId(turn.context.traceId, startSpan),- );- setAgentInvocationUsage(span, turn.modelUsage);- span.addEvent("turn.started", undefined, turn.startTimeMs);- if (turn.terminal !== undefined) {- span.setAttribute(- "agent.turn.outcome",- turn.terminal.type === "turn.completed"- ? "completed"- : turn.terminal.type === "turn.cancelled"- ? "cancelled"- : "failed",- );- span.addEvent(turn.terminal.type);- if (turn.terminal.type === "turn.failed") {- recordError(span, turn.terminal.error);- }- }- span.end();- }- await input.stateStore.deleteTurn(event.sessionId, event.turnId);- }- }- // `session.waiting` is not terminal — the session may resume with a new- // turn that still needs its metadata — so only release session-scoped- // state on terminal transitions.- if (event.type === "session.completed" || event.type === "session.failed") {- await tools.deleteForSession(event.sessionId);- await input.stateStore.deleteSession(event.sessionId);- }- };-- const onModelCallStarted = (- event: InstrumentationModelCallStartedEvent,- ): void | PromiseLike<void> => {- const attempt = steps.get(event.scope);- if (attempt === undefined) return;- attempt.span.setAttribute("agent.model.id", event.model.modelId);- const span = input.tracer.startSpan(- modelSpanName(event.model.modelId),- {- attributes: {- "gen_ai.agent.name": event.scope.functionId,- "gen_ai.operation.name": "chat",- "gen_ai.provider.name": event.model.provider,- "gen_ai.request.model": event.model.modelId,- ...agentSpanNamingAttributes(modelSpanName(event.model.modelId), "chat"),- ...agentTraceIdentityAttributes({- rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,- traceSessionId: traceSessionIdOf(event.scope),- sessionId: event.scope.sessionId,- }),- ...runtimeAttributes.runtimeContextAttributes(event.runtimeContext),- },- kind: SpanKind.CLIENT,- },- attempt.context,- );- let inputMessagesAttribute: string | undefined;- if (recordInputs && event.input !== undefined) {- inputMessagesAttribute = genAiInputMessagesAttribute(event.input.messages);- if (inputMessagesAttribute !== undefined) {- span.setAttribute("gen_ai.input.messages", inputMessagesAttribute);- }- const genAiSystem = genAiSystemInstructionsAttribute(event.input.instructions);- if (genAiSystem !== undefined) {- span.setAttribute("gen_ai.system_instructions", genAiSystem);- }- const toolDefinitions = genAiToolDefinitionsAttribute(event.input.tools);- if (toolDefinitions !== undefined) {- span.setAttribute("gen_ai.tool.definitions", toolDefinitions);- }- }- const state = { context: trace.setSpan(attempt.context, span), span };- getExecutionContexts(event.scope).set(event.idempotencyKey, state.context);- getSpanStates(modelSpans, event.scope).set(event.idempotencyKey, state);- return rememberTurnInputMessages(input.stateStore, event.scope, inputMessagesAttribute);- };-- const onModelCallTerminal = async (- event: InstrumentationModelCallTerminalEvent,- ): Promise<void> => {- executionContexts.get(event.scope)?.delete(event.idempotencyKey);- const state = takeSpanState(modelSpans, event.scope, event.idempotencyKey);- if (state === undefined) return;- if (event.type === "model.call.failed") {- recordError(state.span, event.error);- } else {- await recordTurnUsage(event);- setGenAiUsage(state.span, event.usage);- if (event.responseId !== undefined) {- state.span.setAttribute("gen_ai.response.id", event.responseId);- }- if (event.responseModelId !== undefined) {- state.span.setAttribute("gen_ai.response.model", event.responseModelId);- }- state.span.setAttribute("gen_ai.response.finish_reasons", [event.finishReason]);- const attempt = steps.get(event.scope);- if (attempt !== undefined) setAgentUsage(attempt.span, event.usage);- if (recordOutputs) {- state.span.setAttribute("ai.response.finish_reason", event.finishReason);- const content = event.content;- if (content === undefined) {- state.span.end();- return;- }- const outputMessages = genAiOutputMessagesAttribute(content, event.finishReason);- if (outputMessages !== undefined) {- state.span.setAttribute("gen_ai.output.messages", outputMessages);- await rememberTurnOutputMessages(input.stateStore, event.scope, outputMessages);- }- const reasoning = textContentAttribute(- content- .filter((part) => part.type === "reasoning")- .map((part) => part.text)- .filter((part) => part.trim().length > 0)- .join("\n"),- );- if (reasoning !== undefined) state.span.setAttribute("ai.response.reasoning", reasoning);- const text = textContentAttribute(- content- .filter((part) => part.type === "text")- .map((part) => part.text)- .join(""),- );- if (text !== undefined) state.span.setAttribute("ai.response.text", text);- const toolCalls = content- .filter((part) => part.type === "tool-call")- .map((part) => ({ callId: part.callId, input: part.input, toolName: part.toolName }));- if (toolCalls.length > 0) {- const json = contentAttribute(toolCalls);- if (json !== undefined) state.span.setAttribute("ai.response.tool_calls", json);- }- // Provider-executed tools (e.g. web_search) run inside the model call,- // never reach eve's tool loop, and so never get an execute_tool span.- // Their results only exist as content parts on the model response.- const toolResults = content- .filter((part) => part.type === "tool-result" || part.type === "tool-error")- .map((part) =>- part.type === "tool-result"- ? {- callId: part.callId,- input: part.input,- output: part.output,- toolName: part.toolName,- }- : {- callId: part.callId,- error: errorText(part.error),- input: part.input,- toolName: part.toolName,- },- );- if (toolResults.length > 0) {- const json = toolResultsContentAttribute(toolResults);- if (json !== undefined) state.span.setAttribute("ai.response.tool_results", json);- }- }- }- state.span.end();- };-- const recordTurnUsage = async (- event: Extract<- InstrumentationModelCallTerminalEvent,- { readonly type: "model.call.completed" }- >,- ): Promise<void> => {- if (event.usage.inputTokens === undefined && event.usage.outputTokens === undefined) return;- // The bridge publishes at most one completion per physical execution.- // Workflow retries restart from pre-step state, so abandoned additions are- // not merged; distinct completed retries consumed tokens and count here.- await input.stateStore.updateTurn(event.scope.sessionId, event.scope.turnId, (turn) => ({- ...turn,- modelUsage: {- inputTokens:- event.usage.inputTokens === undefined- ? turn.modelUsage?.inputTokens- : (turn.modelUsage?.inputTokens ?? 0) + event.usage.inputTokens,- outputTokens:- event.usage.outputTokens === undefined- ? turn.modelUsage?.outputTokens- : (turn.modelUsage?.outputTokens ?? 0) + event.usage.outputTokens,- },- }));- };-- const channelDeliveries = createAgentChannelDeliveryInstrumentation({- recordInputs,- stateStore: input.stateStore,- });-- const onStepMetadata = (event: InstrumentationStepAttemptMetadataEvent): void => {- const attempt = steps.get(event.scope);- if (attempt === undefined) return;- // Vercel AI Gateway reports per-call cost in providerMetadata.gateway;- // attributes exist only when it was actually the gateway serving the call.- const costAttributes = readGatewayCost(event.providerMetadata);- if (costAttributes === undefined) return;- // The vendored OTel Span surface only has singular setAttribute.- for (const [key, value] of Object.entries(costAttributes)) {- attempt.span.setAttribute(key, value);- }- };-- return {- hook: {- events: {- ...channelDeliveries,- ...approvals,- ...memory.events,- "step.attempt.completed": onStepTerminal,- "step.attempt.failed": onStepTerminal,- "step.attempt.metadata": onStepMetadata,- "step.attempt.started": onStepStarted,- "model.call.completed": onModelCallTerminal,- "model.call.failed": onModelCallTerminal,- "model.call.started": onModelCallStarted,- "session.completed": onSessionTransition,- "session.failed": onSessionTransition,- "session.started": onSessionStarted,- "session.waiting": onSessionTransition,- ...tools.events,- "turn.cancelled": onTurnTerminal,- "turn.completed": onTurnTerminal,- "turn.failed": onTurnTerminal,- "turn.started": onTurnStarted,- },- name: "eve.otel",- projectEvent,- tracePolicy: () => ({ emit: true, recordInputs, recordOutputs }),- },- prepareSessionTrace,- prepareTurnTrace,- async runInContext(operation, execute) {- if (operation.type === "memory.operation") return memory.runInContext(operation, execute);- const scope = attemptScopes.get(operation.scope.attemptId) ?? operation.scope;- const contexts = executionContexts.get(scope);- let parent =- operation.type === "model.call"- ? contexts?.get(operation.idempotencyKey)- : tools.contextFor(operation.scope.attemptId, operation.idempotencyKey);- if (parent === undefined) {- const turn = await input.stateStore.getTurn(- operation.scope.sessionId,- operation.scope.turnId,- );- if (turn !== undefined) {- parent = withChannelAudience(- contextFromSpanContext(turn.context),- operation.scope.channelAudience,- );- if (!isSampledTrace(turn.context)) parent = suppressTracing(parent);- }- }- const session = await input.stateStore.getSession(operation.scope.sessionId);- const seed = resolveForwardedTraceSeed(contextStorage.getStore()?.get(SessionTraceSeedKey));- const decision = seed?.decision ?? session?.decision;- const effective =- decision === undefined- ? undefined- : applyLiveDeliveryAudienceCeiling(- decision,- normalizeChannelAudience(operation.scope.channelAudience),- seed?.forwardedTracePolicy,- environment,- );- const toolContentPolicy = {- recordInputs: recordInputs && effective?.action === "record" && effective.recordInputs,- recordOutputs: recordOutputs && effective?.action === "record" && effective.recordOutputs,- };- const run = (executionParent: Context | undefined) => {- const activeParent = executionParent ?? parent;- if (activeParent === undefined) return execute();- const withErrorPolicy = withErrorContent(activeParent, toolContentPolicy.recordOutputs);- const operationContext =- operation.type === "tool.call"- ? withAgentToolContentPolicy(withErrorPolicy, toolContentPolicy)- : withErrorPolicy;- return context.with(- markAgentTraceContext(withOperationConversation(operationContext, operation.scope)),- execute,- );- };- return operation.type === "tool.call"- ? tools.runInContext(- { ...operation, input: toolContentPolicy.recordInputs ? operation.input : undefined },- run,- toolContentPolicy.recordOutputs,- )- : run(undefined);- },- };-- function getExecutionContexts(scope: InstrumentationAttemptScope): Map<string, Context> {- let state = executionContexts.get(scope);- if (state === undefined) {- state = new Map();- executionContexts.set(scope, state);- }- return state;- }-- function drainOpenSpans(event: InstrumentationStepAttemptTerminalEvent): void {- for (const state of modelSpans.get(event.scope)?.values() ?? []) {- if (event.type === "step.attempt.failed") recordError(state.span, event.error);- state.span.end();- }- modelSpans.delete(event.scope);- }-}--function agentActivationLinks(turn: AgentTurnTraceState): Link[] | undefined {- const links: Link[] = [];- if (turn.caller?.isRemote === true) {- links.push({ context: turn.caller, attributes: { "eve.link.type": "agent.dispatch" } });- }- if (turn.channelDelivery?.requestTraceContext !== undefined) {- links.push({- context: turn.channelDelivery.requestTraceContext,- attributes: { "eve.link.type": "channel.request" },- });- }- return links.length === 0 ? undefined : links;-}--function sessionIdForEvent(event: InstrumentationEvent): string {- return "scope" in event ? event.scope.sessionId : event.sessionId;-}--function audienceForEvent(- event: InstrumentationEvent,- sessionAudience: ChannelAudience | undefined,-): ChannelAudience {- if ("delivery" in event) return normalizeChannelAudience(event.delivery.channelAudience);- if ("scope" in event && event.scope.channelAudience !== undefined) {- return normalizeChannelAudience(event.scope.channelAudience);- }- if (event.type === "session.started") {- return normalizeChannelAudience(event.channelAudience);- }- return normalizeChannelAudience(sessionAudience);-}--function getSpanStates<T>(- spans: WeakMap<InstrumentationAttemptScope, Map<string, T>>,- scope: InstrumentationAttemptScope,-): Map<string, T> {- let scoped = spans.get(scope);- if (scoped === undefined) {- scoped = new Map();- spans.set(scope, scoped);- }- return scoped;-}--function takeSpanState<T>(- spans: WeakMap<InstrumentationAttemptScope, Map<string, T>>,- scope: InstrumentationAttemptScope,- id: string,-): T | undefined {- const scoped = spans.get(scope);- const state = scoped?.get(id);- if (scoped === undefined) return undefined;- scoped.delete(id);- if (scoped.size === 0) spans.delete(scope);- return state;-}--function contextFromSpanContext(spanContext: SpanContext): Context {- return trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(spanContext));-}--const errorText = (error: unknown): unknown => (error instanceof Error ? error.message : error);diff --git a/packages/eve/src/tracing/agent-otel-runtime-context.test.ts b/packages/eve/src/tracing/agent-otel-runtime-context.test.tsdeleted file mode 100644index 6c6a93b3b..000000000--- a/packages/eve/src/tracing/agent-otel-runtime-context.test.ts+++ /dev/null@@ -1,187 +0,0 @@-import { describe, expect, it } from "vitest";--import { agentActivationAttributes } from "#tracing/agent-otel-runtime-context.js";-import type { AgentSessionTraceState, AgentTurnTraceState } from "#tracing/agent-trace-state.js";--const context = {- spanId: "1".repeat(16),- traceFlags: 1,- traceId: "2".repeat(32),-};--function turn(overrides: Partial<AgentTurnTraceState> = {}): AgentTurnTraceState {- return {- context,- rootSessionId: "root-session",- traceSessionId: "root-session",- sequence: 0,- startTimeMs: 1,- ...overrides,- };-}--function session(overrides: Partial<AgentSessionTraceState> = {}): AgentSessionTraceState {- return {- agentName: "general",- channelAudience: "public",- channelKind: "slack",- context,- decision: { action: "record", recordInputs: true, recordOutputs: true },- rootSessionId: "root-session",- traceSessionId: "root-session",- title: "Call the general agent",- ...overrides,- };-}--describe("agentActivationAttributes", () => {- it.each([- {- name: "root",- sessionId: "root-session",- traceSessionId: "root-session",- parent: undefined,- ownsMetadata: true,- },- {- name: "remote",- sessionId: "remote-session",- traceSessionId: "remote-session",- parent: "caller",- ownsMetadata: true,- },- {- name: "remote local child",- sessionId: "child",- traceSessionId: "remote-session",- parent: "remote-session",- ownsMetadata: false,- },- {- name: "local child",- sessionId: "child",- traceSessionId: "root-session",- parent: "root-session",- ownsMetadata: false,- },- ])(- "derives $name metadata ownership from explicit session identity",- ({ sessionId, traceSessionId, parent, ownsMetadata }) => {- const attributes = agentActivationAttributes({- frameworkVersion: "test",- session: session({ traceSessionId, channelKind: "http", title: "Task" }),- sessionId,- turnId: "turn_0",- turn: turn({- traceSessionId,- parentLineage:- parent === undefined- ? undefined- : { sessionId: parent, callId: "call", turnId: "turn_0" },- }),- });- expect(attributes["agent.channel.kind"]).toBe(ownsMetadata ? "http" : undefined);- expect(attributes["agent.session.origin"]).toBe(ownsMetadata ? "channel" : undefined);- expect(attributes["agent.session.title"]).toBe(ownsMetadata ? "Task" : undefined);- expect(attributes["agent.run.type"]).toBe(parent === undefined ? "session" : "subagent");- expect(attributes["agent.parent_run.id"]).toBe(parent);- expect(attributes["agent.parent_call.id"]).toBe(parent === undefined ? undefined : "call");- },- );- it("emits root-session inventory and trace-policy metadata", () => {- expect(- agentActivationAttributes({- agentName: "general",- frameworkVersion: "test",- session: session(),- sessionId: "root-session",- turn: turn(),- turnId: "turn_0",- }),- ).toMatchObject({- "agent.channel.audience": "public",- "agent.channel.kind": "slack",- "agent.run.type": "session",- "agent.session.origin": "channel",- "agent.session.title": "Call the general agent",- "agent.trace.content.input": true,- "agent.trace.content.output": true,- });- });-- it("keeps root-only provenance and title off delegated activations", () => {- const attributes = agentActivationAttributes({- agentName: "general",- frameworkVersion: "test",- session: session({- channelAudience: "private",- decision: { action: "record", recordInputs: false, recordOutputs: true },- scheduleId: "daily-report",- }),- sessionId: "child-session",- turn: turn({- parentLineage: {- callId: "call-1",- sessionId: "parent-session",- subagentName: "general",- turnId: "turn_0",- },- }),- turnId: "turn_0",- });-- expect(attributes).toMatchObject({- "agent.channel.audience": "private",- "agent.run.type": "subagent",- "agent.trace.content.input": false,- "agent.trace.content.output": true,- });- expect(attributes["agent.channel.kind"]).toBeUndefined();- expect(attributes["agent.schedule.id"]).toBeUndefined();- expect(attributes["agent.session.origin"]).toBeUndefined();- expect(attributes["agent.session.title"]).toBeUndefined();- });-- it("normalizes a raw channel type when the normalized kind is absent", () => {- const attributes = agentActivationAttributes({- agentName: "general",- frameworkVersion: "test",- session: session({ channelKind: undefined, channelType: "slack" }),- sessionId: "root-session",- turn: turn(),- turnId: "turn_0",- });-- expect(attributes["agent.channel.kind"]).toBe("unknown");- });-- it("keeps actual channel delivery metadata on delegated activations", () => {- const attributes = agentActivationAttributes({- agentName: "general",- frameworkVersion: "test",- session: session(),- sessionId: "child-session",- turn: turn({- channelDelivery: {- channelKind: "channel:slack",- channelName: "slack",- deliveryId: "delivery-1",- },- parentLineage: {- callId: "call-1",- sessionId: "parent-session",- subagentName: "general",- turnId: "turn_0",- },- }),- turnId: "turn_1",- });-- expect(attributes).toMatchObject({- "agent.channel.delivery.id": "delivery-1",- "agent.channel.kind": "channel:slack",- "agent.channel.name": "slack",- "agent.run.type": "subagent",- });- });-});diff --git a/packages/eve/src/tracing/agent-otel-runtime-context.ts b/packages/eve/src/tracing/agent-otel-runtime-context.tsdeleted file mode 100644index 40055a224..000000000--- a/packages/eve/src/tracing/agent-otel-runtime-context.ts+++ /dev/null@@ -1,183 +0,0 @@-import type { AgentSessionTraceState, AgentTurnTraceState } from "#tracing/agent-trace-state.js";-import type { InstrumentationStepAttemptStartedEvent } from "#instrumentation/lifecycle.js";-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";-import { agentInvocationSpanName } from "#tracing/agent-span-contract.js";-import { agentTraceIdentityAttributes, traceSessionIdOf } from "#tracing/agent-otel-attributes.js";-import { normalizeInstrumentationChannelKind } from "#internal/instrumentation.js";--type SpanAttributePrimitive = string | number | boolean;-type SpanAttributeValue = SpanAttributePrimitive | SpanAttributePrimitive[];--export function agentActivationAttributes(input: {- readonly agentName?: string;- readonly frameworkVersion: string;- readonly session?: AgentSessionTraceState;- readonly sessionId: string;- readonly turnId: string;- readonly turn: AgentTurnTraceState;-}): Record<string, string | number | boolean | undefined> {- const recordsTrace = input.session?.decision?.action === "record";- const recordsInputs = recordsTrace && input.session?.decision?.recordInputs === true;- const recordsOutputs = recordsTrace && input.session?.decision?.recordOutputs === true;- const parentLineage = input.turn.parentLineage ?? input.session?.parentLineage;- const isSubagent = parentLineage !== undefined;- const ownsSessionMetadata = !isSubagent || input.turn.traceSessionId === input.sessionId;- const channelClassification = agentChannelClassificationAttributes(- input.session,- input.turn,- input.sessionId,- );- const scheduleId = isSubagent ? undefined : input.session?.scheduleId;- return {- "agent.framework.name": "eve",- "agent.framework.version": input.frameworkVersion,- "agent.name": input.agentName,- "agent.channel.audience": input.session?.channelAudience,- ...agentPrincipalAttributes(input.turn),- "agent.channel.delivery.id": input.turn.channelDelivery?.deliveryId,- "agent.channel.delivery.input": input.turn.channelDelivery?.inputAttribute,- ...channelClassification,- "agent.channel.name": input.turn.channelDelivery?.channelName,- "agent.channel.request.id": input.turn.channelDelivery?.requestId,- "agent.parent_call.id": parentLineage?.callId,- "agent.parent_run.id": parentLineage?.sessionId,- "agent.run.type": isSubagent ? "subagent" : "session",- "agent.schedule.id": scheduleId,- "agent.session.title": ownsSessionMetadata && recordsInputs ? input.session?.title : undefined,- "agent.subagent.name": input.turn.subagentName,- "agent.trace.content.input": recordsInputs,- "agent.trace.content.output": recordsOutputs,- "agent.turn.id": input.turnId,- "agent.turn.sequence": input.turn.sequence,- "gen_ai.agent.name": input.agentName,- "gen_ai.input.messages": input.turn.inputMessagesAttribute,- "gen_ai.operation.name": "invoke_agent",- "gen_ai.output.messages": input.turn.outputMessagesAttribute,- ...agentSpanNamingAttributes(agentInvocationSpanName(input.agentName), "invoke_agent"),- ...agentTraceIdentityAttributes({- rootSessionId: input.turn.rootSessionId,- traceSessionId: input.turn.traceSessionId,- sessionId: input.sessionId,- }),- };-}--export function agentChannelClassificationAttributes(- session: AgentSessionTraceState | undefined,- turn: AgentTurnTraceState,- sessionId: string,-): {- readonly "agent.channel.kind": string | undefined;- readonly "agent.session.origin": string | undefined;-} {- const isSubagent = (turn.parentLineage ?? session?.parentLineage) !== undefined;- const ownsSessionMetadata = !isSubagent || turn.traceSessionId === sessionId;- const channelKind =- turn.channelDelivery?.channelKind ??- (!ownsSessionMetadata- ? undefined- : (session?.channelKind ??- (session?.channelType === undefined- ? undefined- : normalizeInstrumentationChannelKind(session.channelType))));- const origin =- !ownsSessionMetadata || channelKind === undefined- ? undefined- : session?.scheduleId !== undefined- ? "schedule"- : "channel";- return {- "agent.channel.kind": channelKind,- "agent.session.origin": origin,- };-}--export function agentStepAttributes(input: {- readonly event: InstrumentationStepAttemptStartedEvent;- readonly frameworkVersion: string;- readonly session?: AgentSessionTraceState;- readonly turn: AgentTurnTraceState;-}) {- const { event, session, turn } = input;- const channelClassification = agentChannelClassificationAttributes(- session,- turn,- event.scope.sessionId,- );- return {- "agent.framework.name": "eve",- "agent.framework.version": input.frameworkVersion,- "agent.step.attempt": event.scope.attemptIndex,- "agent.step.index": event.scope.stepIndex,- "agent.turn.id": event.scope.turnId,- "agent.name": event.scope.functionId,- // Leave an unresolved kind open for a later delivery to classify.- ...(channelClassification["agent.channel.kind"] === "unknown"- ? undefined- : channelClassification),- "gen_ai.operation.name": "workflow",- ...agentSpanNamingAttributes("agent.step", "workflow"),- ...agentTraceIdentityAttributes({- rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,- traceSessionId: traceSessionIdOf(event.scope),- sessionId: event.scope.sessionId,- }),- ...runtimeContextAttributes(event.runtimeContext),- };-}--export function agentPrincipalAttributes(turn: AgentTurnTraceState): Record<string, string> {- const attributes: Record<string, string> = {};- setOptionalAttribute(attributes, "agent.principal.current.id", turn.currentPrincipal?.id);- setOptionalAttribute(attributes, "agent.principal.current.type", turn.currentPrincipal?.type);- setOptionalAttribute(attributes, "agent.principal.initiator.id", turn.initiatorPrincipal?.id);- setOptionalAttribute(attributes, "agent.principal.initiator.type", turn.initiatorPrincipal?.type);- return attributes;-}--/** Flattens merged runtime context into AI SDK-compatible span attributes. */-export function runtimeContextAttributes(- runtimeContext: Readonly<Record<string, unknown>> | undefined,-): Record<string, SpanAttributeValue> {- const attributes: Record<string, SpanAttributeValue> = {};- if (runtimeContext === undefined) return attributes;- for (const [key, value] of Object.entries(runtimeContext)) {- flattenContextAttribute(attributes, `ai.settings.context.${key}`, value);- }- return attributes;-}--function flattenContextAttribute(- attributes: Record<string, SpanAttributeValue>,- key: string,- value: unknown,-): void {- if (value == null) return;- if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {- attributes[key] = value;- return;- }- if (Array.isArray(value)) {- const primitives = value.filter(- (entry): entry is SpanAttributePrimitive =>- typeof entry === "string" || typeof entry === "number" || typeof entry === "boolean",- );- if (primitives.length !== value.length) return;- if (new Set(primitives.map((entry) => typeof entry)).size !== 1) return;- attributes[key] = primitives;- return;- }- if (typeof value === "object") {- for (const [nestedKey, nestedValue] of Object.entries(value)) {- flattenContextAttribute(attributes, `${key}.${nestedKey}`, nestedValue);- }- }-}--function setOptionalAttribute(- attributes: Record<string, string>,- key: string,- value: string | undefined,-): void {- if (value !== undefined) attributes[key] = value;-}diff --git a/packages/eve/src/tracing/agent-otel-session-context.ts b/packages/eve/src/tracing/agent-otel-session-context.tsdeleted file mode 100644index 8a360a78c..000000000--- a/packages/eve/src/tracing/agent-otel-session-context.ts+++ /dev/null@@ -1,245 +0,0 @@-import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";--import {- type InstrumentationSessionStartedEvent,- type InstrumentationTraceContext,- type InstrumentationTraceSeed,- type InstrumentationTurnStartedEvent,-} from "#instrumentation/lifecycle.js";-import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";-import { normalizeChannelAudience } from "#shared/channel-audience.js";-import type { ChannelAudience } from "#shared/channel-audience.js";-import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";-import {- isSampledTrace,- resolveTracePolicy,- resolveTracePolicyDecision,-} from "#tracing/sampled-trace.js";-import type { AgentSessionTraceState, AgentTraceStateStore } from "#tracing/agent-trace-state.js";-import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";-import {- agentInvocationSpanName,- type AgentSamplingOperation,-} from "#tracing/agent-span-contract.js";-import { agentActivationAttributes } from "#tracing/agent-otel-runtime-context.js";-import type { AgentTurnTraceState } from "#tracing/agent-trace-state.js";-import { applyPrincipalTraceDecision } from "#instrumentation/principal-summary.js";-import { normalizeInstrumentationChannelKind } from "#internal/instrumentation.js";-import type { ConversationEnvironment } from "#shared/conversation-context.js";-import { traceSessionIdOf } from "#tracing/agent-otel-attributes.js";--interface AgentOtelSessionContextInput {- readonly environment: ConversationEnvironment;- readonly frameworkVersion: string;- readonly idGenerator: AgentSpanIdGenerator;- readonly samplesTrace?: (traceId: string, operation?: AgentSamplingOperation) => boolean;- readonly stateStore: AgentTraceStateStore;- readonly tracePolicy?: TraceCapturePolicy;-}--type SessionMetadata = Omit<InstrumentationSessionStartedEvent, "idempotencyKey" | "type">;--interface AgentOtelSessionContext {- readonly ensureSessionContext: (event: SessionMetadata) => Promise<AgentSessionTraceState>;- readonly prepareSessionTrace: (- event: InstrumentationSessionStartedEvent,- ) => Promise<InstrumentationTraceSeed>;- readonly prepareTurnTrace: (- event: InstrumentationTurnStartedEvent,- ) => Promise<InstrumentationTraceSeed>;-}--export function createAgentOtelSessionContext(- input: AgentOtelSessionContextInput,-): AgentOtelSessionContext {- const ensureSessionContext = async (event: SessionMetadata): Promise<AgentSessionTraceState> => {- let state = await input.stateStore.getSession(event.sessionId);- if (state === undefined) {- const channelAudience = normalizeChannelAudience(event.channelAudience);- const decision = resolveSessionTraceDecision(- event,- channelAudience,- input.environment,- input.tracePolicy,- );- state = {- agentName: event.agentName,- channelAudience,- channelKind: event.channelKind,- channelType: event.channelType,- decision,- context: initialSessionContext(input, event, decision),- parentLineage: event.parentLineage,- rootSessionId: event.rootSessionId,- traceSessionId: traceSessionIdOf(event),- scheduleId: event.scheduleId,- title: event.title,- };- await input.stateStore.setSession(event.sessionId, state);- }- return state;- };-- const prepareSessionTrace = async (- event: InstrumentationSessionStartedEvent,- ): Promise<InstrumentationTraceSeed> => {- const session = await ensureSessionContext(event);- return portableSpanContext(session.context, session.decision);- };-- const prepareTurnTrace = async (- event: InstrumentationTurnStartedEvent,- ): Promise<InstrumentationTraceSeed> => {- const prepared = await input.stateStore.getTurn(event.sessionId, event.turnId);- if (prepared !== undefined) {- const session = await input.stateStore.getSession(event.sessionId);- return portableSpanContext(prepared.context, session?.decision);- }-- const session = await ensureSessionContext(event);- const useInitialContext = event.sequence === 0;- const caller = useInitialContext ? event.parentTraceContext : undefined;- let turnContext = useInitialContext- ? { ...session.context, isRemote: false }- : freshTurnContext(input, event.idempotencyKey, session.decision);- if (caller !== undefined && !("isRemote" in caller && caller.isRemote === true))- turnContext = { ...turnContext, traceId: caller.traceId };- const turn: AgentTurnTraceState = {- caller: caller === undefined ? undefined : adoptedSpanContext(caller),- context: turnContext,- currentPrincipal: applyPrincipalTraceDecision(event.currentPrincipal, session.decision),- initiatorPrincipal: applyPrincipalTraceDecision(event.initiatorPrincipal, session.decision),- parentLineage: event.parentLineage ?? session.parentLineage,- rootSessionId: event.rootSessionId,- traceSessionId: traceSessionIdOf(event),- sequence: event.sequence,- startTimeMs: Date.now(),- subagentName: (event.parentLineage ?? session.parentLineage)?.subagentName,- };- if (- isSampledTrace(turn.context) &&- (caller === undefined || ("isRemote" in caller && caller.isRemote === true))- ) {- const agentName = session.agentName ?? turn.subagentName;- const sampled =- input.samplesTrace?.(turn.context.traceId, {- name: agentInvocationSpanName(agentName),- attributes: agentActivationAttributes({- agentName,- frameworkVersion: input.frameworkVersion,- session,- sessionId: event.sessionId,- turnId: event.turnId,- turn,- }),- }) ?? true;- turnContext = { ...turnContext, traceFlags: sampled ? 1 : 0 };- }- await input.stateStore.setTurn(event.sessionId, event.turnId, {- ...turn,- context: turnContext,- });- return portableSpanContext(turnContext, session.decision);- };-- return { ensureSessionContext, prepareSessionTrace, prepareTurnTrace };-}--function portableSpanContext(- spanContext: SpanContext,- decision?: InstrumentationTraceSeed["decision"],-): InstrumentationTraceSeed {- return {- decision,- spanId: spanContext.spanId,- traceFlags: spanContext.traceFlags,- traceId: spanContext.traceId,- };-}--function adoptedSpanContext(handed: InstrumentationTraceContext): SpanContext {- return {- isRemote: "isRemote" in handed && handed.isRemote === true,- spanId: handed.spanId,- traceFlags: handed.traceFlags,- traceId: handed.traceId,- };-}--function initialSessionContext(- input: AgentOtelSessionContextInput,- event: SessionMetadata,- decision: ReturnType<typeof resolveTracePolicy>,-): SpanContext {- const handed = event.traceSeed;- if (handed !== undefined) {- return {- ...adoptedSpanContext(handed),- traceFlags: decision.action === "drop" ? 0 : handed.traceFlags,- };- }- const traceId =- (event.parentTraceContext !== undefined &&- "isRemote" in event.parentTraceContext &&- event.parentTraceContext.isRemote === true- ? undefined- : event.parentTraceContext?.traceId) ??- input.idGenerator.deriveTraceId(`session:${event.sessionId}`);- const sampled = decision.action === "record";- return {- isRemote: false,- spanId: input.idGenerator.deriveSpanId(`session:${event.sessionId}`),- traceFlags: sampled ? 1 : 0,- traceId,- };-}--function freshTurnContext(- input: AgentOtelSessionContextInput,- idempotencyKey: string,- decision: AgentSessionTraceState["decision"],-): SpanContext {- const traceId = input.idGenerator.deriveTraceId(`turn:${idempotencyKey}`);- const sampled = decision?.action === "record";- return {- isRemote: false,- spanId: input.idGenerator.deriveSpanId(`turn:${idempotencyKey}`),- traceFlags: sampled ? 1 : 0,- traceId,- };-}--function resolveSessionTraceDecision(- event: SessionMetadata,- audience: ChannelAudience,- environment: ConversationEnvironment,- policy: TraceCapturePolicy | undefined,-): ReturnType<typeof resolveTracePolicy> {- const content = { audience, environment };- if (event.parentTraceContext !== undefined && !isSampledTrace(event.parentTraceContext)) {- return { action: "drop" };- }- if (event.traceSeed?.decision !== undefined) {- return readInstrumentationDecision(event.traceSeed.decision) ?? { action: "drop" };- }- if (event.traceSeed !== undefined) {- return resolveTracePolicyDecision(isSampledTrace(event.traceSeed), content);- }- if (event.parentTraceContext !== undefined) {- return resolveTracePolicyDecision(isSampledTrace(event.parentTraceContext), content);- }- if (event.agentName === undefined) {- return policy === undefined ? resolveTracePolicyDecision(true, content) : { action: "drop" };- }- // The tool loop can evaluate the same policy before this first-session- // preparation path; the persisted decision removes that window on replay.- return resolveTracePolicy(policy, {- agentName: event.agentName,- audience,- channel: {- kind: normalizeInstrumentationChannelKind(event.channelKind ?? event.channelType),- },- environment,- principalType: "unknown",- });-}diff --git a/packages/eve/src/tracing/agent-otel-turn-state.test.ts b/packages/eve/src/tracing/agent-otel-turn-state.test.tsdeleted file mode 100644index b293d33bd..000000000--- a/packages/eve/src/tracing/agent-otel-turn-state.test.ts+++ /dev/null@@ -1,46 +0,0 @@-import { describe, expect, it } from "vitest";--import {- rememberTurnInputMessages,- rememberTurnOutputMessages,-} from "#tracing/agent-otel-turn-state.js";-import { InMemoryAgentTraceStateStore } from "#internal/testing/in-memory-agent-trace-state-store.js";--describe("agent OTel turn state", () => {- it("keeps the first model input and latest model output across a multi-call turn", async () => {- const store = new InMemoryAgentTraceStateStore();- store.setTurn("session-1", "turn-1", {- context: spanContext(),- rootSessionId: "session-1",- traceSessionId: "session-1",- sequence: 0,- startTimeMs: 1,- });- const scope = {- attemptId: "attempt-1",- attemptIndex: 0,- functionId: "agent",- sessionId: "session-1",- stepIndex: 0,- turnId: "turn-1",- };-- await rememberTurnInputMessages(store, scope, "first input");- await rememberTurnInputMessages(store, scope, "second input");- await rememberTurnOutputMessages(store, scope, "first output");- await rememberTurnOutputMessages(store, scope, "final output");-- expect(store.getTurn("session-1", "turn-1")).toMatchObject({- inputMessagesAttribute: "first input",- outputMessagesAttribute: "final output",- });- });-});--function spanContext() {- return {- spanId: "2222222222222222",- traceFlags: 1,- traceId: "11111111111111111111111111111111",- };-}diff --git a/packages/eve/src/tracing/agent-otel-turn-state.ts b/packages/eve/src/tracing/agent-otel-turn-state.tsdeleted file mode 100644index e799fea1c..000000000--- a/packages/eve/src/tracing/agent-otel-turn-state.ts+++ /dev/null@@ -1,28 +0,0 @@-import type { InstrumentationAttemptScope } from "#instrumentation/lifecycle.js";-import type { AgentTraceStateStore } from "#tracing/agent-trace-state.js";--/** Keeps the first model input so the turn-level invoke_agent span represents its task. */-export function rememberTurnInputMessages(- store: AgentTraceStateStore,- scope: InstrumentationAttemptScope,- inputMessagesAttribute: string | undefined,-): void | PromiseLike<void> {- if (inputMessagesAttribute === undefined) return;- return store.updateTurn(scope.sessionId, scope.turnId, (turn) => ({- ...turn,- inputMessagesAttribute: turn.inputMessagesAttribute ?? inputMessagesAttribute,- }));-}--/** Keeps the latest model output so the completed invoke_agent span represents its result. */-export function rememberTurnOutputMessages(- store: AgentTraceStateStore,- scope: InstrumentationAttemptScope,- outputMessagesAttribute: string | undefined,-): void | PromiseLike<void> {- if (outputMessagesAttribute === undefined) return;- return store.updateTurn(scope.sessionId, scope.turnId, (turn) => ({- ...turn,- outputMessagesAttribute,- }));-}diff --git a/packages/eve/src/tracing/agent-otel-usage.ts b/packages/eve/src/tracing/agent-otel-usage.tsdeleted file mode 100644index 1c136dacc..000000000--- a/packages/eve/src/tracing/agent-otel-usage.ts+++ /dev/null@@ -1,84 +0,0 @@-import type { Span } from "#compiled/@opentelemetry/api/index.js";--import type { InstrumentationUsage } from "#instrumentation/lifecycle.js";-import type { AgentTurnTraceState } from "#tracing/agent-trace-state.js";-import { AGENT_USAGE_ATTRIBUTES } from "#tracing/agent-span-contract.js";--/** Applies eve's structural token usage attributes to an agent span. */-export function setAgentUsage(span: Span, usage: InstrumentationUsage): void {- if (usage.costUsd !== undefined) {- span.setAttribute(AGENT_USAGE_ATTRIBUTES.costUsd, usage.costUsd);- }- if (usage.inputTokens !== undefined) {- span.setAttribute(AGENT_USAGE_ATTRIBUTES.inputTokens, usage.inputTokens);- }- if (usage.outputTokens !== undefined) {- span.setAttribute(AGENT_USAGE_ATTRIBUTES.outputTokens, usage.outputTokens);- }- const details = usage.inputTokenDetails;- if (details?.cacheReadTokens !== undefined) {- span.setAttribute(AGENT_USAGE_ATTRIBUTES.cacheReadTokens, details.cacheReadTokens);- }- if (details?.cacheWriteTokens !== undefined) {- span.setAttribute(AGENT_USAGE_ATTRIBUTES.cacheWriteTokens, details.cacheWriteTokens);- }-}--/** Applies standard GenAI token usage and retains eve's cost attribute. */-export function setGenAiUsage(span: Span, usage: InstrumentationUsage): void {- if (usage.costUsd !== undefined) {- span.setAttribute(AGENT_USAGE_ATTRIBUTES.costUsd, usage.costUsd);- }- if (usage.inputTokens !== undefined) {- span.setAttribute("gen_ai.usage.input_tokens", usage.inputTokens);- }- if (usage.outputTokens !== undefined) {- span.setAttribute("gen_ai.usage.output_tokens", usage.outputTokens);- }- const details = usage.inputTokenDetails;- if (details?.cacheReadTokens !== undefined) {- span.setAttribute("gen_ai.usage.cache_read.input_tokens", details.cacheReadTokens);- }- if (details?.cacheWriteTokens !== undefined) {- span.setAttribute("gen_ai.usage.cache_write.input_tokens", details.cacheWriteTokens);- }-}--export function setAgentInvocationUsage(- span: Span,- modelUsage: AgentTurnTraceState["modelUsage"],-): void {- if (modelUsage === undefined) return;- setGenAiUsage(span, modelUsage);-}--/** Projects Vercel AI Gateway cost metadata onto GenAI span attributes. */-export function readGatewayCost(- providerMetadata: Readonly<Record<string, unknown>>,-): Record<string, string | number> | undefined {- const gateway = providerMetadata.gateway;- if (!isRecord(gateway)) return undefined;- const attributes: Record<string, string | number> = {};- const cost = readUsd(gateway.cost);- if (cost !== undefined) attributes["gen_ai.usage.cost"] = cost;- const gatewayCost = readUsd(gateway.gatewayCost);- if (gatewayCost !== undefined) attributes["gen_ai.usage.gateway_cost"] = gatewayCost;- const inputCost = readUsd(gateway.inputInferenceCost);- if (inputCost !== undefined) attributes["gen_ai.usage.input_cost"] = inputCost;- const outputCost = readUsd(gateway.outputInferenceCost);- if (outputCost !== undefined) attributes["gen_ai.usage.output_cost"] = outputCost;- if (typeof gateway.generationId === "string" && gateway.generationId.length > 0) {- attributes["gen_ai.generation.id"] = gateway.generationId;- }- return Object.keys(attributes).length === 0 ? undefined : attributes;-}--function readUsd(value: unknown): number | undefined {- if (typeof value !== "string") return undefined;- const parsed = Number(value);- return Number.isFinite(parsed) ? parsed : undefined;-}--function isRecord(value: unknown): value is Record<string, unknown> {- return typeof value === "object" && value !== null && !Array.isArray(value);-}diff --git a/packages/eve/src/tracing/agent-span-contract.ts b/packages/eve/src/tracing/agent-span-contract.tsdeleted file mode 100644index 917aec7e0..000000000--- a/packages/eve/src/tracing/agent-span-contract.ts+++ /dev/null@@ -1,62 +0,0 @@-export const AGENT_TRACE_SCHEMA_VERSION = 4;--export const AGENT_SPAN_NAMES = {- approval: "agent.approval",- channelRequest: "agent.channel.request",- step: "agent.step",-} as const;--export const AGENT_USAGE_ATTRIBUTES = {- costUsd: "agent.usage.cost_usd",- inputTokens: "agent.usage.input_tokens",- outputTokens: "agent.usage.output_tokens",- cacheReadTokens: "agent.usage.cache_read_tokens",- cacheWriteTokens: "agent.usage.cache_write_tokens",-} as const;--export function agentInvocationSpanName(agentName: string | undefined): string {- return agentName === undefined ? "invoke_agent" : `invoke_agent ${agentName}`;-}--export function modelSpanName(modelId: string): string {- return `chat ${modelId}`;-}--export interface AgentSamplingOperation {- readonly name: string;- readonly attributes?: Readonly<Record<string, string | number | boolean | undefined>>;-}--interface AgentSpanRecord {- readonly name: string;- readonly attributes: Readonly<Record<string, unknown>>;-}--/** Marks the `execute_tool` span of a tool call made outside any conversation. */-export const DIRECT_TOOL_CALL_ATTRIBUTE = "eve.tool.invocation";-export const DIRECT_TOOL_CALL_VALUE = "direct";--/**- * A direct tool call has no turn, so its own span is what starts and ends- * its claim on the trace, the way an activation span does for a turn.- */-export function isDirectToolCallSpan(span: Pick<AgentSpanRecord, "attributes">): boolean {- return span.attributes[DIRECT_TOOL_CALL_ATTRIBUTE] === DIRECT_TOOL_CALL_VALUE;-}--export function isAgentActivationSpan(span: AgentSpanRecord): boolean {- return (- span.name === "agent.turn" ||- (span.attributes["gen_ai.operation.name"] === "invoke_agent" &&- span.attributes["agent.invocation.role"] !== "caller" &&- typeof span.attributes["agent.turn.id"] === "string")- );-}--export function agentTurnIdentity(span: AgentSpanRecord): string | undefined {- const sessionId = span.attributes["agent.run.id"];- const turnId = span.attributes["agent.turn.id"];- return typeof turnId === "string"- ? `${typeof sessionId === "string" ? sessionId : ""}\0${turnId}`- : undefined;-}diff --git a/packages/eve/src/tracing/agent-span-error.ts b/packages/eve/src/tracing/agent-span-error.tsdeleted file mode 100644index e2b4dcf8a..000000000--- a/packages/eve/src/tracing/agent-span-error.ts+++ /dev/null@@ -1,21 +0,0 @@-import { SpanStatusCode, type Span } from "#compiled/@opentelemetry/api/index.js";-import { boundedTraceError } from "#tracing/bounded-error.js";-import { truncateTelemetryText } from "#tracing/telemetry-budget.js";--/** Call with policy-projected errors; undefined retains failure status without content. */-export function recordAgentSpanError(span: Span, error: unknown, errorType?: string): void {- span.setAttribute(- "error.type",- truncateTelemetryText(- errorType ?? (error instanceof Error ? error.name || "Error" : "_OTHER"),- 128,- ),- );- if (error instanceof Error) {- const bounded = boundedTraceError(error);- span.recordException(bounded);- span.setStatus({ code: SpanStatusCode.ERROR, message: bounded.message });- } else {- span.setStatus({ code: SpanStatusCode.ERROR });- }-}diff --git a/packages/eve/src/tracing/agent-span-id-generator.ts b/packages/eve/src/tracing/agent-span-id-generator.tsdeleted file mode 100644index ca569cb8b..000000000--- a/packages/eve/src/tracing/agent-span-id-generator.ts+++ /dev/null@@ -1,82 +0,0 @@-import { createHash } from "node:crypto";--/**- * Id generator shared by `registerOTel` and the agent OTel provider. A span- * whose lifetime crosses durable worker boundaries (`invoke_agent`) is emitted- * at its terminal; priming the next span id lets that span carry the- * pre-allocated id its descendants already parented to.- */-export class AgentSpanIdGenerator {- #primedSpanId: string | undefined;- #primedTraceId: string | undefined;-- /** Reserves a span id for a span emitted later via {@link withSpanId}. */- allocateSpanId(): string {- return randomHexId(16);- }-- /** Derives one span id for a replay-stable instrumentation event. */- deriveSpanId(key: string): string {- return deriveAgentSpanId(key);- }-- deriveTraceId(key: string): string {- const traceId = createHash("sha256").update(key).digest("hex").slice(0, 32);- return /^0+$/u.test(traceId) ? "00000000000000000000000000000001" : traceId;- }-- generateSpanId(): string {- const primed = this.#primedSpanId;- if (primed !== undefined) {- this.#primedSpanId = undefined;- return primed;- }- return randomHexId(16);- }-- generateTraceId(): string {- const primed = this.#primedTraceId;- if (primed !== undefined) {- this.#primedTraceId = undefined;- return primed;- }- return randomHexId(32);- }-- /** Runs `startSpan` so the next span started carries `spanId`. */- withSpanId<T>(spanId: string, startSpan: () => T): T {- this.#primedSpanId = spanId;- try {- return startSpan();- } finally {- this.#primedSpanId = undefined;- }- }-- withTraceId<T>(traceId: string, startSpan: () => T): T {- this.#primedTraceId = traceId;- try {- return startSpan();- } finally {- this.#primedTraceId = undefined;- }- }-}--function deriveAgentSpanId(key: string): string {- const spanId = createHash("sha256").update(key).digest("hex").slice(0, 16);- return /^0+$/u.test(spanId) ? "0000000000000001" : spanId;-}--const HEX_DIGITS = "0123456789abcdef";--function randomHexId(length: number): string {- let id: string;- do {- id = "";- for (let index = 0; index < length; index += 1) {- id += HEX_DIGITS[Math.trunc(Math.random() * 16)];- }- } while (!/[1-9a-f]/u.test(id));- return id;-}diff --git a/packages/eve/src/tracing/agent-span-naming.ts b/packages/eve/src/tracing/agent-span-naming.tsdeleted file mode 100644index b268155ae..000000000--- a/packages/eve/src/tracing/agent-span-naming.ts+++ /dev/null@@ -1,7 +0,0 @@-// Datadog maps operation and resource separately from the OTel span name.-export function agentSpanNamingAttributes(- name: string,- operation: string = name,-): Record<string, string> {- return { "operation.name": operation, "resource.name": name };-}diff --git a/packages/eve/src/tracing/agent-tool-instrumentation.ts b/packages/eve/src/tracing/agent-tool-instrumentation.tsdeleted file mode 100644index 256d34dba..000000000--- a/packages/eve/src/tracing/agent-tool-instrumentation.ts+++ /dev/null@@ -1,563 +0,0 @@-import {- ROOT_CONTEXT,- SpanKind,- type Context,- type Attributes,- type Span,- type SpanContext,- type Tracer,- trace,-} from "#compiled/@opentelemetry/api/index.js";--import type {- InstrumentationAttemptScope,- InstrumentationExecutionOperation,- InstrumentationToolCallStartedEvent,- InstrumentationToolCallTerminalEvent,-} from "#instrumentation/lifecycle.js";-import { actionIdempotencyKey, attemptIdempotencyKey } from "#instrumentation/lifecycle.js";-import { contentAttribute, textContentAttribute } from "#tracing/agent-otel-content.js";-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";-import { agentTraceIdentityAttributes, traceSessionIdOf } from "#tracing/agent-otel-attributes.js";-import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";-import type { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";-import type { AgentActionTraceState, AgentTraceStateStore } from "#tracing/agent-trace-state.js";-import { normalizeChannelAudience } from "#shared/channel-audience.js";-import { isSampledTrace } from "#tracing/sampled-trace.js";-import { setAgentUsage } from "#tracing/agent-otel-usage.js";-import { recordAgentSpanError as recordError } from "#tracing/agent-span-error.js";-import { withAgentToolSpanContext } from "#tracing/agent-tool-span-context.js";--interface ToolSpanState {- readonly actionKey: string;- readonly attemptId: string;- context: Context;- readonly additionalAttributes: Attributes;- readonly event: InstrumentationToolCallStartedEvent;- readonly fallbackParent: Context;- readonly idempotencyKey: string;- readonly spanId: string;- readonly startTimeMs: number;- finished?: true;- span?: Span;- terminal?: InstrumentationToolCallTerminalEvent;- pendingError?: { readonly error: unknown; readonly errorType?: string };- correlated?: true;-}--export interface AgentToolContext {- readonly context: Context;- readonly spanContext: SpanContext;-}--interface AgentToolInstrumentation {- deleteForSession(sessionId: string): Promise<void>;- failForAttempt(scope: InstrumentationAttemptScope, error: unknown): Promise<void>;- dispatchContextFor(- sessionId: string,- turnId: string,- callId: string,- ): Promise<AgentToolContext | undefined>;- contextFor(attemptId: string, idempotencyKey: string): Context | undefined;- drain(attemptId: string, failure?: { readonly error: unknown }): void;- readonly events: {- readonly "tool.call.completed": (event: InstrumentationToolCallTerminalEvent) => Promise<void>;- readonly "tool.call.failed": (event: InstrumentationToolCallTerminalEvent) => Promise<void>;- readonly "tool.call.started": (event: InstrumentationToolCallStartedEvent) => Promise<void>;- };- runInContext<T>(- operation: Extract<InstrumentationExecutionOperation, { type: "tool.call" }>,- execute: (parent: Context | undefined) => PromiseLike<T>,- recordOutputs: boolean,- ): Promise<T>;-}--/** Enriches durable tool calls, exporting SDK-only calls under their step. */-export function createAgentToolInstrumentation(input: {- readonly frameworkVersion: string;- readonly resolveTraceContext: (- event: InstrumentationToolCallStartedEvent,- ) => SpanContext | undefined | PromiseLike<SpanContext | undefined>;- readonly idGenerator: AgentSpanIdGenerator;- readonly recordInputs: boolean;- readonly recordOutputs: boolean;- readonly resolveFallback: (- event: InstrumentationToolCallStartedEvent,- ) => { readonly context: Context; readonly spanContext: SpanContext } | undefined;- readonly tracer: Tracer;- readonly stateStore: AgentTraceStateStore;-}): AgentToolInstrumentation {- const byAction = new Map<string, ToolSpanState>();- const byAttempt = new Map<string, Map<string, ToolSpanState>>();- const dispatchesByAttempt = new Map<string, Set<string>>();-- async function dispatchContextFor(- sessionId: string,- turnId: string,- callId: string,- ): Promise<AgentToolContext | undefined> {- const state =- (await input.stateStore.getAction(actionIdempotencyKey(sessionId, turnId, callId))) ??- (await input.stateStore.findAction(sessionId, callId));- if (state === undefined) return undefined;- const spanContext = {- isRemote: false,- spanId: state.spanId,- traceFlags: state.parent.traceFlags,- traceId: state.parent.traceId,- };- return {- context: withChannelAudience(contextFromSpanContext(spanContext), state.channelAudience),- spanContext,- };- }-- async function onDispatchStarted(event: InstrumentationToolCallStartedEvent): Promise<void> {- const traceContext = await input.resolveTraceContext(event);- if (traceContext === undefined || !isSampledTrace(traceContext)) return;- const existing = await input.stateStore.getAction(event.idempotencyKey);- const state: AgentActionTraceState = existing ?? {- attemptIndex: event.scope.attemptIndex,- callId: event.callId,- channelAudience: normalizeChannelAudience(event.scope.channelAudience),- inputAttribute: input.recordInputs ? contentAttribute(event.input) : undefined,- kind: event.kind ?? "tool-call",- name: event.toolName,- toolAttributes: { "agent.tool.is_framework": event.frameworkTool === true },- parent: {- spanId: input.idGenerator.deriveSpanId(attemptIdempotencyKey(event.scope)),- traceFlags: traceContext.traceFlags,- traceId: traceContext.traceId,- },- rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,- traceSessionId: traceSessionIdOf(event.scope),- sessionId: event.scope.sessionId,- spanId: input.idGenerator.deriveSpanId(`action:${event.idempotencyKey}`),- startTimeMs: Math.min(- event.startedAtMs ?? Date.now(),- byAction.get(event.idempotencyKey)?.startTimeMs ?? Infinity,- ),- stepIndex: event.scope.stepIndex,- turnId: event.scope.turnId,- };- await input.stateStore.setAction(event.idempotencyKey, state);- if (event.isWorkflowTool === true)- await input.stateStore.setActionAnchor(event.idempotencyKey, state);- const keys = dispatchesByAttempt.get(event.scope.attemptId) ?? new Set<string>();- keys.add(event.idempotencyKey);- dispatchesByAttempt.set(event.scope.attemptId, keys);- const execution = byAction.get(event.idempotencyKey);- if (execution !== undefined && execution.finished !== true) {- await correlate(execution);- finishIfReady(execution);- }- }-- async function onDispatchTerminal(event: InstrumentationToolCallTerminalEvent): Promise<void> {- const state = await input.stateStore.getAction(event.idempotencyKey);- if (state === undefined) return;- try {- const span = startDispatchSpan(state);- const outcome = event.outcome ?? (event.type === "tool.call.failed" ? "failed" : "completed");- span.setAttribute(- "agent.action.outcome",- state.toolFailed === true && outcome === "completed" ? "failed" : outcome,- );- if (event.usage !== undefined) setAgentUsage(span, event.usage);- if (event.type === "tool.call.failed") {- recordToolError(span, event.error, event.errorCode);- } else if (event.output.type === "error") {- recordToolError(span, event.output.error);- } else if (state.toolFailed === true) {- recordToolError(- span,- state.toolErrorAttribute,- typeof state.toolAttributes?.["error.type"] === "string"- ? state.toolAttributes["error.type"]- : undefined,- );- } else if (input.recordOutputs) {- const result = contentAttribute(event.output.output);- if (result !== undefined) span.setAttribute("gen_ai.tool.call.result", result);- }- span.end(event.acceptedAtMs ?? state.toolEndTimeMs);- } finally {- await input.stateStore.deleteAction(event.idempotencyKey);- for (const [attemptId, keys] of dispatchesByAttempt) {- keys.delete(event.idempotencyKey);- if (keys.size === 0) dispatchesByAttempt.delete(attemptId);- }- }- }-- function startDispatchSpan(state: AgentActionTraceState): Span {- const invocation = state.kind === "subagent-call" || state.kind === "remote-agent-call";- const span = input.idGenerator.withSpanId(state.spanId, () =>- input.tracer.startSpan(- `execute_tool ${state.name}`,- {- attributes: {- "agent.action.kind": state.kind,- "agent.action.name": state.name,- "gen_ai.operation.name": "execute_tool",- "gen_ai.tool.call.id": state.callId,- "gen_ai.tool.name": state.name,- "gen_ai.tool.type": "function",- "agent.framework.name": "eve",- "agent.framework.version": input.frameworkVersion,- "agent.step.attempt": state.attemptIndex,- "agent.step.index": state.stepIndex,- "agent.turn.id": state.turnId,- ...agentSpanNamingAttributes(`execute_tool ${state.name}`, "execute_tool"),- ...agentTraceIdentityAttributes({- rootSessionId: state.rootSessionId,- traceSessionId: state.traceSessionId,- sessionId: state.sessionId,- }),- ...(invocation- ? { "gen_ai.agent.name": state.name, "agent.invocation.role": "caller" }- : undefined),- ...state.toolAttributes,- },- kind: state.kind === "remote-agent-call" ? SpanKind.CLIENT : SpanKind.INTERNAL,- startTime: state.startTimeMs,- },- withChannelAudience(- contextFromSpanContext({ ...state.parent, isRemote: false }),- state.channelAudience,- ),- ),- );- if (state.inputAttribute !== undefined)- span.setAttribute("gen_ai.tool.call.arguments", state.inputAttribute);- return span;- }-- const onStarted = async (event: InstrumentationToolCallStartedEvent): Promise<void> => {- if (byAttempt.get(event.scope.attemptId)?.has(event.idempotencyKey)) return;- const actionKey = actionIdempotencyKey(event.scope.sessionId, event.scope.turnId, event.callId);- const fallback = input.resolveFallback(event);- let state = fallback === undefined ? undefined : reserve(event, actionKey, fallback);- const actionParent = await dispatchContextFor(- event.scope.sessionId,- event.scope.turnId,- event.callId,- );- if (state === undefined) {- if (actionParent === undefined) return;- state = reserve(event, actionKey, actionParent);- }- if (actionParent !== undefined) await correlate(state);- };-- const onTerminal = async (event: InstrumentationToolCallTerminalEvent): Promise<void> => {- const state = byAttempt.get(event.scope.attemptId)?.get(event.idempotencyKey);- if (state === undefined) return;- state.terminal = event;-- if (state.span === undefined) {- const actionParent = await dispatchContextFor(- state.event.scope.sessionId,- state.event.scope.turnId,- state.event.callId,- );- if (actionParent !== undefined) await correlate(state);- }- finishIfReady(state);- };-- return {- dispatchContextFor,- async deleteForSession(sessionId) {- await input.stateStore.deleteActions(sessionId);- await input.stateStore.deleteActionAnchors(sessionId);- },- async failForAttempt(scope, error) {- const keys = dispatchesByAttempt.get(scope.attemptId);- dispatchesByAttempt.delete(scope.attemptId);- for (const key of keys ?? []) {- const state = await input.stateStore.getAction(key);- if (state === undefined) continue;- const span = startDispatchSpan(state);- recordError(span, error);- span.end();- await input.stateStore.deleteAction(key);- }- },- contextFor: (attemptId, idempotencyKey) =>- byAttempt.get(attemptId)?.get(idempotencyKey)?.context,- drain(attemptId, failure) {- const states = byAttempt.get(attemptId);- if (states === undefined) return;- for (const state of states.values()) {- if (state.finished === true) continue;- if (state.span === undefined && state.correlated !== true)- startSpan(state, state.fallbackParent);- finish(state, failure);- }- byAttempt.delete(attemptId);- },- events: {- "tool.call.completed": onDispatchTerminal,- "tool.call.failed": onDispatchTerminal,- "tool.call.started": onDispatchStarted,- },- async runInContext(operation, execute, recordOutputs) {- if (operation.callId === undefined || operation.toolName === undefined) {- return execute(- byAttempt.get(operation.scope.attemptId)?.get(operation.idempotencyKey)?.context,- );- }- await onStarted({- ...operation,- callId: operation.callId,- toolName: operation.toolName,- input: operation.input,- type: "tool.call.started",- });- const state = byAttempt.get(operation.scope.attemptId)?.get(operation.idempotencyKey);- const startedAtMs = operation.startedAtMs ?? Date.now();- try {- const result = await execute(state?.context);- await onTerminal({- type: operation.failed ? "tool.call.failed" : "tool.call.completed",- idempotencyKey: operation.idempotencyKey,- scope: operation.scope,- completedAtMs: operation.completedAtMs ?? Date.now(),- durationMs: (operation.completedAtMs ?? Date.now()) - startedAtMs,- output: { type: "result", output: recordOutputs ? result : undefined },- });- return result;- } catch (error) {- await onTerminal({- type: "tool.call.failed",- idempotencyKey: operation.idempotencyKey,- scope: operation.scope,- completedAtMs: Date.now(),- error: recordOutputs ? error : undefined,- });- throw error;- }- },- };-- function getAttemptStates(attemptId: string): Map<string, ToolSpanState> {- let states = byAttempt.get(attemptId);- if (states === undefined) {- states = new Map();- byAttempt.set(attemptId, states);- }- return states;- }-- function reserve(- event: InstrumentationToolCallStartedEvent,- actionKey: string,- parent: { readonly context: Context; readonly spanContext: SpanContext },- ): ToolSpanState {- const spanId = input.idGenerator.deriveSpanId(`action:${actionKey}`);- const state: ToolSpanState = {- actionKey,- attemptId: event.scope.attemptId,- additionalAttributes: {},- context: withChannelAudience(- contextFromSpanContext({- isRemote: false,- spanId,- traceFlags: parent.spanContext.traceFlags,- traceId: parent.spanContext.traceId,- }),- event.scope.channelAudience,- ),- event,- fallbackParent: parent.context,- idempotencyKey: event.idempotencyKey,- spanId,- startTimeMs: event.startedAtMs ?? Date.now(),- };- state.context = withAgentToolSpanContext(state.context, {- recordInputs: input.recordInputs,- recordOutputs: input.recordOutputs,- setAttributes(attributes) {- Object.assign(state.additionalAttributes, attributes);- if (state.span === undefined) return;- for (const [name, value] of Object.entries(attributes)) {- if (value !== undefined) state.span.setAttribute(name, value);- }- },- recordError(error, errorType) {- state.pendingError = { error, errorType };- if (state.span !== undefined) recordError(state.span, error, errorType);- },- });- getAttemptStates(event.scope.attemptId).set(event.idempotencyKey, state);- byAction.set(actionKey, state);- return state;- }-- function startSpan(state: ToolSpanState, parent: Context): void {- if (state.span !== undefined || state.finished === true) return;- state.span = input.idGenerator.withSpanId(state.spanId, () =>- input.tracer.startSpan(- `execute_tool ${state.event.toolName}`,- {- attributes: { ...toolAttributes(state.event), ...state.additionalAttributes },- kind: SpanKind.INTERNAL,- startTime: state.startTimeMs,- },- parent,- ),- );- if (state.pendingError !== undefined) {- recordError(state.span, state.pendingError.error, state.pendingError.errorType);- }- if (input.recordInputs) {- const args = contentAttribute(state.event.input);- if (args !== undefined) state.span.setAttribute("gen_ai.tool.call.arguments", args);- }- }-- function finishIfReady(state: ToolSpanState): void {- if ((state.span === undefined && state.correlated !== true) || state.terminal === undefined)- return;- finish(state);- }-- async function correlate(state: ToolSpanState): Promise<void> {- const action =- (await input.stateStore.getAction(state.actionKey)) ??- (await input.stateStore.findAction(state.event.scope.sessionId, state.event.callId));- if (action === undefined) return;- state.correlated = true;- state.context = trace.setSpan(- state.context,- trace.wrapSpanContext({- isRemote: false,- spanId: action.spanId,- traceFlags: action.parent.traceFlags,- traceId: action.parent.traceId,- }),- );- const terminal = state.terminal;- const toolFailed =- state.pendingError !== undefined ||- terminal?.type === "tool.call.failed" ||- terminal?.output.type === "error";- const error =- state.pendingError?.error ??- (terminal?.type === "tool.call.failed"- ? terminal.error- : terminal?.output.type === "error"- ? terminal.output.error- : undefined);- const attributes: Attributes = {- ...action.toolAttributes,- ...(state.event.scope.functionId === undefined ||- action.kind === "subagent-call" ||- action.kind === "remote-agent-call"- ? undefined- : { "gen_ai.agent.name": state.event.scope.functionId }),- "agent.tool.is_framework": state.event.frameworkTool === true,- ...state.additionalAttributes,- ...(state.pendingError?.errorType === undefined- ? undefined- : { "error.type": state.pendingError.errorType }),- ...(terminal?.type === "tool.call.completed" && terminal.durationMs !== undefined- ? { "gen_ai.execute_tool.duration": terminal.durationMs / 1000 }- : undefined),- };- await input.stateStore.setAction(- actionIdempotencyKey(action.sessionId, action.turnId, action.callId),- {- ...action,- startTimeMs: Math.min(action.startTimeMs, state.startTimeMs),- toolAttributes: attributes,- toolEndTimeMs: terminal?.completedAtMs ?? action.toolEndTimeMs,- toolFailed: toolFailed || action.toolFailed,- toolErrorAttribute:- input.recordOutputs && error !== undefined- ? contentAttribute(error instanceof Error ? error.message : error)- : action.toolErrorAttribute,- },- );- }-- function finish(state: ToolSpanState, failure?: { readonly error: unknown }): void {- const span = state.span;- if (state.finished === true) return;- if (state.correlated === true) {- state.finished = true;- forget(state);- return;- }- if (span === undefined) return;- state.finished = true;- const terminal = state.terminal;- // As `@ai-sdk/otel` records it: execute's own run time, in seconds, success or error.- if (terminal?.type === "tool.call.completed" && terminal.durationMs !== undefined) {- span.setAttribute("gen_ai.execute_tool.duration", terminal.durationMs / 1000);- }- if (failure !== undefined) {- recordError(span, failure.error);- } else if (terminal?.type === "tool.call.failed") {- recordError(span, terminal.error);- } else if (terminal?.output.type === "error") {- recordError(span, terminal.output.error);- } else if (terminal !== undefined && input.recordOutputs) {- const result = contentAttribute(terminal.output.output);- if (result !== undefined) span.setAttribute("gen_ai.tool.call.result", result);- }- span.end(terminal?.completedAtMs);- forget(state);- }-- function forget(state: ToolSpanState): void {- byAction.delete(state.actionKey);- const states = byAttempt.get(state.attemptId);- states?.delete(state.idempotencyKey);- if (states?.size === 0) byAttempt.delete(state.attemptId);- }-}--function toolAttributes(event: InstrumentationToolCallStartedEvent): Attributes {- return {- "agent.tool.is_framework": event.frameworkTool === true,- ...(event.scope.functionId === undefined- ? {}- : { "gen_ai.agent.name": event.scope.functionId }),- "gen_ai.operation.name": "execute_tool",- "gen_ai.tool.call.id": event.callId,- "gen_ai.tool.name": event.toolName,- "gen_ai.tool.type": "function",- ...agentSpanNamingAttributes(`execute_tool ${event.toolName}`, "execute_tool"),- ...agentTraceIdentityAttributes({- rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,- traceSessionId: traceSessionIdOf(event.scope),- sessionId: event.scope.sessionId,- }),- };-}--function contextFromSpanContext(spanContext: SpanContext): Context {- return trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(spanContext));-}--function recordToolError(span: Span, error: unknown, errorType?: string): void {- if (error instanceof Error || error === undefined) {- recordError(span, error, errorType);- return;- }- const serialized =- typeof error === "string" ? textContentAttribute(error) : contentAttribute(error);- const message =- typeof error === "object" && error !== null && !Array.isArray(error)- ? Reflect.get(error, "message")- : undefined;- const detail =- typeof message === "string"- ? textContentAttribute(serialized === undefined ? message : `${message}\n${serialized}`)- : serialized;- const normalized = detail === undefined ? undefined : new Error(detail);- if (normalized !== undefined && errorType !== undefined) normalized.name = errorType;- recordError(span, normalized, errorType);-}diff --git a/packages/eve/src/tracing/agent-tool-span-context.test.ts b/packages/eve/src/tracing/agent-tool-span-context.test.tsdeleted file mode 100644index b9227c111..000000000--- a/packages/eve/src/tracing/agent-tool-span-context.test.ts+++ /dev/null@@ -1,37 +0,0 @@-import { ROOT_CONTEXT } from "#compiled/@opentelemetry/api/index.js";-import { describe, expect, it, vi } from "vitest";--import {- agentToolContentPolicy,- annotateAgentToolSpan,- recordAgentToolSpanError,- withAgentToolContentPolicy,- withAgentToolSpanContext,-} from "#tracing/agent-tool-span-context.js";--describe("agent tool span context", () => {- it("narrows the capture policy without dropping the span writer", () => {- const setAttributes = vi.fn();- const recordError = vi.fn();- const context = withAgentToolSpanContext(ROOT_CONTEXT, {- recordError,- recordInputs: true,- recordOutputs: true,- setAttributes,- });- const narrowed = withAgentToolContentPolicy(context, {- recordInputs: false,- recordOutputs: false,- });-- expect(agentToolContentPolicy(narrowed)).toEqual({- recordInputs: false,- recordOutputs: false,- });- expect(annotateAgentToolSpan({ "mcp.method.name": "tools/call" }, narrowed)).toBe(true);- recordAgentToolSpanError(new Error("private result"), "tool_error", narrowed);-- expect(setAttributes).toHaveBeenCalledWith({ "mcp.method.name": "tools/call" });- expect(recordError).toHaveBeenCalledWith(undefined, "tool_error");- });-});diff --git a/packages/eve/src/tracing/agent-tool-span-context.ts b/packages/eve/src/tracing/agent-tool-span-context.tsdeleted file mode 100644index f08b7e27b..000000000--- a/packages/eve/src/tracing/agent-tool-span-context.ts+++ /dev/null@@ -1,73 +0,0 @@-import {- context as otelContext,- createContextKey,- type Attributes,- type Context,-} from "#compiled/@opentelemetry/api/index.js";--interface AgentToolContentPolicy {- readonly recordInputs: boolean;- readonly recordOutputs: boolean;-}--interface AgentToolSpanContext extends AgentToolContentPolicy {- readonly recordError?: (error: unknown, errorType?: string) => void;- readonly setAttributes?: (attributes: Attributes) => void;-}--const AGENT_TOOL_SPAN_CONTEXT_KEY = createContextKey("eve.agent.tool-span-context");-const NO_CONTENT_POLICY: AgentToolContentPolicy = { recordInputs: false, recordOutputs: false };--export function withAgentToolSpanContext(context: Context, value: AgentToolSpanContext): Context {- return context.setValue(AGENT_TOOL_SPAN_CONTEXT_KEY, value);-}--export function withAgentToolContentPolicy(- context: Context,- policy: AgentToolContentPolicy,-): Context {- const current = agentToolSpanContext(context);- return context.setValue(AGENT_TOOL_SPAN_CONTEXT_KEY, {- ...current,- ...policy,- } satisfies AgentToolSpanContext);-}--export function agentToolSpanContext(- context: Context = otelContext.active(),-): AgentToolSpanContext | undefined {- return context.getValue(AGENT_TOOL_SPAN_CONTEXT_KEY) as AgentToolSpanContext | undefined;-}--export function agentToolContentPolicy(- context: Context = otelContext.active(),-): AgentToolContentPolicy {- const spanContext = agentToolSpanContext(context);- return spanContext === undefined- ? NO_CONTENT_POLICY- : {- recordInputs: spanContext.recordInputs,- recordOutputs: spanContext.recordOutputs,- };-}--export function annotateAgentToolSpan(- attributes: Attributes,- context: Context = otelContext.active(),-): boolean {- const setAttributes = agentToolSpanContext(context)?.setAttributes;- if (setAttributes === undefined) return false;- setAttributes(attributes);- return true;-}--export function recordAgentToolSpanError(- error: unknown,- errorType?: string,- context: Context = otelContext.active(),-): boolean {- const spanContext = agentToolSpanContext(context);- if (spanContext?.recordError === undefined) return false;- spanContext.recordError(spanContext.recordOutputs ? error : undefined, errorType);- return true;-}diff --git a/packages/eve/src/tracing/agent-trace-context-codec.ts b/packages/eve/src/tracing/agent-trace-context-codec.tsdeleted file mode 100644index 3f823b836..000000000--- a/packages/eve/src/tracing/agent-trace-context-codec.ts+++ /dev/null@@ -1,318 +0,0 @@-import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";--import type {- AgentActionTraceState,- AgentSessionTraceState,- AgentTurnTraceState,-} from "#tracing/agent-trace-state.js";-import { normalizeChannelAudience } from "#shared/channel-audience.js";-import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";-import { boundedTraceError } from "#tracing/bounded-error.js";-import { CONTENT_ATTRIBUTE_LIMIT } from "#tracing/agent-otel-content.js";-import { boundedPrincipalId } from "#tracing/telemetry-budget.js";-import { isInstrumentationPrincipalType } from "#instrumentation/lifecycle.js";--export const AGENT_TRACE_CONTEXT_KEY = "eve.harness.agentTrace";--export function decodeTraceSessionId(state: {- readonly traceSessionId?: unknown;- readonly rootSessionId?: unknown;-}): string {- // Persisted state from before trace-session identity uses the lineage root.- return typeof state.traceSessionId === "string"- ? state.traceSessionId- : typeof state.rootSessionId === "string"- ? state.rootSessionId- : "";-}--export interface AgentTraceContextState {- readonly actionAnchors: Readonly<Record<string, AgentActionTraceState>>;- readonly actions: Readonly<Record<string, AgentActionTraceState>>;- readonly sessions: Readonly<Record<string, AgentSessionTraceState>>;- readonly turns: Readonly<Record<string, AgentTurnTraceState>>;-}--export function emptyAgentTraceContextState(): AgentTraceContextState {- return { actionAnchors: {}, actions: {}, sessions: {}, turns: {} };-}--export function serializeAgentTraceContextState(state: AgentTraceContextState): unknown {- return {- actionAnchors: state.actionAnchors,- actions: state.actions,- sessions: Object.fromEntries(- Object.entries(state.sessions).map(([id, value]) => [- id,- { ...value, context: serializeSpanContext(value.context) },- ]),- ),- turns: Object.fromEntries(- Object.entries(state.turns).map(([id, value]) => [- id,- {- ...value,- caller: value.caller === undefined ? undefined : serializeSpanContext(value.caller),- channelDelivery:- value.channelDelivery === undefined- ? undefined- : {- ...value.channelDelivery,- requestTraceContext:- value.channelDelivery.requestTraceContext === undefined- ? undefined- : serializeSpanContext(value.channelDelivery.requestTraceContext),- },- context: serializeSpanContext(value.context),- terminal:- value.terminal === undefined- ? undefined- : value.terminal.type === "turn.failed"- ? { error: serializeError(value.terminal.error), type: value.terminal.type }- : { type: value.terminal.type },- },- ]),- ),- };-}--export function deserializeAgentTraceContextState(data: unknown): AgentTraceContextState {- if (!isRecord(data)) return emptyAgentTraceContextState();- return {- actionAnchors: deserializeRecord(data.actionAnchors, deserializeAction),- actions: deserializeRecord(data.actions, deserializeAction),- sessions: deserializeRecord(data.sessions, deserializeSession),- turns: deserializeRecord(data.turns, deserializeTurn),- };-}--function deserializeSession(value: unknown): AgentSessionTraceState | undefined {- if (!isRecord(value) || !isSpanContext(value.context)) return undefined;- return {- agentName: typeof value.agentName === "string" ? value.agentName : undefined,- channelAudience: normalizeChannelAudience(value.channelAudience),- channelKind: typeof value.channelKind === "string" ? value.channelKind : undefined,- channelType: typeof value.channelType === "string" ? value.channelType : undefined,- context: value.context,- decision: readInstrumentationDecision(value.decision),- parentLineage: deserializeParentLineage(value.parentLineage),- rootSessionId: typeof value.rootSessionId === "string" ? value.rootSessionId : "",- traceSessionId: decodeTraceSessionId(value),- scheduleId: typeof value.scheduleId === "string" ? value.scheduleId : undefined,- title: typeof value.title === "string" ? value.title : undefined,- };-}--function deserializeTurn(value: unknown): AgentTurnTraceState | undefined {- if (!isRecord(value) || !isSpanContext(value.context) || typeof value.startTimeMs !== "number") {- return undefined;- }- return {- caller: isSpanContext(value.caller) ? value.caller : undefined,- channelDelivery: deserializeTurnChannelDelivery(value.channelDelivery),- context: value.context,- currentPrincipal: deserializePrincipalSummary(value.currentPrincipal),- initiatorPrincipal: deserializePrincipalSummary(value.initiatorPrincipal),- inputMessagesAttribute: deserializeContentAttribute(value.inputMessagesAttribute),- modelUsage: deserializeModelUsage(value.modelUsage),- outputMessagesAttribute: deserializeContentAttribute(value.outputMessagesAttribute),- parentLineage: deserializeParentLineage(value.parentLineage),- rootSessionId: typeof value.rootSessionId === "string" ? value.rootSessionId : "",- traceSessionId: decodeTraceSessionId(value),- sequence: typeof value.sequence === "number" ? value.sequence : 0,- startTimeMs: value.startTimeMs,- subagentName: typeof value.subagentName === "string" ? value.subagentName : undefined,- terminal: deserializeTurnTerminal(value.terminal),- };-}--function deserializeTurnChannelDelivery(value: unknown): AgentTurnTraceState["channelDelivery"] {- if (- !isRecord(value) ||- typeof value.channelKind !== "string" ||- typeof value.channelName !== "string" ||- typeof value.deliveryId !== "string"- ) {- return undefined;- }- return {- channelKind: value.channelKind,- channelName: value.channelName,- deliveryId: value.deliveryId,- inputAttribute: typeof value.inputAttribute === "string" ? value.inputAttribute : undefined,- requestId: typeof value.requestId === "string" ? value.requestId : undefined,- requestTraceContext: isSpanContext(value.requestTraceContext)- ? value.requestTraceContext- : undefined,- };-}--function deserializePrincipalSummary(- value: unknown,-): AgentTurnTraceState["currentPrincipal"] | undefined {- if (!isRecord(value) || !isInstrumentationPrincipalType(value.type)) return undefined;- const id = value.type === "none" ? undefined : boundedPrincipalId(value.id);- return id === undefined ? { type: value.type } : { id, type: value.type };-}--function deserializeAction(value: unknown): AgentActionTraceState | undefined {- if (- !isRecord(value) ||- typeof value.attemptIndex !== "number" ||- typeof value.callId !== "string" ||- !isActionKind(value.kind) ||- typeof value.name !== "string" ||- !isSpanContext(value.parent) ||- typeof value.rootSessionId !== "string" ||- typeof value.sessionId !== "string" ||- typeof value.spanId !== "string" ||- typeof value.startTimeMs !== "number" ||- typeof value.stepIndex !== "number" ||- typeof value.turnId !== "string"- ) {- return undefined;- }- return {- attemptIndex: value.attemptIndex,- callId: value.callId,- channelAudience: normalizeChannelAudience(value.channelAudience),- inputAttribute: typeof value.inputAttribute === "string" ? value.inputAttribute : undefined,- kind: value.kind,- name: value.name,- parent: value.parent,- rootSessionId: value.rootSessionId,- traceSessionId: decodeTraceSessionId(value),- sessionId: value.sessionId,- spanId: value.spanId,- startTimeMs: value.startTimeMs,- stepIndex: value.stepIndex,- turnId: value.turnId,- toolEndTimeMs: typeof value.toolEndTimeMs === "number" ? value.toolEndTimeMs : undefined,- toolFailed: value.toolFailed === true,- toolErrorAttribute:- typeof value.toolErrorAttribute === "string" ? value.toolErrorAttribute : undefined,- toolAttributes: isRecord(value.toolAttributes)- ? Object.fromEntries(- Object.entries(value.toolAttributes).filter(- (- entry,- ): entry is [string, NonNullable<AgentActionTraceState["toolAttributes"]>[string]] => {- const attribute = entry[1];- return (- typeof attribute === "string" ||- typeof attribute === "number" ||- typeof attribute === "boolean" ||- (Array.isArray(attribute) &&- attribute.every((entry) =>- ["string", "number", "boolean"].includes(typeof entry),- ))- );- },- ),- )- : undefined,- };-}--function deserializeContentAttribute(value: unknown): string | undefined {- return typeof value === "string" && value.length <= CONTENT_ATTRIBUTE_LIMIT ? value : undefined;-}--function deserializeModelUsage(- value: unknown,-): NonNullable<AgentTurnTraceState["modelUsage"]> | undefined {- if (!isRecord(value)) return undefined;- const inputTokens = typeof value.inputTokens === "number" ? value.inputTokens : undefined;- const outputTokens = typeof value.outputTokens === "number" ? value.outputTokens : undefined;- return inputTokens === undefined && outputTokens === undefined- ? undefined- : { inputTokens, outputTokens };-}--function deserializeParentLineage(value: unknown): AgentSessionTraceState["parentLineage"] {- if (- !isRecord(value) ||- typeof value.callId !== "string" ||- typeof value.sessionId !== "string" ||- typeof value.turnId !== "string"- ) {- return undefined;- }- return {- callId: value.callId,- sessionId: value.sessionId,- subagentName: typeof value.subagentName === "string" ? value.subagentName : undefined,- turnId: value.turnId,- };-}--function deserializeTurnTerminal(value: unknown): AgentTurnTraceState["terminal"] {- if (!isRecord(value) || !isTurnTerminalType(value.type)) return undefined;- return value.type === "turn.failed"- ? { error: deserializeError(value.error), type: value.type }- : { type: value.type };-}--function deserializeRecord<T>(- value: unknown,- deserialize: (entry: unknown) => T | undefined,-): Record<string, T> {- if (!isRecord(value)) return {};- const result: Record<string, T> = {};- for (const [key, entry] of Object.entries(value)) {- const parsed = deserialize(entry);- if (parsed !== undefined) result[key] = parsed;- }- return result;-}--function serializeSpanContext(context: SpanContext): Record<string, unknown> {- return {- isRemote: context.isRemote,- spanId: context.spanId,- traceFlags: context.traceFlags,- traceId: context.traceId,- };-}--function serializeError(error: unknown): unknown {- if (!(error instanceof Error)) return undefined;- const bounded = boundedTraceError(error);- return { message: bounded.message, name: bounded.name, stack: bounded.stack };-}--function deserializeError(value: unknown): Error | undefined {- if (!isRecord(value) || typeof value.message !== "string") return undefined;- const error = new Error(value.message);- if (typeof value.name === "string") error.name = value.name;- if (typeof value.stack === "string") error.stack = value.stack;- return error;-}--function isActionKind(value: unknown): value is AgentActionTraceState["kind"] {- return (- value === "load-skill" ||- value === "remote-agent-call" ||- value === "subagent-call" ||- value === "tool-call"- );-}--function isTurnTerminalType(- value: unknown,-): value is "turn.cancelled" | "turn.completed" | "turn.failed" {- return value === "turn.cancelled" || value === "turn.completed" || value === "turn.failed";-}--function isSpanContext(value: unknown): value is SpanContext {- return (- isRecord(value) &&- typeof value.spanId === "string" &&- typeof value.traceFlags === "number" &&- typeof value.traceId === "string"- );-}--function isRecord(value: unknown): value is Record<string, unknown> {- return typeof value === "object" && value !== null && !Array.isArray(value);-}diff --git a/packages/eve/src/tracing/agent-trace-context-store.ts b/packages/eve/src/tracing/agent-trace-context-store.tsdeleted file mode 100644index b23f0267a..000000000--- a/packages/eve/src/tracing/agent-trace-context-store.ts+++ /dev/null@@ -1,291 +0,0 @@-import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";--import type { SessionTraceContext } from "#channel/types.js";-import { contextStorage, loadContext } from "#context/container.js";-import { ContextKey } from "#context/key.js";-import type { ContextAccessor } from "#context/key.js";-import { SessionTraceSeedKey, type SessionTraceSeed } from "#context/keys.js";-import type {- AgentActionTraceState,- AgentSessionTraceState,- AgentTraceStateStore,- AgentTurnTraceState,-} from "#tracing/agent-trace-state.js";-import { actionIdempotencyKey } from "#instrumentation/lifecycle.js";-import type { SessionStateMap } from "#harness/types.js";-import { getBlockingWorkflowToolRuns } from "#harness/workflow-tool-runs.js";--import { createLogger } from "#internal/logging.js";-import type { InstrumentationDecision } from "#shared/instrumentation-decision.js";-import {- decisionToTraceContentCeiling,- resolveForwardedTraceSeed,-} from "#shared/forwarded-trace-policy.js";-import {- deserializeAgentTraceContextState,- AGENT_TRACE_CONTEXT_KEY,- emptyAgentTraceContextState,- serializeAgentTraceContextState,- type AgentTraceContextState,-} from "#tracing/agent-trace-context-codec.js";--const AgentTraceContextKey = new ContextKey<AgentTraceContextState>(AGENT_TRACE_CONTEXT_KEY, {- codec: {- deserialize: deserializeAgentTraceContextState,- serialize: serializeAgentTraceContextState,- },-});--/** Run after task-provider commits, so a pending workflow tool call keeps its anchor. */-export function pruneAgentTraceState(- context: ContextAccessor,- sessionId: string,- sessionState: SessionStateMap | undefined,-): void {- try {- pruneTraceOwnership(context, sessionId, sessionState);- } catch (error) {- createLogger("tracing.retention").warn(- "could not reconcile trace ownership; preserving trace state",- { error },- );- }-}--function pruneTraceOwnership(- context: ContextAccessor,- sessionId: string,- sessionState: SessionStateMap | undefined,-): void {- const state = context.get(AgentTraceContextKey);- if (state === undefined) return;- const calls = new Set(getBlockingWorkflowToolRuns(sessionState).map((run) => run.callId));- const actionAnchors = Object.fromEntries(- Object.entries(state.actionAnchors).filter(- ([key, action]) =>- action.sessionId !== sessionId ||- state.actions[key] !== undefined ||- calls.has(action.callId),- ),- );- context.set(AgentTraceContextKey, { ...state, actionAnchors });-}--export function readSessionTraceDecision(- context: ContextAccessor,- sessionId: string,-): InstrumentationDecision | undefined {- return context.get(AgentTraceContextKey)?.sessions[sessionId]?.decision;-}--/** Keeps only framework trace state from an interrupted step's context changes. */-export function preserveSerializedAgentTraceState(- original: Record<string, unknown>,- interrupted: Record<string, unknown>,-): Record<string, unknown> {- const traceState = interrupted[AgentTraceContextKey.name];- return traceState === undefined- ? original- : { ...original, [AgentTraceContextKey.name]: traceState };-}--/** Reads the active turn context straight out of a serialized Workflow context. */-export function readTurnTraceContext(- serializedContext: Readonly<Record<string, unknown>>,- sessionId: string,- turnId: string,-): SessionTraceContext | undefined {- const raw = serializedContext[AgentTraceContextKey.name];- if (raw === undefined) return undefined;- const state = deserializeAgentTraceContextState(raw);- const turn = state.turns[turnKey(sessionId, turnId)];- return turn === undefined- ? undefined- : withTraceDecision(serializedContext, turn.context, state.sessions[sessionId]?.decision);-}--/** Reads the durable action span that should parent a dispatched child agent. */-export function readActionTraceContext(- serializedContext: Readonly<Record<string, unknown>>,- sessionId: string,- turnId: string,- callId: string,-): SessionTraceContext | undefined {- const raw = serializedContext[AgentTraceContextKey.name];- if (raw === undefined) return undefined;- const state = deserializeAgentTraceContextState(raw);- const key = actionIdempotencyKey(sessionId, turnId, callId);- const action = state.actions[key] ?? state.actionAnchors[key];- if (action === undefined) return undefined;- return withTraceDecision(- serializedContext,- {- isRemote: false,- spanId: action.spanId,- traceFlags: action.parent.traceFlags,- traceId: action.parent.traceId,- },- state.sessions[action.sessionId]?.decision,- );-}--function withTraceDecision(- serializedContext: Readonly<Record<string, unknown>>,- context: SpanContext,- storedDecision?: InstrumentationDecision,-): SessionTraceContext {- const seed = serializedContext[SessionTraceSeedKey.name] as SessionTraceSeed | undefined;- const traceState = resolveForwardedTraceSeed({- decision: storedDecision ?? seed?.decision,- forwardedTracePolicy: seed?.forwardedTracePolicy,- traceFlags: context.traceFlags,- })!;- const decision = traceState.decision;- const forwardedTracePolicy = traceState.forwardedTracePolicy;- const ceiling = decisionToTraceContentCeiling(decision);- const resolvedContext = { ...context, traceFlags: traceState.traceFlags };- if (forwardedTracePolicy === undefined) {- return decision === undefined ? resolvedContext : { ...resolvedContext, decision };- }- const narrowedForwardedTracePolicy =- ceiling === undefined ? forwardedTracePolicy : { ...forwardedTracePolicy, ceiling };- if (decision === undefined) {- return { ...resolvedContext, forwardedTracePolicy: narrowedForwardedTracePolicy };- }- return {- ...resolvedContext,- decision,- forwardedTracePolicy: narrowedForwardedTracePolicy,- };-}--/** Durable trace state backed by eve's serialized Workflow context. */-export class ContextAgentTraceStateStore implements AgentTraceStateStore {- deleteAction(idempotencyKey: string): void {- updateState((state) => {- const actions = { ...state.actions };- delete actions[idempotencyKey];- return { ...state, actions };- });- }-- deleteActionAnchors(sessionId: string): void {- updateState((state) => ({- ...state,- actionAnchors: Object.fromEntries(- Object.entries(state.actionAnchors).filter(([, anchor]) => anchor.sessionId !== sessionId),- ),- }));- }-- deleteActions(sessionId: string, turnId?: string): void {- updateState((state) => {- const actions = { ...state.actions };- for (const [key, action] of Object.entries(actions)) {- if (action.sessionId === sessionId && (turnId === undefined || action.turnId === turnId)) {- delete actions[key];- }- }- return { ...state, actions };- });- }-- deleteSession(sessionId: string): void {- updateState((state) => {- const sessions = { ...state.sessions };- delete sessions[sessionId];- return { ...state, sessions };- });- }-- deleteTurn(sessionId: string, turnId: string): void {- updateState((state) => {- const turns = { ...state.turns };- delete turns[turnKey(sessionId, turnId)];- return { ...state, turns };- });- }-- findAction(sessionId: string, callId: string): AgentActionTraceState | undefined {- return Object.values(contextStorage.getStore()?.get(AgentTraceContextKey)?.actions ?? {}).find(- (state) => state.sessionId === sessionId && state.callId === callId,- );- }-- findActionAnchor(- sessionId: string,- turnId: string,- callId: string,- ): AgentActionTraceState | undefined {- return Object.values(- contextStorage.getStore()?.get(AgentTraceContextKey)?.actionAnchors ?? {},- ).find(- (anchor) =>- anchor.sessionId === sessionId && anchor.turnId === turnId && anchor.callId === callId,- );- }-- getAction(idempotencyKey: string): AgentActionTraceState | undefined {- return contextStorage.getStore()?.get(AgentTraceContextKey)?.actions[idempotencyKey];- }-- getSession(sessionId: string): AgentSessionTraceState | undefined {- return contextStorage.getStore()?.get(AgentTraceContextKey)?.sessions[sessionId];- }-- getTurn(sessionId: string, turnId: string): AgentTurnTraceState | undefined {- return contextStorage.getStore()?.get(AgentTraceContextKey)?.turns[turnKey(sessionId, turnId)];- }-- setAction(idempotencyKey: string, value: AgentActionTraceState): void {- updateState((state) => ({- ...state,- actions: { ...state.actions, [idempotencyKey]: value },- }));- }-- setActionAnchor(idempotencyKey: string, value: AgentActionTraceState): void {- updateState((state) => ({- ...state,- actionAnchors: { ...state.actionAnchors, [idempotencyKey]: value },- }));- }-- setSession(sessionId: string, value: AgentSessionTraceState): void {- updateState((state) => ({- ...state,- sessions: { ...state.sessions, [sessionId]: value },- }));- }-- setTurn(sessionId: string, turnId: string, value: AgentTurnTraceState): void {- updateState((state) => ({- ...state,- turns: { ...state.turns, [turnKey(sessionId, turnId)]: value },- }));- }-- updateTurn(- sessionId: string,- turnId: string,- update: (state: AgentTurnTraceState) => AgentTurnTraceState,- ): void {- updateState((state) => {- const key = turnKey(sessionId, turnId);- const current = state.turns[key];- return current === undefined- ? state- : { ...state, turns: { ...state.turns, [key]: update(current) } };- });- }-}--function updateState(update: (state: AgentTraceContextState) => AgentTraceContextState): void {- loadContext().set(AgentTraceContextKey, (state) =>- update(state ?? emptyAgentTraceContextState()),- );-}--function turnKey(sessionId: string, turnId: string): string {- return `${sessionId}\0${turnId}`;-}diff --git a/packages/eve/src/tracing/agent-trace-state.ts b/packages/eve/src/tracing/agent-trace-state.tsdeleted file mode 100644index 74278d1a3..000000000--- a/packages/eve/src/tracing/agent-trace-state.ts+++ /dev/null@@ -1,116 +0,0 @@-import type { Attributes, SpanContext } from "#compiled/@opentelemetry/api/index.js";--import type {- InstrumentationToolCallKind,- InstrumentationParentLineage,- InstrumentationPrincipalSummary,- InstrumentationTraceContext,- InstrumentationTurnFailedEvent,- InstrumentationTurnSettledEvent,-} from "#instrumentation/lifecycle.js";-import type { ChannelAudience } from "#shared/channel-audience.js";-import type { InstrumentationDecision } from "#shared/instrumentation-decision.js";--export interface AgentSessionTraceState {- readonly traceSessionId: string;- readonly channelAudience?: ChannelAudience;- readonly agentName?: string;- readonly channelKind?: string;- readonly channelType?: string;- readonly context: SpanContext;- readonly decision?: InstrumentationDecision;- readonly parentLineage?: InstrumentationParentLineage;- readonly rootSessionId: string;- readonly scheduleId?: string;- readonly title?: string;-}--export interface AgentTurnTraceState {- readonly traceSessionId: string;- readonly caller?: SpanContext;- readonly channelDelivery?: AgentTurnChannelDeliveryTraceState;- readonly context: SpanContext;- readonly currentPrincipal?: InstrumentationPrincipalSummary;- readonly initiatorPrincipal?: InstrumentationPrincipalSummary;- readonly parentLineage?: InstrumentationParentLineage;- /** First model input projected onto the turn-level invoke_agent span. */- readonly inputMessagesAttribute?: string;- readonly modelUsage?: { readonly inputTokens?: number; readonly outputTokens?: number };- /** Latest model output projected onto the turn-level invoke_agent span. */- readonly outputMessagesAttribute?: string;- readonly rootSessionId: string;- readonly sequence: number;- readonly startTimeMs: number;- readonly subagentName?: string;- readonly terminal?:- | { readonly error: unknown; readonly type: InstrumentationTurnFailedEvent["type"] }- | { readonly type: InstrumentationTurnSettledEvent["type"] };-}--export interface AgentTurnChannelDeliveryTraceState {- readonly channelKind: string;- readonly channelName: string;- readonly deliveryId: string;- readonly inputAttribute?: string;- readonly requestId?: string;- readonly requestTraceContext?: SpanContext;-}--export interface AgentActionTraceState {- readonly traceSessionId: string;- readonly attemptIndex: number;- readonly callId: string;- readonly channelAudience?: ChannelAudience;- readonly inputAttribute?: string;- readonly kind: InstrumentationToolCallKind;- readonly name: string;- readonly parent: InstrumentationTraceContext;- readonly rootSessionId: string;- readonly sessionId: string;- readonly spanId: string;- readonly startTimeMs: number;- readonly stepIndex: number;- readonly turnId: string;- readonly toolAttributes?: Attributes;- readonly toolEndTimeMs?: number;- readonly toolFailed?: boolean;- readonly toolErrorAttribute?: string;-}--/** Provider-owned serializable storage for durable agent trace state. */-export interface AgentTraceStateStore {- deleteAction(idempotencyKey: string): void | PromiseLike<void>;- deleteActionAnchors(sessionId: string): void | PromiseLike<void>;- deleteActions(sessionId: string, turnId?: string): void | PromiseLike<void>;- deleteSession(sessionId: string): void | PromiseLike<void>;- deleteTurn(sessionId: string, turnId: string): void | PromiseLike<void>;- findAction(- sessionId: string,- callId: string,- ): AgentActionTraceState | undefined | PromiseLike<AgentActionTraceState | undefined>;- findActionAnchor(- sessionId: string,- turnId: string,- callId: string,- ): AgentActionTraceState | undefined | PromiseLike<AgentActionTraceState | undefined>;- getAction(- idempotencyKey: string,- ): AgentActionTraceState | undefined | PromiseLike<AgentActionTraceState | undefined>;- getSession(- sessionId: string,- ): AgentSessionTraceState | undefined | PromiseLike<AgentSessionTraceState | undefined>;- getTurn(- sessionId: string,- turnId: string,- ): AgentTurnTraceState | undefined | PromiseLike<AgentTurnTraceState | undefined>;- setAction(idempotencyKey: string, state: AgentActionTraceState): void | PromiseLike<void>;- setActionAnchor(idempotencyKey: string, state: AgentActionTraceState): void | PromiseLike<void>;- setSession(sessionId: string, state: AgentSessionTraceState): void | PromiseLike<void>;- setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void | PromiseLike<void>;- /** Atomically updates an existing turn and does nothing after that turn is deleted. */- updateTurn(- sessionId: string,- turnId: string,- update: (state: AgentTurnTraceState) => AgentTurnTraceState,- ): void | PromiseLike<void>;-}diff --git a/packages/eve/src/tracing/bounded-error.ts b/packages/eve/src/tracing/bounded-error.tsdeleted file mode 100644index c457c476b..000000000--- a/packages/eve/src/tracing/bounded-error.ts+++ /dev/null@@ -1,8 +0,0 @@-import { truncateTelemetryText } from "#tracing/telemetry-budget.js";--export function boundedTraceError(value: Error): Error {- const error = new Error(truncateTelemetryText(value.message, 4096));- error.name = truncateTelemetryText(value.name, 128);- error.stack = value.stack === undefined ? undefined : truncateTelemetryText(value.stack, 8192);- return error;-}diff --git a/packages/eve/src/tracing/content-attributes.test.ts b/packages/eve/src/tracing/content-attributes.test.tsdeleted file mode 100644index f60c580ea..000000000--- a/packages/eve/src/tracing/content-attributes.test.ts+++ /dev/null@@ -1,125 +0,0 @@-import { describe, expect, it } from "vitest";--import { withoutDeclinedContent } from "#tracing/content-attributes.js";--const ATTRIBUTES = {- "agent.channel.delivery.input": '{"message":"private"}',- "agent.session.title": "Private title",- "agent.trace.content.input": true,- "agent.trace.content.output": true,- "gen_ai.input.messages": "what the user said",- "gen_ai.system_instructions": '[{"content":"private instructions","type":"text"}]',- "ai.response.finish_reason": "stop",- "ai.response.text": "what the model said",- "gen_ai.request.model": "test-model",- "gen_ai.tool.call.arguments": "{}",- "gen_ai.tool.call.result": "42",- "gen_ai.tool.name": "weather",-};--describe("withoutDeclinedContent", () => {- it("keeps everything when the destination declined nothing", () => {- expect(- withoutDeclinedContent(ATTRIBUTES, { recordInputs: true, recordOutputs: true }),- ).toBeUndefined();- });-- it("keeps everything when the span carries none of what was declined", () => {- expect(- withoutDeclinedContent(- { "gen_ai.request.model": "test-model" },- { recordInputs: false, recordOutputs: false },- ),- ).toBeUndefined();- });-- it("drops inputs alone", () => {- expect(- withoutDeclinedContent(ATTRIBUTES, { recordInputs: false, recordOutputs: true }),- ).toEqual({- "agent.trace.content.input": false,- "agent.trace.content.output": true,- "ai.response.finish_reason": "stop",- "ai.response.text": "what the model said",- "gen_ai.request.model": "test-model",- "gen_ai.tool.call.result": "42",- "gen_ai.tool.name": "weather",- });- });-- it("drops outputs alone", () => {- expect(- withoutDeclinedContent(ATTRIBUTES, { recordInputs: true, recordOutputs: false }),- ).toEqual({- "agent.channel.delivery.input": '{"message":"private"}',- "agent.session.title": "Private title",- "agent.trace.content.input": true,- "agent.trace.content.output": false,- "gen_ai.input.messages": "what the user said",- "gen_ai.system_instructions": '[{"content":"private instructions","type":"text"}]',- "ai.response.finish_reason": "stop",- "gen_ai.request.model": "test-model",- "gen_ai.tool.call.arguments": "{}",- "gen_ai.tool.name": "weather",- });- });-- it("drops system instructions when the destination declined inputs", () => {- expect(- withoutDeclinedContent(- { "gen_ai.system_instructions": '[{"content":"private","type":"text"}]' },- { recordInputs: false, recordOutputs: true },- ),- ).toEqual({});- });-- // The prefixes are shared: `ai.response.finish_reason` and `gen_ai.tool.name`- // say what happened rather than what was said, so declining content cannot- // cost a destination the ability to read its own traces.- it("keeps metadata that shares a prefix with content", () => {- expect(- withoutDeclinedContent(ATTRIBUTES, { recordInputs: false, recordOutputs: false }),- ).toEqual({- "agent.trace.content.input": false,- "agent.trace.content.output": false,- "ai.response.finish_reason": "stop",- "gen_ai.request.model": "test-model",- "gen_ai.tool.name": "weather",- });- });-- it("leaves the attributes it was handed alone", () => {- const attributes = { ...ATTRIBUTES };- withoutDeclinedContent(attributes, { recordInputs: false, recordOutputs: false });- expect(attributes).toEqual(ATTRIBUTES);- });-- it("narrows content policy attributes even when no content is present", () => {- expect(- withoutDeclinedContent(- {- "agent.trace.content.input": true,- "agent.trace.content.output": true,- },- { recordInputs: false, recordOutputs: true },- ),- ).toEqual({- "agent.trace.content.input": false,- "agent.trace.content.output": true,- });- });-- it("redacts recalled memory records as input content", () => {- const searched = {- "gen_ai.memory.records": '[{"content":"Private preference"}]',- "gen_ai.operation.name": "search_memory",- };-- expect(withoutDeclinedContent(searched, { recordInputs: false, recordOutputs: true })).toEqual({- "gen_ai.operation.name": "search_memory",- });- expect(- withoutDeclinedContent(searched, { recordInputs: true, recordOutputs: false }),- ).toBeUndefined();- });-});diff --git a/packages/eve/src/tracing/content-attributes.ts b/packages/eve/src/tracing/content-attributes.tsdeleted file mode 100644index 2409ab0cf..000000000--- a/packages/eve/src/tracing/content-attributes.ts+++ /dev/null@@ -1,116 +0,0 @@-/**- * Which span attributes carry conversation content, and in which direction.- *- * Two vocabularies land on the same spans: the ones eve sets on its own- * `agent.*` spans, and the ones the AI SDK's OpenTelemetry integration sets on- * the model-call spans beneath them. A destination that declined content has to- * be rid of both, so both are listed here rather than in the module that writes- * each.- *- * Listed by name rather than by prefix because the prefixes are shared with- * metadata that must survive: `ai.response.finish_reason` and- * `gen_ai.tool.name` say what happened, not what was said. The cost is that a- * new content attribute in a dependency is not covered until it is added here.- */--/** Prompts, instructions, tool arguments — what went in. */-const INPUT_CONTENT_ATTRIBUTES: ReadonlySet<string> = new Set([- "agent.channel.delivery.input",- "agent.session.title",- "agent.approval.request",- "ai.documents",- "ai.prompt",- "ai.prompt.toolChoice",- "ai.prompt.tools",- "ai.value",- "ai.values",- "gen_ai.input.messages",- "gen_ai.memory.records",- "gen_ai.system_instructions",- "gen_ai.tool.call.arguments",- "gen_ai.tool.definitions",-]);--/**- * Responses, reasoning, tool results — what came out.- *- * `ai.toolCall.args` is here rather than above because the AI SDK gates it on- * `recordOutputs`; matching that is what keeps one destination's view coherent.- */-const OUTPUT_CONTENT_ATTRIBUTES: ReadonlySet<string> = new Set([- "agent.approval.response",- "ai.embedding",- "ai.embeddings",- "ai.ranking",- "ai.response.files",- "ai.response.object",- "ai.response.reasoning",- "ai.response.text",- "ai.response.toolCalls",- "ai.response.tool_calls",- "ai.response.tool_results",- "ai.toolCall.args",- "ai.toolCall.result",- "gen_ai.output.messages",- "gen_ai.tool.call.result",-]);--const INPUT_POLICY_ATTRIBUTE = "agent.trace.content.input";-const OUTPUT_POLICY_ATTRIBUTE = "agent.trace.content.output";--/** What one destination is willing to receive. */-export interface ResolvedContentOptions {- readonly recordInputs: boolean;- readonly recordOutputs: boolean;-}--function isDeclined(key: string, content: ResolvedContentOptions): boolean {- if (!content.recordInputs && INPUT_CONTENT_ATTRIBUTES.has(key)) return true;- return !content.recordOutputs && OUTPUT_CONTENT_ATTRIBUTES.has(key);-}--function narrowsContentPolicy(- attributes: Readonly<Record<string, unknown>>,- content: ResolvedContentOptions,-): boolean {- return (- (!content.recordInputs && attributes[INPUT_POLICY_ATTRIBUTE] === true) ||- (!content.recordOutputs && attributes[OUTPUT_POLICY_ATTRIBUTE] === true)- );-}--function exportedAttributeValue(- key: string,- value: unknown,- content: ResolvedContentOptions,-): unknown {- if (key === INPUT_POLICY_ATTRIBUTE && value === true && !content.recordInputs) return false;- if (key === OUTPUT_POLICY_ATTRIBUTE && value === true && !content.recordOutputs) return false;- return value;-}--/**- * The attributes with the declined content removed, or `undefined` when there- * was none to remove.- *- * The `undefined` return is what lets the caller forward the original span- * untouched in the common case, so a destination that declined a direction the- * span never carried costs one pass over its keys and no allocation.- */-export function withoutDeclinedContent(- attributes: Readonly<Record<string, unknown>>,- content: ResolvedContentOptions,-): Record<string, unknown> | undefined {- const keys = Object.keys(attributes);- if (!keys.some((key) => isDeclined(key, content)) && !narrowsContentPolicy(attributes, content)) {- return undefined;- }-- const kept: Record<string, unknown> = {};- for (const key of keys) {- if (!isDeclined(key, content)) {- kept[key] = exportedAttributeValue(key, attributes[key], content);- }- }- return kept;-}diff --git a/packages/eve/src/tracing/error-content-context.ts b/packages/eve/src/tracing/error-content-context.tsdeleted file mode 100644index d32415030..000000000--- a/packages/eve/src/tracing/error-content-context.ts+++ /dev/null@@ -1,11 +0,0 @@-import { context, createContextKey, type Context } from "#compiled/@opentelemetry/api/index.js";--const ERROR_CONTENT_KEY = createContextKey("eve.trace.error-content");--export function withErrorContent(context: Context, allowed: boolean): Context {- return context.setValue(ERROR_CONTENT_KEY, allowed);-}--export function capturesErrorContent(): boolean {- return context.active().getValue(ERROR_CONTENT_KEY) !== false;-}diff --git a/packages/eve/src/tracing/eve/agent-approval-instrumentation.ts b/packages/eve/src/tracing/eve/agent-approval-instrumentation.tsnew file mode 100644index 000000000..b641ed8bc--- /dev/null+++ b/packages/eve/src/tracing/eve/agent-approval-instrumentation.ts@@ -0,0 +1,63 @@+import type {+ InstrumentationHandlerContext,+ InstrumentationInputRequestedEvent,+ InstrumentationInputResolvedEvent,+ InstrumentationProviderDefinition,+} from "#instrumentation/lifecycle.js";+import { resumeTool } from "#tracing/eve/agent-tool-instrumentation.js";+import type { AgentActionTraceState } from "#tracing/eve/agent-trace-state.js";+import type { AgentTracing } from "#compiled/@eve/tracing/index.js";++/** Approval spans live under their tool call; hook state keeps only the locator. */+export function createAgentApprovalInstrumentation(input: {+ readonly tracing: AgentTracing;+ readonly actionStateFor: (+ sessionId: string,+ turnId: string,+ callId: string,+ ) => Promise<AgentActionTraceState | undefined>;+}): Pick<+ NonNullable<InstrumentationProviderDefinition["events"]>,+ "input.requested" | "input.resolved"+> {+ async function toolFor(sessionId: string, turnId: string, callId: string) {+ const state = await input.actionStateFor(sessionId, turnId, callId);+ return state === undefined ? undefined : resumeTool(input.tracing, state);+ }++ async function onRequested(+ event: InstrumentationInputRequestedEvent,+ ctx: InstrumentationHandlerContext,+ ): Promise<void> {+ if (event.kind !== "tool-approval" || ctx.state.get() !== undefined) return;+ const tool = await toolFor(event.scope.sessionId, event.scope.turnId, event.action.callId);+ const approval = await tool?.approval({ requestId: event.requestId, request: event.request });+ if (approval === undefined) return;+ ctx.state.set({+ callId: event.action.callId,+ sessionId: event.scope.sessionId,+ turnId: event.scope.turnId,+ });+ }++ async function onResolved(+ event: InstrumentationInputResolvedEvent,+ ctx: InstrumentationHandlerContext,+ ): Promise<void> {+ const stored = ctx.state.get();+ if (stored === null || typeof stored !== "object" || Array.isArray(stored)) return;+ const { callId, sessionId, turnId } = stored as Record<string, unknown>;+ if (typeof callId !== "string" || typeof sessionId !== "string" || typeof turnId !== "string")+ return;+ const approval = (await toolFor(sessionId, turnId, callId))?.findApproval(event.requestId);+ if (approval === undefined) return;+ if (event.outcome === "failed") await approval.fail(event.error);+ else+ await approval.complete({+ outcome: event.outcome as "approved" | "denied" | "cancelled" | "ignored" | "invalid",+ response: event.response,+ });+ }++ return { "input.requested": onRequested, "input.resolved": onResolved };+}diff --git a/packages/eve/src/tracing/eve/agent-channel-delivery-instrumentation.ts b/packages/eve/src/tracing/eve/agent-channel-delivery-instrumentation.tsindex 599afe3af..66e09242a 100644--- a/packages/eve/src/tracing/eve/agent-channel-delivery-instrumentation.ts+++ b/packages/eve/src/tracing/eve/agent-channel-delivery-instrumentation.ts@@ -9,9 +9,9 @@ import type {import { contextStorage } from "#context/container.js";import { ActiveChannelDeliveriesKey } from "#context/keys.js";import type { JsonValue } from "#shared/json.js";-import { contentAttribute } from "#tracing/agent-otel-content.js";-import type { AgentTraceStateStore } from "#tracing/agent-trace-state.js";-import { isSampledTrace } from "#tracing/sampled-trace.js";+import { contentAttribute } from "#compiled/@eve/tracing/otel.js";+import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";+import { isSampledTrace } from "#shared/trace-policy.js";interface ChannelDeliveryState {readonly inputAttribute?: string;@@ -33,11 +33,11 @@ export function createAgentChannelDeliveryInstrumentation(input: {event: InstrumentationChannelDeliveryStartedEvent,ctx: InstrumentationHandlerContext,): Promise<void> => {- const session = await input.stateStore.getSession(event.sessionId);+ const session = await input.stateStore.get("session", event.sessionId);const turn =event.turnId === undefined || event.sequence === undefined? undefined- : await input.stateStore.getTurn(event.sessionId, event.turnId);+ : await input.stateStore.get("turn", JSON.stringify([event.sessionId, event.turnId]));const traceContext = turn?.context ?? session?.context;if (traceContext === undefined || !isSampledTrace(traceContext)) return;const inputAttribute = input.recordInputs ? contentAttribute(event.input) : undefined;@@ -59,7 +59,10 @@ export function createAgentChannelDeliveryInstrumentation(input: {): Promise<void> => {const state = readState(ctx.state.get());if (state === undefined || event.turnId === undefined) return;- const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);+ const turn = await input.stateStore.get(+ "turn",+ JSON.stringify([event.sessionId, event.turnId]),+ );if (turn === undefined ||!isSampledTrace(turn.context) ||@@ -67,23 +70,27 @@ export function createAgentChannelDeliveryInstrumentation(input: {) {return;}- await input.stateStore.updateTurn(event.sessionId, event.turnId, (current) => ({- ...current,- channelDelivery: {- channelKind: event.delivery.channelKind,- channelName: event.delivery.channelName,- deliveryId: event.delivery.deliveryId,- ...(state.inputAttribute === undefined- ? undefined- : { inputAttribute: state.inputAttribute }),- ...(event.delivery.requestId === undefined- ? undefined- : { requestId: event.delivery.requestId }),- ...(state.requestTraceContext === undefined- ? undefined- : { requestTraceContext: state.requestTraceContext }),- },- }));+ await input.stateStore.update(+ "turn",+ JSON.stringify([event.sessionId, event.turnId]),+ (current) => ({+ ...current,+ channelDelivery: {+ channelKind: event.delivery.channelKind,+ channelName: event.delivery.channelName,+ deliveryId: event.delivery.deliveryId,+ ...(state.inputAttribute === undefined+ ? undefined+ : { inputAttribute: state.inputAttribute }),+ ...(event.delivery.requestId === undefined+ ? undefined+ : { requestId: event.delivery.requestId }),+ ...(state.requestTraceContext === undefined+ ? undefined+ : { requestTraceContext: state.requestTraceContext }),+ },+ }),+ );};return {diff --git a/packages/eve/src/tracing/eve/agent-invocation-coordinator.test.ts b/packages/eve/src/tracing/eve/agent-invocation-coordinator.test.tsindex 556873179..41ca2382d 100644--- a/packages/eve/src/tracing/eve/agent-invocation-coordinator.test.ts+++ b/packages/eve/src/tracing/eve/agent-invocation-coordinator.test.ts@@ -4,7 +4,7 @@ import { ContextContainer, contextStorage } from "#context/container.js";import { SessionTraceSeedKey } from "#context/keys.js";import { serializeContext } from "#context/serialize.js";import { resolveToolCallAgentTrace } from "#tracing/eve/agent-invocation-coordinator.js";-import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";+import { ContextAgentTraceStateStore } from "#tracing/eve/agent-trace-context-store.js";import { actionIdempotencyKey } from "#instrumentation/lifecycle.js";import type { ConversationContext } from "#shared/conversation-context.js";@@ -32,14 +32,14 @@ describe("resolveToolCallAgentTrace", () => {});await contextStorage.run(context, () => {const store = new ContextAgentTraceStateStore();- store.setTurn("session-1", "turn-1", {+ store.set("turn", JSON.stringify(["session-1", "turn-1"]), {context: action.parent,rootSessionId: "session-1",traceSessionId: "session-1",sequence: 0,startTimeMs: 1,});- if (recorded) store.setActionAnchor(actionKey, action);+ if (recorded) store.set("anchor", actionKey, action);});const dispatch = resolveToolCallAgentTrace({@@ -52,7 +52,7 @@ describe("resolveToolCallAgentTrace", () => {expect(dispatch.parentTraceContext).toMatchObject({...action.parent,- spanId: recorded ? action.spanId : action.parent.spanId,+ spanId: recorded ? action.context.spanId : action.parent.spanId,decision: { action: "record", recordInputs: false, recordOutputs: false },});expect(dispatch.originAudience).toBe("public");@@ -64,6 +64,7 @@ function workflowAction() {return {attemptIndex: 0,callId: "workflow",+ context: { spanId: "3".repeat(16), traceId: "1".repeat(32), traceFlags: 1 },channelAudience: "private" as const,kind: "tool-call" as const,name: "coordinate",@@ -73,10 +74,7 @@ function workflowAction() {traceId: "1".repeat(32),},rootSessionId: "session-1",- traceSessionId: "session-1",sessionId: "session-1",- spanId: "3".repeat(16),- startTimeMs: 1,stepIndex: 0,turnId: "turn-1",};diff --git a/packages/eve/src/tracing/eve/agent-invocation-coordinator.ts b/packages/eve/src/tracing/eve/agent-invocation-coordinator.tsindex 936891113..dfc7d615f 100644--- a/packages/eve/src/tracing/eve/agent-invocation-coordinator.ts+++ b/packages/eve/src/tracing/eve/agent-invocation-coordinator.ts@@ -10,7 +10,7 @@ import {import {readActionTraceContext,readTurnTraceContext,-} from "#tracing/agent-trace-context-store.js";+} from "#tracing/eve/agent-trace-context-store.js";export interface AgentChildTraceDispatch {readonly conversationId?: string;@@ -31,22 +31,9 @@ export function resolveToolCallAgentTrace(input: {readonly turnId: string;}): AgentChildTraceDispatch {const { serializedContext, sessionId, turnId } = input;- return toChildTraceDispatch({- conversation: input.conversation,- conversationId: readConversationId(serializedContext[ConversationIdKey.name]),- stored:- readActionTraceContext(serializedContext, sessionId, turnId, input.callId) ??- readTurnTraceContext(serializedContext, sessionId, turnId),- });-}--/** Caps a stored parent trace context to the live delivery audience. */-function toChildTraceDispatch(input: {- readonly conversation?: ConversationContext;- readonly conversationId?: string;- readonly stored?: SessionTraceContext;-}): AgentChildTraceDispatch {- const { stored } = input;+ const stored =+ readActionTraceContext(serializedContext, sessionId, turnId, input.callId) ??+ readTurnTraceContext(serializedContext, sessionId, turnId);const liveAudience = input.conversation?.audience ?? "unknown";const environment = input.conversation?.environment ?? "production";const forwardedTracePolicy = readForwardedTraceAssertion(stored?.forwardedTracePolicy);@@ -63,7 +50,7 @@ function toChildTraceDispatch(input: {),};return {- conversationId: input.conversationId,+ conversationId: readConversationId(serializedContext[ConversationIdKey.name]),originAudience: forwardedTracePolicy?.originAudience ?? liveAudience,parentTraceContext,};diff --git a/packages/eve/src/tracing/eve/agent-memory-instrumentation.ts b/packages/eve/src/tracing/eve/agent-memory-instrumentation.tsnew file mode 100644index 000000000..d51f69b5e--- /dev/null+++ b/packages/eve/src/tracing/eve/agent-memory-instrumentation.ts@@ -0,0 +1,85 @@+import { ROOT_CONTEXT, context, trace } from "#compiled/@opentelemetry/api/index.js";+import { contextStorage } from "#context/container.js";+import { SessionTraceSeedKey } from "#context/keys.js";+import type {+ InstrumentationMemoryExecutionOperation,+ MemoryInstrumentationResult,+} from "#instrumentation/memory.js";+import { normalizeChannelAudience } from "#shared/channel-audience.js";+import {+ applyLiveDeliveryAudienceCeiling,+ resolveForwardedTraceSeed,+} from "#shared/forwarded-trace-policy.js";+import type { ConversationEnvironment } from "#shared/conversation-context.js";+import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";+import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";+import { eveTurnIdentity } from "#tracing/eve/operation-input.js";+import { type AgentTracing } from "#compiled/@eve/tracing/index.js";+import { activeTraceOperation } from "#compiled/@eve/tracing/otel.js";++export function createAgentMemoryInstrumentation(input: {+ tracing: AgentTracing;+ environment: ConversationEnvironment;+ recordOutputs?: boolean;+ stateStore: AgentTraceStateStore;+}) {+ return {+ async runInContext<T>(+ operation: InstrumentationMemoryExecutionOperation,+ execute: () => PromiseLike<T>,+ ): Promise<T> {+ const session = await input.stateStore.get("session", operation.sessionId);+ const turn =+ operation.turnId === undefined+ ? undefined+ : await input.stateStore.get(+ "turn",+ JSON.stringify([operation.sessionId, operation.turnId]),+ );+ const active = activeTraceOperation();+ const reference = active?.reference ?? turn?.context ?? session?.context;+ if (reference === undefined) return await execute();+ const host =+ active === undefined+ ? withChannelAudience(+ trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(reference)),+ session?.channelAudience,+ )+ : context.active();+ const seed = resolveForwardedTraceSeed(contextStorage.getStore()?.get(SessionTraceSeedKey));+ const decision = seed?.decision ?? session?.decision;+ const effective =+ decision === undefined+ ? undefined+ : applyLiveDeliveryAudienceCeiling(+ decision,+ normalizeChannelAudience(session?.channelAudience),+ seed?.forwardedTracePolicy,+ input.environment,+ );+ const capture = {+ emit: (reference.traceFlags & 1) !== 0,+ recordInputs: effective?.action === "record" && effective.recordInputs,+ recordOutputs:+ input.recordOutputs === true && effective?.action === "record" && effective.recordOutputs,+ };+ const data = {+ identity: eveTurnIdentity({ ...operation, turnId: operation.turnId ?? "" }),+ parent: reference,+ context: host,+ capture,+ phase: operation.phase,+ slot: operation.slot,+ storeId: operation.storeId,+ describe(value: T) {+ const result = value as MemoryInstrumentationResult<unknown>;+ return { recordCount: result.recordCount, records: result.outputRecords };+ },+ };+ const memory = input.tracing.memory;+ return operation.operationName === "search_memory"+ ? await memory.search(data, execute)+ : await memory.write(data, execute);+ },+ };+}diff --git a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts b/packages/eve/src/tracing/eve/agent-otel-provider.test.tsindex 2ba86ad5d..d408c4fa7 100644--- a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/eve/agent-otel-provider.test.ts@@ -1,4 +1,10 @@-import { SpanKind, SpanStatusCode, trace as apiTrace } from "@opentelemetry/api";+import {+ SpanKind,+ SpanStatusCode,+ context as apiContext,+ trace as apiTrace,+} from "@opentelemetry/api";+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";import {BasicTracerProvider,InMemorySpanExporter,@@ -25,10 +31,12 @@ import { createActionResultEvent, createActionsRequestedEvent } from "#protocol/import {createAgentOtelInstrumentation,type AgentOtelInstrumentationInput,-} from "#tracing/agent-otel-provider.js";-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";-import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";-import type { AgentTraceStateStore } from "#tracing/agent-trace-state.js";+} from "#tracing/eve/agent-otel-provider.js";+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";+import { otelTelemetry } from "#compiled/@eve/tracing/index.js";+import { eveOutputMapping } from "#tracing/eve/profile.js";+import { ContextAgentTraceStateStore } from "#tracing/eve/agent-trace-context-store.js";+import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";import { InMemoryAgentTraceStateStore } from "#internal/testing/in-memory-agent-trace-state-store.js";import {createInstrumentationHooks,@@ -44,14 +52,10 @@ import {} from "#instrumentation/lifecycle.js";import type { ChannelAudience } from "#shared/channel-audience.js";import { channelAudienceFromContext } from "#tracing/eve/channel-audience-context.js";-import {- agentToolContentPolicy,- agentToolSpanContext,- annotateAgentToolSpan,-} from "#tracing/agent-tool-span-context.js";+import { activeTraceOperation } from "#compiled/@eve/tracing/otel.js";import { contentFilteringProcessor } from "#tracing/eve/content-span-processor.js";import { parseLocalTraceSegment } from "#tracing/local/trace-reader.js";-import { CONTENT_ATTRIBUTE_LIMIT } from "#tracing/agent-otel-content.js";+import { CONTENT_ATTRIBUTE_LIMIT } from "#compiled/@eve/tracing/otel.js";import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";import type { TraceCaptureContext } from "#shared/trace-policy.js";import {@@ -95,7 +99,7 @@ function createRuntime(recordOutputs: true,}),extraSpanProcessors: readonly SpanProcessor[] = [],- samplesTrace?: AgentOtelInstrumentationInput["samplesTrace"],+ samplesTrace?: import("#tracing/eve/otel-registration.js").RegisteredOtelPipeline["samplesTrace"],): TestRuntime {const exporter = new InMemorySpanExporter();const idGenerator = new AgentSpanIdGenerator();@@ -110,12 +114,17 @@ function createRuntime(// Pin the environment so an inherited EVE_DEV/VERCEL_ENV cannot relax audience ceilings.environment: "production",frameworkVersion: "test",- idGenerator,recordInputs: true,recordOutputs: true,- samplesTrace,stateStore,- tracer,+ telemetry: otelTelemetry({+ provider,+ tracerName: "eve.agent",+ idGenerator,+ samplesTrace: samplesTrace ?? (() => true),+ mapping: eveOutputMapping(),+ }),+ idGenerator,};if (tracePolicy !== null) agentOtelInput.tracePolicy = tracePolicy;const agentOtel = createAgentOtelInstrumentation(agentOtelInput);@@ -493,54 +502,6 @@ describe("createAgentOtelInstrumentation", () => {},);- it.each(["completed", "failed", "cancelled"] as const)(- "exports the %s turn outcome after content redaction and event removal",- async (outcome) => {- const metadata = new InMemorySpanExporter();- const runtime = createRuntime(undefined, undefined, [- contentFilteringProcessor(new SimpleSpanProcessor(metadata), {- span: () => ({ redact: true, inputs: true, outputs: true }),- }),- ]);- await publishTurnStarted({- hooks: runtime.hooks,- sessionId: "session-1",- turnId: "turn_0",- turnSequence: 0,- });- const identity = {- idempotencyKey: turnIdempotencyKey("session-1", "turn_0"),- sessionId: "session-1",- turnId: "turn_0",- };- await runtime.hooks.publish(- outcome === "failed"- ? { ...identity, type: "turn.failed", error: new Error("private failure") }- : { ...identity, type: outcome === "completed" ? "turn.completed" : "turn.cancelled" },- );- await runtime.hooks.publish({- idempotencyKey: sessionIdempotencyKey("session-1"),- sessionId: "session-1",- turnId: "turn_0",- type: "session.waiting",- });- await runtime.provider.forceFlush();- const spans = metadata.getFinishedSpans();- const serialized = new TextDecoder().decode(- JsonTraceSerializer.serializeRequest(- spans.map((span) => ({ ...span, events: [], spanContext: () => span.spanContext() })),- )!,- );- const turn = parseLocalTraceSegment(serialized, spans[0]!.spanContext().traceId).find(- (span) => span.name === "invoke_agent weather",- )!;- expect(turn.attributes["agent.turn.outcome"]).toBe(outcome);- expect(turn.statusCode).toBe(outcome === "failed" ? 2 : 0);- expect(serialized).not.toContain("private failure");- await runtime.provider.shutdown();- },- );-it("prepares a stable stream trace before lifecycle hooks observe the turn", async () => {const runtime = createRuntime();const sessionEvent = {@@ -612,7 +573,7 @@ describe("createAgentOtelInstrumentation", () => {turnId: "turn-1",type: "turn.started",});- const turn = await store.getTurn("session-1", "turn-1");+ const turn = await store.get("turn", JSON.stringify(["session-1", "turn-1"]));expect(turn?.currentPrincipal).toStrictEqual(includesId ? { id: "current-secret", type: "service" } : { type: "service" },);@@ -662,12 +623,14 @@ describe("createAgentOtelInstrumentation", () => {const contextWith = vi.spyOn(context, "with");await runtime.runInContext({- idempotencyKey: operation.idempotencyKey,- sessionId: operation.sessionId,- turnId: operation.turnId,+ ...operation,type: "memory.operation",},- async () => undefined,+ async () => ({+ value: undefined,+ recordCount: 1,+ outputRecords: [{ content: "The user prefers dark mode.", id: "preference" }],+ }),);const memoryContext = contextWith.mock.calls[0]?.[0];contextWith.mockRestore();@@ -847,7 +810,6 @@ describe("createAgentOtelInstrumentation", () => {[],samplesTrace,);- const setSession = vi.spyOn(store, "setSession");const sessionId = "session-early";await runtime.prepareSessionTrace({agentName: "weather",@@ -900,7 +862,6 @@ describe("createAgentOtelInstrumentation", () => {"agent.session.origin": "schedule","agent.session.title": "Prepared before sampling",});- expect(setSession).toHaveBeenCalledTimes(1);});it("falls back to fresh ids when no trace seed is present", async () => {@@ -1089,104 +1050,109 @@ describe("createAgentOtelInstrumentation", () => {},);- it("maps one channel delivery onto its activation with an HTTP request link", async () => {- const runtime = createRuntime();- const ctx = new ContextContainer();- const requestTraceContext = {- spanId: "2222222222222222",- traceFlags: 1,- traceId: "22222222222222222222222222222222",- };- const started = {- agentName: "support",- delivery: {+ it.each(["private", "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)])(+ "maps a channel delivery onto its activation despite bounded content (case %#)",+ async (message) => {+ const runtime = createRuntime();+ const ctx = new ContextContainer();+ const requestTraceContext = {+ spanId: "2222222222222222",+ traceFlags: 1,+ traceId: "22222222222222222222222222222222",+ };+ const started = {+ agentName: "support",+ delivery: {+ channelAudience: "public" as const,+ channelKind: "channel:slack",+ channelName: "slack",+ deliveryId: "delivery-1",+ requestId: "iad1::request-1",+ requestTraceContext,+ },+ idempotencyKey: "channel-delivery:session-1:delivery-1",+ input: { message },+ rootSessionId: "session-1",+ sequence: 0,+ sessionId: "session-1",+ turnId: "turn_0",+ type: "channel.delivery.started" as const,+ };+ const completed = {+ agentName: started.agentName,+ delivery: started.delivery,+ idempotencyKey: started.idempotencyKey,+ outcome: "completed" as const,+ rootSessionId: started.rootSessionId,+ sequence: 0,+ sessionId: started.sessionId,+ turnId: "turn_0",+ type: "channel.delivery.completed" as const,+ };+ const sessionEvent = {+ agentName: "support",channelAudience: "public" as const,channelKind: "channel:slack",- channelName: "slack",- deliveryId: "delivery-1",- requestId: "iad1::request-1",- requestTraceContext,- },- idempotencyKey: "channel-delivery:session-1:delivery-1",- input: { message: "private" },- rootSessionId: "session-1",- sequence: 0,- sessionId: "session-1",- turnId: "turn_0",- type: "channel.delivery.started" as const,- };- const completed = {- agentName: started.agentName,- delivery: started.delivery,- idempotencyKey: started.idempotencyKey,- outcome: "completed" as const,- rootSessionId: started.rootSessionId,- sequence: 0,- sessionId: started.sessionId,- turnId: "turn_0",- type: "channel.delivery.completed" as const,- };- const sessionEvent = {- agentName: "support",- channelAudience: "public" as const,- channelKind: "channel:slack",- idempotencyKey: sessionIdempotencyKey("session-1"),- rootSessionId: "session-1",- sessionId: "session-1",- type: "session.started" as const,- };- const turnEvent = {- idempotencyKey: turnIdempotencyKey("session-1", "turn_0"),- rootSessionId: "session-1",- sequence: 0,- sessionId: "session-1",- turnId: "turn_0",- type: "turn.started" as const,- };+ idempotencyKey: sessionIdempotencyKey("session-1"),+ rootSessionId: "session-1",+ sessionId: "session-1",+ type: "session.started" as const,+ };+ const turnEvent = {+ idempotencyKey: turnIdempotencyKey("session-1", "turn_0"),+ rootSessionId: "session-1",+ sequence: 0,+ sessionId: "session-1",+ turnId: "turn_0",+ type: "turn.started" as const,+ };- await contextStorage.run(ctx, async () => {- await runtime.prepareSessionTrace(sessionEvent);- await runtime.prepareTurnTrace(turnEvent);- await runtime.hooks.publish(started);- await runtime.hooks.publish(sessionEvent);- await runtime.hooks.publish(turnEvent);- ctx.set(ActiveChannelDeliveriesKey, [- {- agentName: started.agentName,- delivery: started.delivery,- policyAgentName: started.agentName,- rootSessionId: started.rootSessionId,- traceSessionId: started.rootSessionId,- sequence: 0,- sessionId: started.sessionId,- turnId: "turn_0",- },- ]);- await runtime.hooks.publish(completed);- await completeTurn(runtime.hooks, "session-1", "turn_0");- });+ await contextStorage.run(ctx, async () => {+ await runtime.prepareSessionTrace(sessionEvent);+ await runtime.prepareTurnTrace(turnEvent);+ await runtime.hooks.publish(started);+ await runtime.hooks.publish(sessionEvent);+ await runtime.hooks.publish(turnEvent);+ ctx.set(ActiveChannelDeliveriesKey, [+ {+ agentName: started.agentName,+ delivery: started.delivery,+ policyAgentName: started.agentName,+ rootSessionId: started.rootSessionId,+ traceSessionId: started.rootSessionId,+ sequence: 0,+ sessionId: started.sessionId,+ turnId: "turn_0",+ },+ ]);+ await runtime.hooks.publish(completed);+ await completeTurn(runtime.hooks, "session-1", "turn_0");+ });- const spans = runtime.exporter.getFinishedSpans();- const turn = spans.find((span) => span.name === "invoke_agent support")!;- expect(byName(spans, "agent.channel.delivery")).toHaveLength(0);- expect(turn.parentSpanContext).toBeUndefined();- expect(turn.links).toEqual([- expect.objectContaining({- attributes: { "eve.link.type": "channel.request" },- context: expect.objectContaining(requestTraceContext),- }),- ]);- expect(turn.attributes).toMatchObject({- "agent.channel.delivery.id": "delivery-1",- "agent.channel.delivery.input": JSON.stringify({ message: "private" }),- "agent.channel.kind": "channel:slack",- "agent.channel.name": "slack",- "agent.channel.request.id": "iad1::request-1",- "agent.turn.id": "turn_0",- "agent.turn.sequence": 0,- "gen_ai.conversation.id": "session-1",- });- });+ const spans = runtime.exporter.getFinishedSpans();+ const turn = spans.find((span) => span.name === "invoke_agent support")!;+ expect(byName(spans, "agent.channel.delivery")).toHaveLength(0);+ expect(turn.parentSpanContext).toBeUndefined();+ expect(turn.links).toEqual([+ expect.objectContaining({+ attributes: { "eve.link.type": "channel.request" },+ context: expect.objectContaining(requestTraceContext),+ }),+ ]);+ expect(turn.attributes).toMatchObject({+ "agent.channel.delivery.id": "delivery-1",+ "agent.channel.kind": "channel:slack",+ "agent.channel.name": "slack",+ "agent.channel.request.id": "iad1::request-1",+ "agent.turn.id": "turn_0",+ "agent.turn.sequence": 0,+ "gen_ai.conversation.id": "session-1",+ });+ if (message === "private")+ expect(turn.attributes["agent.channel.delivery.input"]).toBe(JSON.stringify({ message }));+ else expect(turn.attributes["agent.channel.delivery.input"]).toBeUndefined();+ },+ );it("does not emit delivery spans when remote deliveries fan in before the session event", async () => {const runtime = createRuntime();@@ -1333,11 +1299,11 @@ describe("createAgentOtelInstrumentation", () => {turnId: "turn_0",type: "channel.delivery.started",});- expect(stateStore.getSession(sessionId)).toBeUndefined();+ expect(stateStore.get("session", sessionId)).toBeUndefined();await runtime.prepareSessionTrace(sessionEvent);await runtime.prepareTurnTrace(turnEvent);- expect(stateStore.getSession(sessionId)).toMatchObject({+ expect(stateStore.get("session", sessionId)).toMatchObject({channelType: "http",scheduleId: "sched-1",title: "Investigate the incident",@@ -1392,12 +1358,12 @@ describe("createAgentOtelInstrumentation", () => {turnId: "turn_0",type: "channel.delivery.started",});- expect(stateStore.getSession(sessionId)).toBeUndefined();+ expect(stateStore.get("session", sessionId)).toBeUndefined();await runtime.prepareSessionTrace(sessionEvent);await runtime.hooks.publish(sessionEvent);- expect(stateStore.getSession(sessionId)).toMatchObject({+ expect(stateStore.get("session", sessionId)).toMatchObject({channelType: "http",parentLineage,});@@ -1673,7 +1639,7 @@ describe("createAgentOtelInstrumentation", () => {it("uses the stored session decision when no event or context seed exists", async () => {const stateStore = new InMemoryAgentTraceStateStore();- stateStore.setSession("session-policy", {+ stateStore.set("session", "session-policy", {channelAudience: "public",context: spanContext("1", "2"),decision: { action: "record", recordInputs: false, recordOutputs: true },@@ -1689,7 +1655,7 @@ describe("createAgentOtelInstrumentation", () => {it("preserves a private decision already intersected with a forwarded ceiling", async () => {const stateStore = new InMemoryAgentTraceStateStore();- stateStore.setSession("session-forwarded-private", {+ stateStore.set("session", "session-forwarded-private", {channelAudience: "private",context: spanContext("1", "2"),decision: { action: "record", recordInputs: true, recordOutputs: false },@@ -1811,7 +1777,7 @@ describe("createAgentOtelInstrumentation", () => {it("reconstructs a sampled public session decision when persisted policy is absent", async () => {const stateStore = new InMemoryAgentTraceStateStore();- stateStore.setSession("session-sampled", {+ stateStore.set("session", "session-sampled", {channelAudience: "public",context: spanContext("1", "2"),rootSessionId: "session-sampled",@@ -1828,7 +1794,7 @@ describe("createAgentOtelInstrumentation", () => {it("reconstructs an unsampled session as content-redacted", async () => {const stateStore = new InMemoryAgentTraceStateStore();- stateStore.setSession("session-unsampled", {+ stateStore.set("session", "session-unsampled", {channelAudience: "public",context: { ...spanContext("1", "2"), traceFlags: 0 },rootSessionId: "session-unsampled",@@ -2423,6 +2389,45 @@ describe("createAgentOtelInstrumentation", () => {withSpy.mockRestore();});+ it("records no third-party spans from a tool call in a dropped turn", async () => {+ const runtime = createRuntime(undefined, () => false);+ const scope: InstrumentationAttemptScope = {+ attemptId: "session-dropped:turn-1:0:0",+ attemptIndex: 0,+ channelAudience: "private",+ functionId: "weather",+ sessionId: "session-dropped",+ stepIndex: 0,+ turnId: "turn-1",+ };+ apiContext.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());+ try {+ await publishTurnStarted({+ channelAudience: "private",+ hooks: runtime.hooks,+ sessionId: scope.sessionId,+ turnId: scope.turnId,+ turnSequence: 0,+ });+ // An MCP client span started while Alice's dropped turn calls a tool.+ await runtime.runInContext(+ {+ idempotencyKey: "tool:session-dropped:turn-1:tool-1",+ scope,+ type: "tool.call",+ callId: "tool-1",+ toolName: "linear__get_issue",+ input: { issue: "ISSUE-1" },+ },+ async () => runtime.tracer.startSpan("mcp tools/call").end(),+ );+ await runtime.provider.forceFlush();+ expect(runtime.exporter.getFinishedSpans()).toHaveLength(0);+ } finally {+ apiContext.disable();+ }+ });+it("enriches the existing tool span with MCP attributes under its content policy", async () => {const runtime = createRuntime(new InMemoryAgentTraceStateStore(), () => ({emit: true,@@ -2476,20 +2481,16 @@ describe("createAgentOtelInstrumentation", () => {async () => {const active = withSpy.mock.calls[0]?.[0];expect(active).toBeDefined();- expect(agentToolContentPolicy(active!)).toEqual({+ expect(activeTraceOperation(active!)?.capture).toMatchObject({recordInputs: false,recordOutputs: false,});- annotateAgentToolSpan(- {- "eve.connection.name": "linear",- "gen_ai.operation.name": "execute_tool",- "gen_ai.tool.name": "get_issue",- "jsonrpc.request.id": "7",- "mcp.method.name": "tools/call",- },- active!,- );+ activeTraceOperation(active!)?.mcp?.update({+ connectionName: "linear",+ method: "tools/call",+ toolName: "get_issue",+ requestId: "7",+ });},);withSpy.mockRestore();@@ -2917,9 +2918,14 @@ describe("createAgentOtelInstrumentation", () => {});const agentOtel = createAgentOtelInstrumentation({frameworkVersion: "test",- idGenerator,stateStore: new InMemoryAgentTraceStateStore(),- tracer: provider.getTracer("eve.agent"),+ telemetry: otelTelemetry({+ provider,+ tracerName: "eve.agent",+ idGenerator,+ mapping: eveOutputMapping(),+ }),+ idGenerator,});const hooks = createInstrumentationHooks([agentOtel.hook]).forTrace!(traceContext("public"));await emitAttempt({@@ -2988,7 +2994,7 @@ describe("createAgentOtelInstrumentation", () => {expect(withSpy).toHaveBeenCalledOnce();const active = withSpy.mock.calls[0]?.[0];expect(active).toBeDefined();- expect(agentToolSpanContext(active!)).toBeUndefined();+ expect(activeTraceOperation(active!)?.type).toBe("model");withSpy.mockRestore();});@@ -3251,22 +3257,22 @@ describe("createAgentOtelInstrumentation", () => {input: undefined,kind: "subagent-call",});- const action = (await store.getAction(key))!;+ const action = (await store.get("action", key))!;await publishTurnStarted({hooks: runtime.hooks,sessionId: "child",rootSessionId: "parent",turnId: "turn_0",turnSequence: 0,- parentTraceContext: { ...action.parent, spanId: action.spanId },+ parentTraceContext: action.context,});await completeTurn(runtime.hooks, "child", "turn_0");await runtime.provider.forceFlush();const spans = runtime.exporter.getFinishedSpans();expect(byName(spans, "agent.action")).toHaveLength(0);const child = byName(spans, "invoke_agent weather")[0]!;- expect(child.parentSpanContext?.spanId).toBe(action.spanId);- expect(child.spanContext().traceId).toBe(action.parent.traceId);+ expect(child.parentSpanContext?.spanId).toBe(action.context.spanId);+ expect(child.spanContext().traceId).toBe(action.context.traceId);});it("starts a remote root with a dispatch link and receiver sampling", async () => {@@ -3459,31 +3465,3 @@ function modelStartedEvent(type: "model.call.started",};}--describe("AgentSpanIdGenerator.withTraceId", () => {- it("primes the next generateTraceId call", () => {- const gen = new AgentSpanIdGenerator();- const primed = "e".repeat(32);- const result = gen.withTraceId(primed, () => gen.generateTraceId());- expect(result).toBe(primed);- // After the callback, a fresh call should produce a different id.- const next = gen.generateTraceId();- expect(next).not.toBe(primed);- });-- it("nests inside withSpanId to prime both ids", () => {- const gen = new AgentSpanIdGenerator();- const primedTraceId = "f".repeat(32);- const primedSpanId = "a".repeat(16);- let capturedTraceId: string | undefined;- let capturedSpanId: string | undefined;- gen.withSpanId(primedSpanId, () =>- gen.withTraceId(primedTraceId, () => {- capturedTraceId = gen.generateTraceId();- capturedSpanId = gen.generateSpanId();- }),- );- expect(capturedTraceId).toBe(primedTraceId);- expect(capturedSpanId).toBe(primedSpanId);- });-});diff --git a/packages/eve/src/tracing/eve/agent-otel-provider.ts b/packages/eve/src/tracing/eve/agent-otel-provider.tsnew file mode 100644index 000000000..7c42b8cdc--- /dev/null+++ b/packages/eve/src/tracing/eve/agent-otel-provider.ts@@ -0,0 +1,295 @@+import { ROOT_CONTEXT, context, trace } from "#compiled/@opentelemetry/api/index.js";++import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";+import { createAgentApprovalInstrumentation } from "#tracing/eve/agent-approval-instrumentation.js";+import { createAgentChannelDeliveryInstrumentation } from "#tracing/eve/agent-channel-delivery-instrumentation.js";+import { createAgentToolInstrumentation } from "#tracing/eve/agent-tool-instrumentation.js";+import { eveActivationMetadata } from "#tracing/eve/metadata.js";+import { createAgentMemoryInstrumentation } from "#tracing/eve/agent-memory-instrumentation.js";+import { readGatewayCostData } from "#tracing/eve/gateway.js";+import { createEveSessionTracing } from "#tracing/eve/session.js";+import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";+import { isSampledTrace } from "#shared/trace-policy.js";+import { createEveCapturePolicy } from "#tracing/eve/policy.js";+import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";+import type {+ InstrumentationStepAttemptMetadataEvent,+ InstrumentationAttemptScope,+ InstrumentationStepAttemptStartedEvent,+ InstrumentationStepAttemptTerminalEvent,+ InstrumentationContextRunner,+ InstrumentationModelCallTerminalEvent,+ InstrumentationModelCallStartedEvent,+ InstrumentationProviderDefinition,+ InstrumentationSessionStartedEvent,+ InstrumentationTraceSeed,+ InstrumentationSessionTransitionEvent,+ InstrumentationTurnStartedEvent,+} from "#instrumentation/lifecycle.js";+import { resolveInstrumentationEnvironment } from "#internal/application/dev-environment.js";+import type { ConversationEnvironment } from "#shared/conversation-context.js";+import { createEveTracing } from "#tracing/eve/operation-input.js";+import { gatewayCostAttributes } from "#tracing/eve/gateway.js";+import type {+ AgentSpanIdGenerator,+ AgentTelemetry,+ AttemptOperation,+ ModelOperation,+} from "#compiled/@eve/tracing/index.js";+import { isUserMessageKind } from "#harness/messages.js";+import { isObject } from "#shared/guards.js";++export interface AgentOtelInstrumentationInput {+ /** Span output for eve's agent spans; its `ids` make turns durable across workers. */+ readonly telemetry: AgentTelemetry;+ readonly idGenerator: AgentSpanIdGenerator;+ readonly environment?: ConversationEnvironment;+ /** Whether any destination records model and tool inputs. */+ readonly recordInputs?: boolean;+ /** Whether any destination records model and tool outputs. */+ readonly recordOutputs?: boolean;+ readonly frameworkVersion: string;+ readonly stateStore: AgentTraceStateStore;+ readonly tracePolicy?: TraceCapturePolicy;+}++/** OTel event definition and its trusted framework context runner. */+interface AgentOtelInstrumentation {+ readonly hook: InstrumentationProviderDefinition;+ readonly prepareSessionTrace: (+ event: InstrumentationSessionStartedEvent,+ ) => Promise<InstrumentationTraceSeed>;+ readonly prepareTurnTrace: (+ event: InstrumentationTurnStartedEvent,+ ) => Promise<InstrumentationTraceSeed>;+ readonly runInContext: InstrumentationContextRunner;+}++/** Creates OTel instrumentation for eve's structural and GenAI spans. */+export function createAgentOtelInstrumentation(+ input: AgentOtelInstrumentationInput,+): AgentOtelInstrumentation {+ const environment = input.environment ?? resolveInstrumentationEnvironment();+ const recordInputs = input.recordInputs ?? false;+ const recordOutputs = input.recordOutputs ?? false;+ const tracing = createEveTracing(input);+ // Attempts run in one process; a lost serverless worker retries the whole step.+ const steps = new Map<string, AttemptOperation>();+ const modelSpans = new Map<string, Map<string, ModelOperation>>();+ const sessionTracing = createEveSessionTracing({ ...input, environment, tracing });+ const { prepareSessionTrace, prepareTurnTrace } = sessionTracing;++ /** The attempt in its turn's tree, hydrating it when another process started it. */+ async function attemptFor(scope: InstrumentationAttemptScope) {+ const known = steps.get(scope.attemptId);+ if (known !== undefined) return known;+ const turn = await sessionTracing.turnFor(+ scope.sessionId,+ scope.turnId,+ withChannelAudience(ROOT_CONTEXT, scope.channelAudience),+ );+ const index = { stepIndex: scope.stepIndex, attempt: scope.attemptIndex };+ return turn?.findAttempt(index) ?? (turn?.finished === false ? turn.attempt(index) : undefined);+ }++ const tools = createAgentToolInstrumentation({+ tracing,+ attemptFor,+ stepFor: (scope) => steps.get(scope.attemptId),+ recordInputs,+ recordOutputs,+ stateStore: input.stateStore,+ });+ const approvals = createAgentApprovalInstrumentation({+ tracing,+ actionStateFor: tools.stateFor,+ });+ const memory = createAgentMemoryInstrumentation({ ...input, environment, tracing });++ const capturePolicy = createEveCapturePolicy({+ stateStore: input.stateStore,+ environment,+ recordInputs,+ recordOutputs,+ });++ function turnState(scope: { sessionId: string; turnId: string }) {+ return input.stateStore.get("turn", JSON.stringify([scope.sessionId, scope.turnId]));+ }++ const onSessionStarted = async (event: InstrumentationSessionStartedEvent): Promise<void> => {+ await prepareSessionTrace(event);+ };++ const onTurnStarted = async (event: InstrumentationTurnStartedEvent): Promise<void> => {+ await prepareTurnTrace(event);+ };++ const onStepStarted = async (event: InstrumentationStepAttemptStartedEvent): Promise<void> => {+ const turn = await turnState(event.scope);+ if (turn === undefined || !isSampledTrace(turn.context)) return;+ const session = await input.stateStore.get("session", event.scope.sessionId);+ const operation = await sessionTracing.turnFor(+ event.scope.sessionId,+ event.scope.turnId,+ withChannelAudience(ROOT_CONTEXT, event.scope.channelAudience),+ );+ if (operation === undefined || operation.finished) return;+ const activeSpanContext = trace.getSpan(context.active())?.spanContext();+ const attempt = await operation.attempt({+ stepIndex: event.scope.stepIndex,+ attempt: event.scope.attemptIndex,+ runtimeContext: event.runtimeContext,+ channel: eveActivationMetadata({ session, turn, sessionId: event.scope.sessionId }).channel,+ links:+ activeSpanContext === undefined || activeSpanContext.traceId === turn.context.traceId+ ? undefined+ : [{ relationship: "execution.delivery", context: activeSpanContext }],+ });+ steps.set(event.scope.attemptId, attempt);+ };++ const onStepTerminal = async (event: InstrumentationStepAttemptTerminalEvent): Promise<void> => {+ const failure = event.type === "step.attempt.failed" ? { error: event.error } : undefined;+ await drainOpenSpans(event);+ await tools.drain(event.scope.attemptId, failure);+ const attempt = steps.get(event.scope.attemptId) ?? (await attemptFor(event.scope));+ steps.delete(event.scope.attemptId);+ if (failure !== undefined) {+ // A failed attempt also fails the tool calls it dispatched.+ await attempt?.fail(failure.error);+ await tools.forgetAttempt(event.scope);+ } else await attempt?.complete();+ };++ const onSessionTransition = async (+ event: InstrumentationSessionTransitionEvent,+ ): Promise<void> => {+ await sessionTracing.sessionTransition(event);+ // `session.waiting` is not terminal — the session may resume with a new+ // turn that still needs its metadata — so only release session-scoped+ // state on terminal transitions.+ if (event.type === "session.completed" || event.type === "session.failed") {+ await tools.deleteForSession(event.sessionId);+ await input.stateStore.delete("session", event.sessionId);+ }+ };++ const onModelCallStarted = async (event: InstrumentationModelCallStartedEvent): Promise<void> => {+ const attempt = steps.get(event.scope.attemptId);+ if (attempt === undefined) return;+ const operation = await attempt.modelCall(+ {+ provider: event.model.provider,+ modelId: event.model.modelId,+ messages: recordInputs+ ? event.input?.messages.map((message) =>+ isObject(message) && message.role === "user"+ ? { ...message, kind: isUserMessageKind(message.kind) ? message.kind : undefined }+ : message,+ )+ : undefined,+ instructions: recordInputs ? event.input?.instructions : undefined,+ tools: recordInputs ? event.input?.tools : undefined,+ runtimeContext: event.runtimeContext,+ },+ event.idempotencyKey,+ );+ let models = modelSpans.get(event.scope.attemptId);+ if (models === undefined) modelSpans.set(event.scope.attemptId, (models = new Map()));+ models.set(event.idempotencyKey, operation);+ };++ const onModelCallTerminal = async (+ event: InstrumentationModelCallTerminalEvent,+ ): Promise<void> => {+ const models = modelSpans.get(event.scope.attemptId);+ const operation = models?.get(event.idempotencyKey);+ models?.delete(event.idempotencyKey);+ if (operation === undefined) return;+ if (event.type === "model.call.failed") await operation.fail(event.error);+ else+ await operation.complete({+ outcome: "completed",+ result: { ...event, content: recordOutputs ? event.content : undefined },+ });+ };++ const channelDeliveries = createAgentChannelDeliveryInstrumentation({+ recordInputs,+ stateStore: input.stateStore,+ });++ const onStepMetadata = (event: InstrumentationStepAttemptMetadataEvent): void => {+ const attempt = steps.get(event.scope.attemptId);+ if (attempt === undefined) return;+ // Vercel AI Gateway reports per-call cost in providerMetadata.gateway;+ // attributes exist only when it was actually the gateway serving the call.+ const cost = readGatewayCostData(event.providerMetadata);+ if (cost !== undefined) attempt.attributes(gatewayCostAttributes(cost));+ };++ return {+ hook: {+ events: {+ ...channelDeliveries,+ ...approvals,+ "step.attempt.completed": onStepTerminal,+ "step.attempt.failed": onStepTerminal,+ "step.attempt.metadata": onStepMetadata,+ "step.attempt.started": onStepStarted,+ "model.call.completed": onModelCallTerminal,+ "model.call.failed": onModelCallTerminal,+ "model.call.started": onModelCallStarted,+ "session.completed": onSessionTransition,+ "session.failed": onSessionTransition,+ "session.started": onSessionStarted,+ "session.waiting": onSessionTransition,+ ...tools.events,+ "turn.cancelled": sessionTracing.turnTerminal,+ "turn.completed": sessionTracing.turnTerminal,+ "turn.failed": sessionTracing.turnTerminal,+ "turn.started": onTurnStarted,+ },+ name: "eve.otel",+ projectEvent: capturePolicy.projectEvent,+ tracePolicy: () => ({ emit: true, recordInputs, recordOutputs }),+ },+ prepareSessionTrace,+ prepareTurnTrace,+ async runInContext(operation, execute) {+ if (operation.type === "memory.operation") return memory.runInContext(operation, execute);+ const ceiling = { emit: true, ...(await capturePolicy.forOperation(operation.scope)) };+ const inTurn = async () => {+ const turn = await sessionTracing.turnFor(+ operation.scope.sessionId,+ operation.scope.turnId,+ withChannelAudience(ROOT_CONTEXT, operation.scope.channelAudience),+ );+ if (turn !== undefined && !turn.finished) return turn.run(execute, ceiling);+ // Third-party spans, such as MCP calls, must not record a dropped turn's work.+ const { sessionId, turnId } = operation.scope;+ return input.telemetry.suppressed !== undefined &&+ (await sessionTracing.dropped(sessionId, turnId))+ ? input.telemetry.suppressed(execute)+ : execute();+ };+ if (operation.type === "tool.call")+ return tools.runInContext(operation, execute, ceiling, inTurn);+ const model = modelSpans.get(operation.scope.attemptId)?.get(operation.idempotencyKey);+ return model !== undefined && !model.finished ? model.run(execute, ceiling) : inTurn();+ },+ };++ async function drainOpenSpans(event: InstrumentationStepAttemptTerminalEvent): Promise<void> {+ for (const operation of modelSpans.get(event.scope.attemptId)?.values() ?? []) {+ if (event.type === "step.attempt.failed") await operation.fail(event.error);+ else+ await operation.complete({+ outcome: "completed",+ result: { finishReason: "unknown", usage: {} },+ });+ }+ modelSpans.delete(event.scope.attemptId);+ }+}diff --git a/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts b/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.tsindex 50ba807f8..0295d7fd7 100644--- a/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts+++ b/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts@@ -31,12 +31,15 @@ import {type InstrumentationAttemptScope,} from "#instrumentation/lifecycle.js";import { bindInstrumentationRuntime } from "#instrumentation/runtime.js";-import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js";-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";-import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";-import { AGENT_TRACE_CONTEXT_KEY } from "#tracing/agent-trace-context-codec.js";+import { createAgentOtelInstrumentation } from "#tracing/eve/agent-otel-provider.js";+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";+import { otelTelemetry } from "#compiled/@eve/tracing/index.js";+import { eveOutputMapping } from "#tracing/eve/profile.js";+import { ContextAgentTraceStateStore } from "#tracing/eve/agent-trace-context-store.js";+import { AGENT_TRACE_CONTEXT_KEY } from "#tracing/eve/agent-trace-context-store.js";import { resolveToolCallAgentTrace } from "#tracing/eve/agent-invocation-coordinator.js";import * as instrumentation from "#instrumentation/runtime.js";+import * as runtimeGlobal from "#instrumentation/runtime-global.js";import {assembleLocalTrace,isAgentTurnSpan,@@ -78,11 +81,16 @@ function createRuntime() {});const agent = createAgentOtelInstrumentation({frameworkVersion: "test",- idGenerator,recordInputs: true,recordOutputs: true,stateStore: new ContextAgentTraceStateStore(),- tracer: provider.getTracer("eve.agent"),+ telemetry: otelTelemetry({+ provider,+ tracerName: "eve.agent",+ idGenerator,+ mapping: eveOutputMapping(),+ }),+ idGenerator,});const hooks = createInstrumentationHooks([agent.hook]);return {@@ -1090,7 +1098,7 @@ describe("exported agent telemetry contract", () => {const runtime = createRuntime();const hooks = runtime.hooks.forTrace!(traceContext("child", "public"));const registered = vi- .spyOn(instrumentation, "getInstrumentationRuntime")+ .spyOn(runtimeGlobal, "getInstrumentationRuntime").mockReturnValue(runtime);let ctx = contextFor("public");ctx.set(ParentTraceContextKey, {diff --git a/packages/eve/src/tracing/eve/agent-tool-instrumentation.ts b/packages/eve/src/tracing/eve/agent-tool-instrumentation.tsnew file mode 100644index 000000000..103304ae5--- /dev/null+++ b/packages/eve/src/tracing/eve/agent-tool-instrumentation.ts@@ -0,0 +1,310 @@+import {+ ROOT_CONTEXT,+ type Context,+ type SpanContext,+ trace,+} from "#compiled/@opentelemetry/api/index.js";++import type {+ InstrumentationAttemptScope,+ InstrumentationExecutionOperation,+ InstrumentationToolCallStartedEvent,+ InstrumentationToolCallTerminalEvent,+} from "#instrumentation/lifecycle.js";+import { actionIdempotencyKey } from "#instrumentation/lifecycle.js";+import { normalizeChannelAudience } from "#shared/channel-audience.js";+import type {+ AgentActionTraceState,+ AgentTraceStateStore,+} from "#tracing/eve/agent-trace-state.js";+import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";+import { eveTurnIdentity } from "#tracing/eve/operation-input.js";+import type {+ AgentTracing,+ AttemptOperation,+ CaptureDecision,+ ToolInput,+ ToolOperation,+} from "#compiled/@eve/tracing/index.js";++export interface AgentToolContext {+ readonly context: Context;+ readonly spanContext: SpanContext;+}++type ToolExecution = Extract<InstrumentationExecutionOperation, { type: "tool.call" }>;++/** What the SDK reported about one tool run, kept until the dispatch or the step settles it. */+interface Execution {+ readonly attemptId: string;+ readonly idempotencyKey: string;+ readonly tool: ToolOperation;+ terminal?: {+ readonly completedAtMs: number;+ readonly durationMs?: number;+ readonly failed: boolean;+ readonly error?: unknown;+ readonly output?: unknown;+ };+}++const OPEN_EXECUTIONS = 10000;++/** Finds an open tool call in its turn's trace tree, in this or a later process. */+export async function resumeTool(+ tracing: AgentTracing,+ state: AgentActionTraceState,+): Promise<ToolOperation | undefined> {+ const turn = await tracing.resume({+ identity: eveTurnIdentity(state),+ context: withChannelAudience(ROOT_CONTEXT, state.channelAudience),+ });+ return turn?.findTool(state.callId);+}++/**+ * One `execute_tool` span per call. eve's tool loop reports the durable+ * dispatch through `tool.call.*` events; the SDK reports the run through the+ * execution context. Both name the same call ID, so they share one span. A run+ * without a dispatch closes under its step when the step ends.+ */+export function createAgentToolInstrumentation(input: {+ readonly tracing: AgentTracing;+ readonly attemptFor: (+ scope: InstrumentationAttemptScope,+ ) => Promise<AttemptOperation | undefined>;+ readonly stepFor: (scope: InstrumentationAttemptScope) => AttemptOperation | undefined;+ readonly recordInputs: boolean;+ readonly recordOutputs: boolean;+ readonly stateStore: AgentTraceStateStore;+}) {+ const executions = new Map<string, Execution>();+ const callKey = (scope: { sessionId: string; turnId: string }, callId: string) =>+ actionIdempotencyKey(scope.sessionId, scope.turnId, callId);++ async function stateFor(sessionId: string, turnId: string, callId: string) {+ const direct = await input.stateStore.get(+ "action",+ actionIdempotencyKey(sessionId, turnId, callId),+ );+ if (direct !== undefined) return direct;+ return (await input.stateStore.entries("action")).find(+ ([, state]) => state.sessionId === sessionId && state.callId === callId,+ )?.[1];+ }++ /** Reports the call again through its parent, so a later report refines the same span. */+ async function reenter(state: AgentActionTraceState, report: ToolInput) {+ const turn = await input.tracing.resume({+ identity: eveTurnIdentity(state),+ context: withChannelAudience(ROOT_CONTEXT, state.channelAudience),+ });+ if (turn?.findTool(state.callId) === undefined) return undefined;+ const parent = turn.findAttempt({ stepIndex: state.stepIndex, attempt: state.attemptIndex });+ return parent === undefined ? turn.findTool(state.callId) : parent.tool(report);+ }++ /** A dispatched call takes the run's timing and failure; its own terminal ends the span. */+ async function settleExecution(execution: Execution, dispatched: AgentActionTraceState) {+ const terminal = execution.terminal;+ if (terminal === undefined) return;+ executions.delete(execution.idempotencyKey);+ if (terminal.failed) execution.tool.recordError(terminal.error);+ else if (terminal.durationMs !== undefined)+ execution.tool.attributes({ "gen_ai.execute_tool.duration": terminal.durationMs / 1000 });+ await input.stateStore.set("action", execution.idempotencyKey, {+ ...dispatched,+ toolEndTimeMs: terminal.completedAtMs,+ });+ }++ async function onDispatchStarted(event: InstrumentationToolCallStartedEvent): Promise<void> {+ let state = await input.stateStore.get("action", event.idempotencyKey);+ if (state === undefined) {+ const parent = await input.attemptFor(event.scope);+ if (parent === undefined) return;+ const tool = await parent.tool({+ callId: event.callId,+ name: event.toolName,+ kind: event.kind ?? "tool-call",+ arguments: input.recordInputs ? event.input : undefined,+ startTimeMs: event.startedAtMs,+ });+ tool.attributes({ "agent.tool.is_framework": event.frameworkTool === true });+ state = {+ attemptId: event.scope.attemptId,+ attemptIndex: event.scope.attemptIndex,+ callId: event.callId,+ channelAudience: normalizeChannelAudience(event.scope.channelAudience),+ context: { ...tool.reference, isRemote: false },+ rootSessionId: event.scope.rootSessionId,+ sessionId: event.scope.sessionId,+ stepIndex: event.scope.stepIndex,+ turnId: event.scope.turnId,+ };+ await input.stateStore.set("action", event.idempotencyKey, state);+ const execution = executions.get(event.idempotencyKey);+ if (execution !== undefined) await settleExecution(execution, state);+ }+ if (event.isWorkflowTool === true) {+ await input.stateStore.set("anchor", event.idempotencyKey, state);+ }+ }++ async function onDispatchTerminal(event: InstrumentationToolCallTerminalEvent): Promise<void> {+ const state = await input.stateStore.get("action", event.idempotencyKey);+ if (state === undefined) return;+ try {+ const tool = await resumeTool(input.tracing, state);+ const error =+ event.type === "tool.call.failed"+ ? event.error+ : event.output.type === "error"+ ? event.output.error+ : undefined;+ await tool?.complete({+ outcome: event.outcome ?? (event.type === "tool.call.failed" ? "failed" : "completed"),+ failed: event.type === "tool.call.failed" || event.output.type === "error",+ errorType: event.type === "tool.call.failed" ? event.errorCode : undefined,+ error,+ output:+ input.recordOutputs &&+ event.type === "tool.call.completed" &&+ event.output.type === "result"+ ? event.output.output+ : undefined,+ usage: event.usage,+ endTimeMs: event.acceptedAtMs ?? state.toolEndTimeMs,+ });+ } finally {+ await input.stateStore.delete("action", event.idempotencyKey);+ }+ }++ async function executionFor(+ operation: ToolExecution & { callId: string; toolName: string },+ ceiling: CaptureDecision,+ ) {+ const key = callKey(operation.scope, operation.callId);+ const known = executions.get(key);+ if (known !== undefined) return known;+ const state = await stateFor(+ operation.scope.sessionId,+ operation.scope.turnId,+ operation.callId,+ );+ const report: ToolInput = {+ callId: operation.callId,+ name: operation.toolName,+ arguments: input.recordInputs && ceiling.recordInputs ? operation.input : undefined,+ startTimeMs: operation.startedAtMs,+ };+ const tool =+ state === undefined+ ? await input.stepFor(operation.scope)?.tool(report)+ : await reenter(state, report);+ if (tool === undefined || executions.size >= OPEN_EXECUTIONS) return undefined;+ const execution: Execution = {+ attemptId: operation.scope.attemptId,+ idempotencyKey: key,+ tool,+ };+ executions.set(key, execution);+ return execution;+ }++ async function finishExecution(execution: Execution, terminal: Execution["terminal"]) {+ execution.terminal = terminal;+ const key = execution.idempotencyKey;+ const dispatched = await input.stateStore.get("action", key);+ if (dispatched !== undefined) await settleExecution(execution, dispatched);+ }++ return {+ stateFor,+ async contextFor(sessionId: string, turnId: string, callId: string) {+ const state = await stateFor(sessionId, turnId, callId);+ return state === undefined ? undefined : toolContext(state);+ },+ async deleteForSession(sessionId: string) {+ for (const kind of ["action", "anchor"] as const)+ for (const [key, state] of await input.stateStore.entries(kind))+ if (state.sessionId === sessionId) await input.stateStore.delete(kind, key);+ },+ // A failed attempt closes its calls in the trace tree; only the locators remain.+ async forgetAttempt(scope: InstrumentationAttemptScope) {+ for (const [key, state] of await input.stateStore.entries("action"))+ if (state.attemptId === scope.attemptId) await input.stateStore.delete("action", key);+ },+ /** Closes SDK runs that no dispatch claimed before their step ended. */+ async drain(attemptId: string, failure?: { readonly error: unknown }) {+ for (const [key, execution] of executions) {+ if (execution.attemptId !== attemptId) continue;+ executions.delete(key);+ if ((await input.stateStore.get("action", key)) !== undefined) continue;+ const terminal = execution.terminal;+ if (failure !== undefined) {+ await execution.tool.fail(failure.error);+ continue;+ }+ if (terminal?.durationMs !== undefined && !terminal.failed)+ execution.tool.attributes({ "gen_ai.execute_tool.duration": terminal.durationMs / 1000 });+ await execution.tool.complete({+ failed: terminal?.failed,+ error: terminal?.error,+ output: terminal?.output,+ endTimeMs: terminal?.completedAtMs,+ });+ }+ },+ /** Runs the SDK's tool call inside its span, or calls `fallback` when it has none. */+ async runInContext<T>(+ operation: ToolExecution,+ execute: () => PromiseLike<T>,+ ceiling: CaptureDecision,+ fallback: () => Promise<T>,+ ): Promise<T> {+ if (operation.callId === undefined || operation.toolName === undefined) return fallback();+ const execution = await executionFor(+ { ...operation, callId: operation.callId, toolName: operation.toolName },+ ceiling,+ );+ if (execution === undefined || execution.tool.finished) return fallback();+ execution.tool.attributes({ "agent.tool.is_framework": operation.frameworkTool === true });+ const startedAtMs = operation.startedAtMs ?? Date.now();+ try {+ const result = await execution.tool.run(execute, ceiling);+ const completedAtMs = operation.completedAtMs ?? Date.now();+ await finishExecution(execution, {+ completedAtMs,+ durationMs: completedAtMs - startedAtMs,+ failed: operation.failed === true,+ output: input.recordOutputs && ceiling.recordOutputs ? result : undefined,+ });+ return result;+ } catch (error) {+ await finishExecution(execution, {+ completedAtMs: Date.now(),+ failed: true,+ error: ceiling.recordOutputs ? error : undefined,+ });+ throw error;+ }+ },+ events: {+ "tool.call.completed": onDispatchTerminal,+ "tool.call.failed": onDispatchTerminal,+ "tool.call.started": onDispatchStarted,+ },+ };+}++function toolContext(state: AgentActionTraceState): AgentToolContext {+ return {+ context: withChannelAudience(+ trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(state.context)),+ state.channelAudience,+ ),+ spanContext: state.context,+ };+}diff --git a/packages/eve/src/tracing/eve/agent-trace-context-store.test.ts b/packages/eve/src/tracing/eve/agent-trace-context-store.test.tsindex c24cd6cb1..1e5cd69ae 100644--- a/packages/eve/src/tracing/eve/agent-trace-context-store.test.ts+++ b/packages/eve/src/tracing/eve/agent-trace-context-store.test.ts@@ -3,29 +3,26 @@ import { describe, expect, it } from "vitest";import { ContextContainer, contextStorage } from "#context/container.js";import { SessionTraceSeedKey } from "#context/keys.js";import { deserializeContext, serializeContext } from "#context/serialize.js";-import { INSTRUMENTATION_PRINCIPAL_TYPES } from "#instrumentation/lifecycle.js";-import { deserializeAgentTraceContextState } from "#tracing/agent-trace-context-codec.js";-import { CONTENT_ATTRIBUTE_LIMIT } from "#tracing/agent-otel-content.js";import {ContextAgentTraceStateStore,preserveSerializedAgentTraceState,readActionTraceContext,readTurnTraceContext,-} from "#tracing/agent-trace-context-store.js";+} from "#tracing/eve/agent-trace-context-store.js";describe("ContextAgentTraceStateStore", () => {it("restores serializable session and turn context", async () => {const context = new ContextContainer();await contextStorage.run(context, () => {const store = new ContextAgentTraceStateStore();- store.setSession("session-1", {+ store.set("session", "session-1", {agentName: "weather",channelType: "http",context: spanContext("1", "2"),rootSessionId: "session-1",traceSessionId: "session-1",});- store.setTurn("session-1", "turn-1", {+ store.set("turn", JSON.stringify(["session-1", "turn-1"]), {channelDelivery: {channelKind: "channel:slack",channelName: "slack",@@ -37,16 +34,13 @@ describe("ContextAgentTraceStateStore", () => {context: spanContext("1", "3"),currentPrincipal: { id: "user-123", type: "user" },initiatorPrincipal: { type: "none" },- inputMessagesAttribute: '[{"parts":[{"content":"hello","type":"text"}],"role":"user"}]',- modelUsage: { inputTokens: 12, outputTokens: 4 },- outputMessagesAttribute: '[{"parts":[{"content":"hi","type":"text"}],"role":"assistant"}]',caller: { ...spanContext("4", "2"), isRemote: true },rootSessionId: "session-1",traceSessionId: "session-1",sequence: 0,startTimeMs: 1_700_000_000_000,subagentName: "researcher",- terminal: { error: new Error("failed"), type: "turn.failed" },+ terminal: { outcome: "failed", errorName: "Error", errorMessage: "failed" },});});@@ -54,12 +48,14 @@ describe("ContextAgentTraceStateStore", () => {const restored = await deserializeContext(serialized);await contextStorage.run(restored, () => {const store = new ContextAgentTraceStateStore();- expect(store.getSession("session-1")).toMatchObject({+ expect(store.get("session", "session-1")).toMatchObject({channelType: "http",context: spanContext("1", "2"),});- expect(store.getTurn("session-1", "turn-1")?.context).toEqual(spanContext("1", "3"));- expect(store.getTurn("session-1", "turn-1")).toMatchObject({+ expect(store.get("turn", JSON.stringify(["session-1", "turn-1"]))?.context).toEqual(+ spanContext("1", "3"),+ );+ expect(store.get("turn", JSON.stringify(["session-1", "turn-1"]))).toMatchObject({channelDelivery: {channelKind: "channel:slack",channelName: "slack",@@ -70,80 +66,31 @@ describe("ContextAgentTraceStateStore", () => {},currentPrincipal: { id: "user-123", type: "user" },initiatorPrincipal: { type: "none" },- inputMessagesAttribute: '[{"parts":[{"content":"hello","type":"text"}],"role":"user"}]',- modelUsage: { inputTokens: 12, outputTokens: 4 },- outputMessagesAttribute: '[{"parts":[{"content":"hi","type":"text"}],"role":"assistant"}]',+ terminal: { outcome: "failed", errorName: "Error", errorMessage: "failed" },caller: { ...spanContext("4", "2"), isRemote: true },startTimeMs: 1_700_000_000_000,subagentName: "researcher",});- expect(store.getTurn("session-1", "turn-1")?.initiatorPrincipal).toStrictEqual({+ expect(+ store.get("turn", JSON.stringify(["session-1", "turn-1"]))?.initiatorPrincipal,+ ).toStrictEqual({type: "none",});- expect(JSON.stringify(store.getTurn("session-1", "turn-1"))).not.toContain("attributes");- const terminal = store.getTurn("session-1", "turn-1")?.terminal;- expect(terminal?.type).toBe("turn.failed");- expect(terminal?.type === "turn.failed" ? terminal.error : undefined).toMatchObject({- message: "failed",- });- });- });-- it("rejects oversized persisted turn content attributes", () => {- const state = deserializeAgentTraceContextState({- turns: {- turn: {- context: spanContext("1", "2"),- inputMessagesAttribute: "x".repeat(CONTENT_ATTRIBUTE_LIMIT + 1),- outputMessagesAttribute: "x".repeat(CONTENT_ATTRIBUTE_LIMIT + 1),- rootSessionId: "session-1",- sequence: 0,- startTimeMs: 1,- },- },+ expect(+ JSON.stringify(store.get("turn", JSON.stringify(["session-1", "turn-1"]))),+ ).not.toContain("attributes");});-- expect(state.turns.turn).toBeDefined();- expect(state.turns.turn?.inputMessagesAttribute).toBeUndefined();- expect(state.turns.turn?.outputMessagesAttribute).toBeUndefined();- });-- it.each([- ...INSTRUMENTATION_PRINCIPAL_TYPES.map((type) => ({- input: { type, id: "user-123" },- expected: type === "none" ? { type } : { type, id: "user-123" },- })),- { input: { type: "unrecognized", id: "user-123" }, expected: undefined },- ...[undefined, null, 123, "", "x".repeat(1025), String.fromCodePoint(0x1f600).repeat(257)].map(- (id) => ({- input: { type: "user", id },- expected: { type: "user" },- }),- ),- ])("restores bounded principal summaries from $input", ({ input, expected }) => {- const state = deserializeAgentTraceContextState({- turns: {- turn: {- context: spanContext("1", "2"),- currentPrincipal: input,- initiatorPrincipal: input,- startTimeMs: 1,- },- },- });- expect(state.turns.turn?.currentPrincipal).toStrictEqual(expected);- expect(state.turns.turn?.initiatorPrincipal).toStrictEqual(expected);});it("removes terminal state", () => {contextStorage.run(new ContextContainer(), () => {const store = new ContextAgentTraceStateStore();- store.setSession("session-1", {+ store.set("session", "session-1", {context: spanContext("1", "2"),rootSessionId: "session-1",traceSessionId: "session-1",});- store.setTurn("session-1", "turn-1", {+ store.set("turn", JSON.stringify(["session-1", "turn-1"]), {context: spanContext("1", "3"),caller: spanContext("4", "2"),rootSessionId: "session-1",@@ -152,18 +99,18 @@ describe("ContextAgentTraceStateStore", () => {startTimeMs: 1_700_000_000_000,});- store.deleteTurn("session-1", "turn-1");- store.deleteSession("session-1");+ store.delete("turn", JSON.stringify(["session-1", "turn-1"]));+ store.delete("session", "session-1");- expect(store.getTurn("session-1", "turn-1")).toBeUndefined();- expect(store.getSession("session-1")).toBeUndefined();+ expect(store.get("turn", JSON.stringify(["session-1", "turn-1"]))).toBeUndefined();+ expect(store.get("session", "session-1")).toBeUndefined();});});it("composes atomic turn updates without recreating deleted turns", () => {contextStorage.run(new ContextContainer(), () => {const store = new ContextAgentTraceStateStore();- store.setTurn("session-1", "turn-1", {+ store.set("turn", JSON.stringify(["session-1", "turn-1"]), {context: spanContext("1", "3"),caller: spanContext("4", "2"),rootSessionId: "session-1",@@ -172,32 +119,32 @@ describe("ContextAgentTraceStateStore", () => {startTimeMs: 1_700_000_000_000,});- store.updateTurn("session-1", "turn-1", (turn) => ({+ store.update("turn", JSON.stringify(["session-1", "turn-1"]), (turn) => ({...turn,- modelUsage: { inputTokens: 12, outputTokens: 4 },+ terminal: { outcome: "completed" },}));- store.updateTurn("session-1", "turn-1", (turn) => ({+ store.update("turn", JSON.stringify(["session-1", "turn-1"]), (turn) => ({...turn,- terminal: { type: "turn.completed" },+ channelDelivery: { channelKind: "http", channelName: "http", deliveryId: "delivery" },}));- expect(store.getTurn("session-1", "turn-1")).toMatchObject({- modelUsage: { inputTokens: 12, outputTokens: 4 },- terminal: { type: "turn.completed" },+ expect(store.get("turn", JSON.stringify(["session-1", "turn-1"]))).toMatchObject({+ terminal: { outcome: "completed" },+ channelDelivery: { deliveryId: "delivery" },});- store.deleteTurn("session-1", "turn-1");- store.updateTurn("session-1", "turn-1", (turn) => ({+ store.delete("turn", JSON.stringify(["session-1", "turn-1"]));+ store.update("turn", JSON.stringify(["session-1", "turn-1"]), (turn) => ({...turn,- modelUsage: { inputTokens: 99 },+ terminal: { outcome: "cancelled" },}));- expect(store.getTurn("session-1", "turn-1")).toBeUndefined();+ expect(store.get("turn", JSON.stringify(["session-1", "turn-1"]))).toBeUndefined();});});it("preserves only trace state from an interrupted context", async () => {const context = new ContextContainer();await contextStorage.run(context, () => {- new ContextAgentTraceStateStore().setSession("session-1", {+ new ContextAgentTraceStateStore().set("session", "session-1", {context: spanContext("1", "2"),rootSessionId: "session-1",traceSessionId: "session-1",@@ -226,13 +173,13 @@ describe("readTurnTraceContext", () => {traceFlags: 1,traceId: "1".repeat(32),});- new ContextAgentTraceStateStore().setSession("session-1", {+ new ContextAgentTraceStateStore().set("session", "session-1", {context: spanContext("1", "2"),decision: { action: "record", recordInputs: true, recordOutputs: false },rootSessionId: "session-1",traceSessionId: "session-1",});- new ContextAgentTraceStateStore().setTurn("session-1", "turn-1", {+ new ContextAgentTraceStateStore().set("turn", JSON.stringify(["session-1", "turn-1"]), {context: spanContext("3", "4"),rootSessionId: "session-1",traceSessionId: "session-1",@@ -258,13 +205,13 @@ describe("readTurnTraceContext", () => {const context = new ContextContainer();await contextStorage.run(context, () => {const store = new ContextAgentTraceStateStore();- store.setSession("session-1", {+ store.set("session", "session-1", {context: spanContext("1", "2"),decision: { action: "record", recordInputs: false, recordOutputs: true },rootSessionId: "session-1",traceSessionId: "session-1",});- store.setTurn("session-1", "turn-1", {+ store.set("turn", JSON.stringify(["session-1", "turn-1"]), {context: spanContext("3", "4"),rootSessionId: "session-1",traceSessionId: "session-1",@@ -294,19 +241,8 @@ describe("readActionTraceContext", () => {traceFlags: 1,traceId: "1".repeat(32),});- new ContextAgentTraceStateStore().setAction("action:session-1:turn-1:call-1", {- attemptIndex: 0,- callId: "call-1",- kind: "subagent-call",- name: "researcher",- parent: spanContext("1", "2"),- rootSessionId: "session-1",- traceSessionId: "session-1",- sessionId: "session-1",- spanId: "3".repeat(16),- startTimeMs: 1_700_000_000_000,- stepIndex: 0,- turnId: "turn-1",+ new ContextAgentTraceStateStore().set("action", "action:session-1:turn-1:call-1", {+ ...actionState(),});});const serialized = await serializeContext(context);@@ -329,25 +265,14 @@ describe("readActionTraceContext", () => {const context = new ContextContainer();await contextStorage.run(context, () => {const store = new ContextAgentTraceStateStore();- store.setSession("session-1", {+ store.set("session", "session-1", {context: spanContext("1", "2"),decision: { action: "record", recordInputs: true, recordOutputs: false },rootSessionId: "session-1",traceSessionId: "session-1",});- store.setAction("action:session-1:turn-1:call-1", {- attemptIndex: 0,- callId: "call-1",- kind: "subagent-call",- name: "researcher",- parent: spanContext("1", "2"),- rootSessionId: "session-1",- traceSessionId: "session-1",- sessionId: "session-1",- spanId: "3".repeat(16),- startTimeMs: 1_700_000_000_000,- stepIndex: 0,- turnId: "turn-1",+ store.set("action", "action:session-1:turn-1:call-1", {+ ...actionState(),});});@@ -362,3 +287,14 @@ describe("readActionTraceContext", () => {function spanContext(traceId: string, spanId: string) {return { spanId: spanId.repeat(16), traceFlags: 1, traceId: traceId.repeat(32) };}+function actionState() {+ return {+ attemptIndex: 0,+ callId: "call-1",+ context: spanContext("1", "3"),+ name: "researcher",+ sessionId: "session-1",+ stepIndex: 0,+ turnId: "turn-1",+ };+}diff --git a/packages/eve/src/tracing/eve/agent-trace-context-store.ts b/packages/eve/src/tracing/eve/agent-trace-context-store.tsnew file mode 100644index 000000000..3cf4e2dc0--- /dev/null+++ b/packages/eve/src/tracing/eve/agent-trace-context-store.ts@@ -0,0 +1,277 @@+import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";++import type { SessionTraceContext } from "#channel/types.js";+import { contextStorage, loadContext } from "#context/container.js";+import { ContextKey } from "#context/key.js";+import type { ContextAccessor } from "#context/key.js";+import { SessionTraceSeedKey, type SessionTraceSeed } from "#context/keys.js";+import type {+ AgentActionTraceState,+ AgentTraceStateStore,+ AgentTraceValues,+ AgentSessionTraceState,+ AgentTurnTraceState,+} from "#tracing/eve/agent-trace-state.js";+import { actionIdempotencyKey } from "#instrumentation/lifecycle.js";+import type { SessionStateMap } from "#harness/types.js";+import { getBlockingWorkflowToolRuns } from "#harness/workflow-tool-runs.js";++import { createLogger } from "#internal/logging.js";+import type { InstrumentationDecision } from "#shared/instrumentation-decision.js";+import {+ decisionToTraceContentCeiling,+ resolveForwardedTraceSeed,+} from "#shared/forwarded-trace-policy.js";+import { parseJsonObject } from "#shared/json.js";++export const AGENT_TRACE_CONTEXT_KEY = "eve.harness.agentTrace";+interface AgentTraceContextState {+ readonly entries: Readonly<Record<string, AgentTraceEntry>>;+}+type AgentTraceEntry =+ | { kind: "session"; value: AgentSessionTraceState }+ | { kind: "turn"; value: AgentTurnTraceState }+ | { kind: "checkpoint"; value: AgentTraceValues["checkpoint"] }+ | { kind: "action"; value: AgentActionTraceState; active: boolean; retained: boolean };+const traceStateKey = (kind: AgentTraceEntry["kind"], key: string) => JSON.stringify([kind, key]);+function traceStateValue<K extends AgentTraceEntry["kind"]>(+ state: AgentTraceContextState | undefined,+ kind: K,+ key: string,+): AgentTraceValues[K] | undefined {+ return state?.entries[traceStateKey(kind, key)]?.value as AgentTraceValues[K] | undefined;+}+const emptyAgentTraceContextState = (): AgentTraceContextState => ({ entries: {} });+function deserializeAgentTraceContextState(data: unknown): AgentTraceContextState {+ try {+ const state = parseJsonObject(data);+ if (typeof state.entries !== "object" || state.entries === null || Array.isArray(state.entries))+ return emptyAgentTraceContextState();+ const entries: Record<string, AgentTraceEntry> = {};+ for (const [key, value] of Object.entries(state.entries)) {+ if (value === null || typeof value !== "object" || Array.isArray(value)) continue;+ const record = value as import("#shared/json.js").JsonObject;+ if (+ record.kind !== "session" &&+ record.kind !== "turn" &&+ record.kind !== "checkpoint" &&+ record.kind !== "action"+ )+ continue;+ if (record.value === null || typeof record.value !== "object" || Array.isArray(record.value))+ continue;+ entries[key] = value as object as AgentTraceEntry;+ }+ return { entries };+ } catch {+ return emptyAgentTraceContextState();+ }+}++const AgentTraceContextKey = new ContextKey<AgentTraceContextState>(AGENT_TRACE_CONTEXT_KEY, {+ codec: {+ deserialize: deserializeAgentTraceContextState,+ serialize: (state) => state,+ },+});++export function currentTraceSessionId(runId: string): string | undefined {+ return traceStateValue(contextStorage.getStore()?.get(AgentTraceContextKey), "session", runId)+ ?.traceSessionId;+}++/** Run after task-provider commits, so a pending workflow tool call keeps its anchor. */+export function pruneAgentTraceState(+ context: ContextAccessor,+ sessionId: string,+ sessionState: SessionStateMap | undefined,+): void {+ try {+ pruneTraceOwnership(context, sessionId, sessionState);+ } catch (error) {+ createLogger("tracing.retention").warn(+ "could not reconcile trace ownership; preserving trace state",+ { error },+ );+ }+}++function pruneTraceOwnership(+ context: ContextAccessor,+ sessionId: string,+ sessionState: SessionStateMap | undefined,+): void {+ const state = context.get(AgentTraceContextKey);+ if (state === undefined) return;+ const calls = new Set(getBlockingWorkflowToolRuns(sessionState).map((run) => run.callId));+ const entries = Object.fromEntries(+ Object.entries(state.entries).filter(+ ([, entry]) =>+ entry.kind !== "action" ||+ entry.value.sessionId !== sessionId ||+ entry.active ||+ calls.has(entry.value.callId),+ ),+ );+ context.set(AgentTraceContextKey, { ...state, entries });+}++export function readSessionTraceDecision(+ context: ContextAccessor,+ sessionId: string,+): InstrumentationDecision | undefined {+ return traceStateValue(context.get(AgentTraceContextKey), "session", sessionId)?.decision;+}++/** Keeps only framework trace state from an interrupted step's context changes. */+export function preserveSerializedAgentTraceState(+ original: Record<string, unknown>,+ interrupted: Record<string, unknown>,+): Record<string, unknown> {+ const traceState = interrupted[AgentTraceContextKey.name];+ return traceState === undefined+ ? original+ : { ...original, [AgentTraceContextKey.name]: traceState };+}++/** Reads the active turn context straight out of a serialized Workflow context. */+export function readTurnTraceContext(+ serializedContext: Readonly<Record<string, unknown>>,+ sessionId: string,+ turnId: string,+): SessionTraceContext | undefined {+ const raw = serializedContext[AgentTraceContextKey.name];+ if (raw === undefined) return undefined;+ const state = deserializeAgentTraceContextState(raw);+ const turn = traceStateValue(state, "turn", JSON.stringify([sessionId, turnId]));+ return turn === undefined+ ? undefined+ : withTraceDecision(+ serializedContext,+ turn.context,+ traceStateValue(state, "session", sessionId)?.decision,+ );+}++/** Reads the durable action span that should parent a dispatched child agent. */+export function readActionTraceContext(+ serializedContext: Readonly<Record<string, unknown>>,+ sessionId: string,+ turnId: string,+ callId: string,+): SessionTraceContext | undefined {+ const raw = serializedContext[AgentTraceContextKey.name];+ if (raw === undefined) return undefined;+ const state = deserializeAgentTraceContextState(raw);+ const key = actionIdempotencyKey(sessionId, turnId, callId);+ const action = traceStateValue(state, "action", key);+ if (action === undefined) return undefined;+ return withTraceDecision(+ serializedContext,+ { ...action.context, isRemote: false },+ traceStateValue(state, "session", action.sessionId)?.decision,+ );+}++function withTraceDecision(+ serializedContext: Readonly<Record<string, unknown>>,+ context: SpanContext,+ storedDecision?: InstrumentationDecision,+): SessionTraceContext {+ const seed = serializedContext[SessionTraceSeedKey.name] as SessionTraceSeed | undefined;+ const traceState = resolveForwardedTraceSeed({+ decision: storedDecision ?? seed?.decision,+ forwardedTracePolicy: seed?.forwardedTracePolicy,+ traceFlags: context.traceFlags,+ })!;+ const decision = traceState.decision;+ const forwardedTracePolicy = traceState.forwardedTracePolicy;+ const ceiling = decisionToTraceContentCeiling(decision);+ const resolvedContext = { ...context, traceFlags: traceState.traceFlags };+ if (forwardedTracePolicy === undefined) {+ return decision === undefined ? resolvedContext : { ...resolvedContext, decision };+ }+ const narrowedForwardedTracePolicy =+ ceiling === undefined ? forwardedTracePolicy : { ...forwardedTracePolicy, ceiling };+ if (decision === undefined) {+ return { ...resolvedContext, forwardedTracePolicy: narrowedForwardedTracePolicy };+ }+ return {+ ...resolvedContext,+ decision,+ forwardedTracePolicy: narrowedForwardedTracePolicy,+ };+}++/** Durable trace state backed by eve's serialized Workflow context. */+export class ContextAgentTraceStateStore implements AgentTraceStateStore {+ get<K extends keyof AgentTraceValues>(kind: K, key: string): AgentTraceValues[K] | undefined {+ const entry = contextStorage.getStore()?.get(AgentTraceContextKey)?.entries[+ traceStateKey(kind === "anchor" ? "action" : kind, key)+ ];+ if (+ entry === undefined ||+ (entry.kind === "action" && !(kind === "anchor" ? entry.retained : entry.active))+ )+ return undefined;+ return entry.value as AgentTraceValues[K];+ }+ set<K extends keyof AgentTraceValues>(kind: K, key: string, value: AgentTraceValues[K]): void {+ updateState((state) => {+ const id = traceStateKey(kind === "anchor" ? "action" : kind, key);+ const previous = state.entries[id];+ const entry: AgentTraceEntry =+ kind === "action" || kind === "anchor"+ ? {+ kind: "action",+ value: value as AgentActionTraceState,+ active: kind === "action" || (previous?.kind === "action" && previous.active),+ retained: kind === "anchor" || (previous?.kind === "action" && previous.retained),+ }+ : ({ kind, value } as AgentTraceEntry);+ return { ...state, entries: { ...state.entries, [id]: entry } };+ });+ }+ delete(kind: keyof AgentTraceValues, key: string): void {+ updateState((state) => {+ const id = traceStateKey(kind === "anchor" ? "action" : kind, key);+ const entry = state.entries[id];+ const entries = { ...state.entries };+ if (entry?.kind === "action" && kind === "action" && entry.retained)+ entries[id] = { ...entry, active: false };+ else if (entry?.kind === "action" && kind === "anchor" && entry.active)+ entries[id] = { ...entry, retained: false };+ else delete entries[id];+ return { ...state, entries };+ });+ }+ update<K extends keyof AgentTraceValues>(+ kind: K,+ key: string,+ update: (value: AgentTraceValues[K]) => AgentTraceValues[K],+ ): void {+ const value = this.get(kind, key);+ if (value !== undefined) this.set(kind, key, update(value));+ }+ entries<K extends keyof AgentTraceValues>(kind: K): [string, AgentTraceValues[K]][] {+ return Object.entries(+ contextStorage.getStore()?.get(AgentTraceContextKey)?.entries ?? {},+ ).flatMap(([key, entry]) =>+ entry.kind === (kind === "anchor" ? "action" : kind) &&+ (entry.kind !== "action" || (kind === "anchor" ? entry.retained : entry.active))+ ? [+ [JSON.parse(key)[1] as string, entry.value as AgentTraceValues[K]] as [+ string,+ AgentTraceValues[K],+ ],+ ]+ : [],+ );+ }+}++function updateState(update: (state: AgentTraceContextState) => AgentTraceContextState): void {+ loadContext().set(AgentTraceContextKey, (state) =>+ update(state ?? emptyAgentTraceContextState()),+ );+}diff --git a/packages/eve/src/tracing/eve/agent-trace-context.ts b/packages/eve/src/tracing/eve/agent-trace-context.tsindex 6f5b76364..1a7742871 100644--- a/packages/eve/src/tracing/eve/agent-trace-context.ts+++ b/packages/eve/src/tracing/eve/agent-trace-context.ts@@ -1,4 +1,5 @@import { createContextKey, type Context } from "#compiled/@opentelemetry/api/index.js";+import { activeTraceOperation } from "#compiled/@eve/tracing/otel.js";// OTel context keys use Symbol.for(description), so this marker survives duplicate// module evaluations while remaining local to the process and async context.@@ -6,6 +7,7 @@ const AGENT_TRACE_KEY = createContextKey("eve.agent.trace");export function isAgentTraceContext(context: unknown): boolean {if (typeof context !== "object" || context === null) return false;+ if (activeTraceOperation(context as Context) !== undefined) return true;const getValue = Reflect.get(context, "getValue");return (typeof getValue === "function" && Reflect.apply(getValue, context, [AGENT_TRACE_KEY]) === truediff --git a/packages/eve/src/tracing/eve/agent-trace-state.ts b/packages/eve/src/tracing/eve/agent-trace-state.tsnew file mode 100644index 000000000..95aaa4ded--- /dev/null+++ b/packages/eve/src/tracing/eve/agent-trace-state.ts@@ -0,0 +1,99 @@+import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";++import type {+ InstrumentationParentLineage,+ InstrumentationPrincipalSummary,+} from "#instrumentation/lifecycle.js";+import type { ChannelAudience } from "#shared/channel-audience.js";+import type { InstrumentationDecision } from "#shared/instrumentation-decision.js";+import type { TraceSnapshot } from "#compiled/@eve/tracing/index.js";++export interface AgentSessionTraceState {+ readonly traceSessionId: string;+ readonly channelAudience?: ChannelAudience;+ readonly agentName?: string;+ readonly channelKind?: string;+ readonly channelType?: string;+ readonly context: SpanContext;+ readonly decision?: InstrumentationDecision;+ readonly parentLineage?: InstrumentationParentLineage;+ readonly rootSessionId: string;+ readonly scheduleId?: string;+ readonly title?: string;+}++export interface AgentTurnTraceState {+ readonly agentName?: string;+ /** Recorded at the turn's terminal event; the span ends at the session transition. */+ readonly terminal?: {+ readonly outcome: "cancelled" | "completed" | "failed";+ readonly errorName?: string;+ readonly errorMessage?: string;+ };+ readonly traceSessionId: string;+ readonly caller?: SpanContext;+ readonly channelDelivery?: AgentTurnChannelDeliveryTraceState;+ readonly context: SpanContext;+ readonly currentPrincipal?: InstrumentationPrincipalSummary;+ readonly initiatorPrincipal?: InstrumentationPrincipalSummary;+ readonly parentLineage?: InstrumentationParentLineage;+ readonly rootSessionId: string;+ readonly sequence: number;+ readonly startTimeMs: number;+ readonly subagentName?: string;+}++export interface AgentTurnChannelDeliveryTraceState {+ readonly channelKind: string;+ readonly channelName: string;+ readonly deliveryId: string;+ readonly inputAttribute?: string;+ readonly requestId?: string;+ readonly requestTraceContext?: SpanContext;+}++/** Locates a dispatched tool call in its turn's trace tree, after the turn's own state is gone. */+export interface AgentActionTraceState {+ readonly attemptId?: string;+ readonly attemptIndex: number;+ readonly callId: string;+ readonly channelAudience?: ChannelAudience;+ readonly context: SpanContext;+ readonly rootSessionId?: string;+ readonly sessionId: string;+ readonly stepIndex: number;+ /** When the SDK execution finished, for dispatches that report no acceptance time. */+ readonly toolEndTimeMs?: number;+ readonly turnId: string;+}++/** Provider-owned serializable storage for durable agent trace state. */+export interface AgentTraceStateStore {+ get<K extends keyof AgentTraceValues>(+ kind: K,+ key: string,+ ): AgentTraceValues[K] | undefined | PromiseLike<AgentTraceValues[K] | undefined>;+ set<K extends keyof AgentTraceValues>(+ kind: K,+ key: string,+ value: AgentTraceValues[K],+ ): void | PromiseLike<void>;+ delete(kind: keyof AgentTraceValues, key: string): void | PromiseLike<void>;+ update<K extends keyof AgentTraceValues>(+ kind: K,+ key: string,+ update: (value: AgentTraceValues[K]) => AgentTraceValues[K],+ ): void | PromiseLike<void>;+ entries<K extends keyof AgentTraceValues>(+ kind: K,+ ):+ | readonly [string, AgentTraceValues[K]][]+ | PromiseLike<readonly [string, AgentTraceValues[K]][]>;+}+export interface AgentTraceValues {+ session: AgentSessionTraceState;+ turn: AgentTurnTraceState;+ action: AgentActionTraceState;+ anchor: AgentActionTraceState;+ checkpoint: TraceSnapshot;+}diff --git a/packages/eve/src/tracing/eve/batch-span-processor.test.ts b/packages/eve/src/tracing/eve/batch-span-processor.test.tsindex 0cfddea7c..d7dd26fb8 100644--- a/packages/eve/src/tracing/eve/batch-span-processor.test.ts+++ b/packages/eve/src/tracing/eve/batch-span-processor.test.ts@@ -1,11 +1,21 @@-import type { SpanExporter } from "#compiled/@vercel/otel/index.js";+import type { SpanExporter } from "./otel-configuration.js";import { context, TraceFlags } from "@opentelemetry/api";import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";import { isTracingSuppressed } from "@opentelemetry/core";import { afterEach, describe, expect, it, vi } from "vitest";-import { batchSpanProcessor } from "#tracing/eve/batch-span-processor.js";-import { captureLogRecords } from "#internal/testing/log-records.js";+import { batchSpanProcessor as createBatchSpanProcessor } from "./batch-span-processor.js";+const diagnostics = vi.fn();++function batchSpanProcessor(+ exporter: SpanExporter,+ options: Parameters<typeof createBatchSpanProcessor>[1] = {},+) {+ return createBatchSpanProcessor(exporter, {+ ...options,+ diagnostic: diagnostics,+ });+}interface RecordingExporter extends SpanExporter {readonly batches: readonly (readonly unknown[])[];@@ -35,6 +45,7 @@ function batchIndexes(batches: readonly (readonly unknown[])[]): number[][] {describe("batchSpanProcessor", () => {afterEach(() => {+ diagnostics.mockClear();vi.useRealTimers();context.disable();});@@ -81,16 +92,13 @@ describe("batchSpanProcessor", () => {});it("logs a refused export rather than failing the turn that produced it", async () => {- const logs = captureLogRecords();const processor = batchSpanProcessor(recordingExporter({ fail: true }), {maxExportBatchSize: 1,});processor.onEnd(span(1));await expect(processor.forceFlush()).resolves.toBeUndefined();- expect(logs.records).toContainEqual(- expect.objectContaining({ level: "warn", message: "span export failed" }),- );+ expect(diagnostics).toHaveBeenCalledWith("span export failed", expect.anything());});it("exports only sampled spans", async () => {@@ -126,7 +134,6 @@ describe("batchSpanProcessor", () => {});it("does not overlap a later batch after the first times out", async () => {- const logs = captureLogRecords();vi.useFakeTimers();const batches: (readonly unknown[])[] = [];const callbacks: Array<(result: { code: number }) => void> = [];@@ -159,16 +166,13 @@ describe("batchSpanProcessor", () => {expect(batchIndexes(batches)).toStrictEqual([[1], [2]]);callbacks[1]?.({ code: 0 });await flushed;- expect(logs.records).toContainEqual(- expect.objectContaining({- level: "warn",- message: "span export timed out; preserving exporter serialization",- }),+ expect(diagnostics).toHaveBeenCalledWith(+ "span export timed out; preserving exporter serialization",+ expect.anything(),);});it("continues exporting after exporter forceFlush times out", async () => {- const logs = captureLogRecords();vi.useFakeTimers();const exporter = recordingExporter() as RecordingExporter & {forceFlush: () => Promise<void>;@@ -188,16 +192,13 @@ describe("batchSpanProcessor", () => {await vi.advanceTimersByTimeAsync(0);expect(batchIndexes(exporter.batches)).toStrictEqual([[1], [2]]);- expect(logs.records).toContainEqual(- expect.objectContaining({- level: "warn",- message: "exporter flush timed out; preserving exporter serialization",- }),+ expect(diagnostics).toHaveBeenCalledWith(+ "exporter flush timed out; preserving exporter serialization",+ expect.anything(),);});it("drains queued batches and closes the exporter after shutdown times out", async () => {- const logs = captureLogRecords();vi.useFakeTimers();const batches: (readonly unknown[])[] = [];const callbacks: Array<(result: { code: number }) => void> = [];@@ -228,11 +229,9 @@ describe("batchSpanProcessor", () => {callbacks[1]?.({ code: 0 });await vi.advanceTimersByTimeAsync(0);expect(shutdown).toHaveBeenCalledOnce();- expect(logs.records).toContainEqual(- expect.objectContaining({- level: "warn",- message: "span export timed out; preserving exporter serialization",- }),+ expect(diagnostics).toHaveBeenCalledWith(+ "span export timed out; preserving exporter serialization",+ expect.anything(),);});diff --git a/packages/eve/src/tracing/eve/batch-span-processor.ts b/packages/eve/src/tracing/eve/batch-span-processor.tsindex c343352c6..6148731a8 100644--- a/packages/eve/src/tracing/eve/batch-span-processor.ts+++ b/packages/eve/src/tracing/eve/batch-span-processor.ts@@ -1,9 +1,5 @@import { TraceFlags, context, createContextKey } from "#compiled/@opentelemetry/api/index.js";-import type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js";--import { createLogger, formatError } from "#internal/logging.js";--const log = createLogger("harness.batch-span-processor");+import type { SpanExporter, SpanProcessor } from "./otel-configuration.js";// This is the Symbol.for-backed key used by @opentelemetry/core's// suppressTracing(), without pulling that package into eve's runtime.@@ -19,6 +15,10 @@ const DEFAULTS = {/** @internal — exposed for tests; authors get the defaults. */interface BatchSpanProcessorOptions {+ readonly diagnostic?: (+ message: string,+ details: { error?: unknown; timeoutMillis?: number },+ ) => void;readonly exportTimeoutMillis?: number;readonly maxExportBatchSize?: number;readonly maxQueueSize?: number;@@ -40,6 +40,11 @@ export function batchSpanProcessor(exporter: SpanExporter,options: BatchSpanProcessorOptions = {},): SpanProcessor {+ const report = (message: string, details: { error?: unknown; timeoutMillis?: number }) => {+ try {+ options.diagnostic?.(message, details);+ } catch {}+ };const exportTimeoutMillis = options.exportTimeoutMillis ?? DEFAULTS.exportTimeoutMillis;const maxExportBatchSize = options.maxExportBatchSize ?? DEFAULTS.maxExportBatchSize;const maxQueueSize = options.maxQueueSize ?? DEFAULTS.maxQueueSize;@@ -93,7 +98,7 @@ export function batchSpanProcessor(} catch (error: unknown) {// A destination that cannot take spans is not a reason to fail the turn// that produced them.- log.warn("span export failed", { error: formatError(error) });+ report("span export failed", { error });}}@@ -150,7 +155,7 @@ export function batchSpanProcessor(let timer: ReturnType<typeof setTimeout> | undefined;const timeout = new Promise<false>((resolve) => {timer = setTimeout(() => {- log.warn(`${name} timed out; preserving exporter serialization`, {+ report(`${name} timed out; preserving exporter serialization`, {timeoutMillis: exportTimeoutMillis,});resolve(false);@@ -169,7 +174,7 @@ export function batchSpanProcessor(try {await operation.call(exporter);} catch (error: unknown) {- log.warn(`${name} failed`, { error: formatError(error) });+ report(`${name} failed`, { error });}}}diff --git a/packages/eve/src/tracing/eve/content-span-processor.ts b/packages/eve/src/tracing/eve/content-span-processor.tsindex 5f5f2969b..43c9aca00 100644--- a/packages/eve/src/tracing/eve/content-span-processor.ts+++ b/packages/eve/src/tracing/eve/content-span-processor.ts@@ -3,7 +3,7 @@ import type { SpanProcessor } from "#compiled/@vercel/otel/index.js";import {withoutDeclinedContent,type ResolvedContentOptions,-} from "#tracing/content-attributes.js";+} from "#compiled/@eve/tracing/otel.js";import { hasConversationRelease, type LocalTracesProcessor } from "#tracing/local/traces.js";import { normalizeChannelAudience } from "#shared/channel-audience.js";import type { ChannelAudience } from "#shared/channel-audience.js";diff --git a/packages/eve/src/tracing/eve/delegation.ts b/packages/eve/src/tracing/eve/delegation.tsnew file mode 100644index 000000000..98980c117--- /dev/null+++ b/packages/eve/src/tracing/eve/delegation.ts@@ -0,0 +1,87 @@+import { createAgentDelegationTransport } from "#compiled/@eve/tracing/delegation.js";+import type { AgentSessionContext } from "#execution/agent-sessions/context.js";+import type { SessionParent, SessionTraceContext } from "#channel/types.js";+import { resolveConversationId } from "#shared/conversation-identity.js";+import { intersectInstrumentationDecisions } from "#shared/instrumentation-decision.js";+import { readConversationBaggage } from "#tracing/eve/conversation-context.js";++const transport = createAgentDelegationTransport();++export const agentDelegation = {+ fetch: transport.fetch,+ resume<T>(+ input: { context: Pick<AgentSessionContext, "trace" | "parent">; agentName: string },+ execute: () => T,+ ): T {+ const { context } = input;+ const caller = context.trace.parentTraceContext;+ if (caller === undefined) return execute();+ const decision = caller.decision;+ return transport.resume(+ {+ caller,+ conversationId:+ context.trace.conversationId ?? resolveConversationId(context.parent.rootSessionId),+ parentRunId: context.parent.sessionId,+ parentCallId: context.parent.callId,+ agentName: input.agentName,+ capture: {+ emit: caller.traceFlags === 1 && decision?.action !== "drop",+ recordInputs: decision?.action === "record" && decision.recordInputs,+ recordOutputs: decision?.action === "record" && decision.recordOutputs,+ },+ },+ execute,+ );+ },+ receive<T>(+ input: {+ headers: Headers;+ delegated: boolean;+ trusted: boolean;+ parent?: SessionParent;+ callerTrace?: SessionTraceContext;+ parentTraceContext?: SessionTraceContext;+ },+ execute: (context: { conversationId?: string; parentTraceContext?: SessionTraceContext }) => T,+ ): T {+ return transport.receive(+ input.headers,+ (handoff) =>+ input.delegated &&+ input.trusted &&+ input.parent !== undefined &&+ handoff.parentRunId === input.parent.sessionId &&+ handoff.parentCallId === input.parent.callId &&+ input.callerTrace?.traceId === handoff.caller.traceId &&+ input.callerTrace.spanId === handoff.caller.spanId,+ (handoff) => {+ const parentTrace = input.parentTraceContext;+ const decision = handoff?.capture.emit+ ? {+ action: "record" as const,+ recordInputs: handoff.capture.recordInputs,+ recordOutputs: handoff.capture.recordOutputs,+ }+ : { action: "drop" as const };+ const inheritedTrace =+ handoff === undefined || parentTrace === undefined+ ? parentTrace+ : {+ ...parentTrace,+ decision:+ parentTrace.decision === undefined+ ? decision+ : intersectInstrumentationDecisions(parentTrace.decision, decision),+ traceFlags: handoff.capture.emit ? parentTrace.traceFlags : 0,+ };+ return execute({+ conversationId: input.delegated+ ? (handoff?.conversationId ?? readConversationBaggage(input.headers.get("baggage")))+ : undefined,+ parentTraceContext: inheritedTrace,+ });+ },+ );+ },+};diff --git a/packages/eve/src/tracing/eve/gateway.ts b/packages/eve/src/tracing/eve/gateway.tsnew file mode 100644index 000000000..4d255597b--- /dev/null+++ b/packages/eve/src/tracing/eve/gateway.ts@@ -0,0 +1,41 @@+/** Converts Gateway response metadata to cost data. */+import type { Attributes } from "#compiled/@eve/tracing/index.js";+export function gatewayCostAttributes(input: {+ cost?: number;+ gatewayCost?: number;+ inputCost?: number;+ outputCost?: number;+ generationId?: string;+}): Attributes {+ return {+ "gen_ai.usage.cost": input.cost,+ "gen_ai.usage.gateway_cost": input.gatewayCost,+ "gen_ai.usage.input_cost": input.inputCost,+ "gen_ai.usage.output_cost": input.outputCost,+ "gen_ai.generation.id": input.generationId,+ };+}+export function readGatewayCostData(+ providerMetadata: Readonly<Record<string, unknown>> | undefined,+) {+ if (providerMetadata === undefined) return undefined;+ const gateway = providerMetadata.gateway;+ if (typeof gateway !== "object" || gateway === null || Array.isArray(gateway)) return undefined;+ const data = gateway as Record<string, unknown>;+ return {+ cost: readUsd(data.cost),+ gatewayCost: readUsd(data.gatewayCost),+ inputCost: readUsd(data.inputInferenceCost),+ outputCost: readUsd(data.outputInferenceCost),+ generationId:+ typeof data.generationId === "string" && data.generationId.length > 0+ ? data.generationId+ : undefined,+ };+}++function readUsd(value: unknown): number | undefined {+ if (typeof value !== "string" || value.trim() === "") return undefined;+ const parsed = Number(value);+ return Number.isFinite(parsed) ? parsed : undefined;+}diff --git a/packages/eve/src/tracing/eve/install-instrumentation-runtime.test.ts b/packages/eve/src/tracing/eve/install-instrumentation-runtime.test.tsindex b7c4b217f..6180f9e21 100644--- a/packages/eve/src/tracing/eve/install-instrumentation-runtime.test.ts+++ b/packages/eve/src/tracing/eve/install-instrumentation-runtime.test.ts@@ -12,22 +12,20 @@ const { forceFlush, internalTerminalState, shutdown } = vi.hoisted(() => ({shutdown: vi.fn(async () => undefined),}));-vi.mock("#tracing/eve/otel-registration.js", async (importOriginal) => {- const actual = await importOriginal<typeof import("#tracing/eve/otel-registration.js")>();- return {- ...actual,- registerOtelPipeline: () => ({- forceFlush,- idGenerator: {- allocateSpanId: () => "1".repeat(16),- withSpanId: (_spanId: string, run: () => unknown) => run(),- },- shutdown,- }),- };-});+vi.mock("#tracing/eve/otel-registration.js", () => ({+ registerOtel: () => ({+ forceFlush,+ shutdown,+ provider: { getTracer: () => ({}) },+ idGenerator: {+ allocateSpanId: () => "1".repeat(16),+ withSpanId: (_spanId: string, run: () => unknown) => run(),+ },+ samplesTrace: () => true,+ }),+}));-vi.mock("#tracing/agent-otel-provider.js", () => ({+vi.mock("#tracing/eve/agent-otel-provider.js", () => ({createAgentOtelInstrumentation: () => ({hook: {events: {diff --git a/packages/eve/src/tracing/eve/install-instrumentation-runtime.ts b/packages/eve/src/tracing/eve/install-instrumentation-runtime.tsindex 5cfd2a751..753b4f683 100644--- a/packages/eve/src/tracing/eve/install-instrumentation-runtime.ts+++ b/packages/eve/src/tracing/eve/install-instrumentation-runtime.ts@@ -1,4 +1,3 @@-import { trace } from "#compiled/@opentelemetry/api/index.js";import type { SpanProcessor } from "#compiled/@vercel/otel/index.js";import { contextStorage } from "#context/container.js";@@ -13,15 +12,15 @@ import {} from "#instrumentation/runtime.js";import { createLogger, formatError } from "#internal/logging.js";import { resolveInstrumentationEnvironment } from "#internal/application/dev-environment.js";-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";-import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";-import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js";+import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";+import { ContextAgentTraceStateStore } from "#tracing/eve/agent-trace-context-store.js";+import { createAgentOtelInstrumentation } from "#tracing/eve/agent-otel-provider.js";import { hasConversationRelease, type LocalTracesProcessor } from "#tracing/local/traces.js";import type { CollectedOtel, RuntimeContextResolver } from "#tracing/eve/otel-declaration.js";-import {- registerOtelPipeline,- type RegisteredOtelPipeline,-} from "#tracing/eve/otel-registration.js";+import { otelTelemetry } from "#compiled/@eve/tracing/index.js";+import type { RegisteredOtelPipeline } from "#tracing/eve/otel-registration.js";+import { registerOtel } from "#tracing/eve/otel-registration.js";+import { eveOutputMapping } from "#tracing/eve/profile.js";import { readConversationId } from "#shared/conversation-identity.js";const log = createLogger("tracing.install-instrumentation-runtime");@@ -48,21 +47,29 @@ export function installInstrumentationRuntime(input: {let prepareSessionTrace: InstrumentationRuntime["prepareSessionTrace"];let prepareTurnTrace: InstrumentationRuntime["prepareTurnTrace"];let runInContext: InstrumentationRuntime["runInContext"] = (_operation, execute) => execute();+ let idGenerator: AgentSpanIdGenerator;if (input.collected.declared) {- otelRuntime = registerOtelPipeline({- pipeline: input.collected.pipeline,+ otelRuntime = registerOtel({serviceName: input.serviceName,+ otel: input.collected.configuration,});+ idGenerator = otelRuntime.idGenerator;const agentOtel = createAgentOtelInstrumentation({environment: resolveInstrumentationEnvironment(),frameworkVersion: input.frameworkVersion,- idGenerator: otelRuntime.idGenerator,+ telemetry: otelTelemetry({+ provider: otelRuntime.provider,+ tracerName: "eve.agent",+ tracerVersion: input.frameworkVersion,+ idGenerator,+ samplesTrace: otelRuntime.samplesTrace,+ mapping: eveOutputMapping(),+ }),+ idGenerator,recordInputs: input.collected.settings.recordInputs,recordOutputs: input.collected.settings.recordOutputs,- samplesTrace: otelRuntime.samplesTrace,stateStore: new ContextAgentTraceStateStore(),- tracer: trace.getTracer("eve.agent", input.frameworkVersion),tracePolicy: input.collected.settings.tracePolicy,});// The span must exist before authored providers observe the lifecycle event.@@ -71,10 +78,12 @@ export function installInstrumentationRuntime(input: {prepareTurnTrace = agentOtel.prepareTurnTrace;runInContext = agentOtel.runInContext;- const releasable = input.collected.pipeline.spanProcessors+ const releasable = input.collected.configuration.spanProcessors.filter(isSpanProcessor).filter(hasConversationRelease);if (releasable.length > 0) serialAfter.push(sessionReleaseProvider(releasable));+ } else {+ idGenerator = new AgentSpanIdGenerator();}const allProviders = [...serialBefore, ...input.providers, ...serialAfter];@@ -89,7 +98,7 @@ export function installInstrumentationRuntime(input: {serialAfter,serialBefore,}),- idGenerator: otelRuntime?.idGenerator ?? new AgentSpanIdGenerator(),+ idGenerator,memoryOperations: otelRuntime !== undefined || input.providers.some(hasMemoryOperationHandler),otelSettings: input.collected.declared ? input.collected.settings : undefined,ownsAgentSpans: otelRuntime !== undefined,diff --git a/packages/eve/src/tracing/eve/invoke-tool-span.ts b/packages/eve/src/tracing/eve/invoke-tool-span.tsindex 09ae1b377..5ddd23e55 100644--- a/packages/eve/src/tracing/eve/invoke-tool-span.ts+++ b/packages/eve/src/tracing/eve/invoke-tool-span.ts@@ -1,11 +1,4 @@-import {- context as otelContext,- SpanKind,- trace,- type Attributes,- type Context,- type Span,-} from "#compiled/@opentelemetry/api/index.js";+import { context as otelContext, type Context } from "#compiled/@opentelemetry/api/index.js";import type { InvokeToolResult } from "#channel/invoke-tool.js";import type { SessionAuthContext } from "#channel/types.js";@@ -21,21 +14,19 @@ import { resolveInstrumentationEnvironment } from "#internal/application/dev-envimport { createLogger, formatError } from "#internal/logging.js";import { applyLiveDeliveryAudienceCeiling } from "#shared/forwarded-trace-policy.js";import { resolveTracePolicy } from "#shared/trace-policy.js";-import { agentTraceIdentityAttributes } from "#tracing/agent-otel-attributes.js";-import { contentAttribute } from "#tracing/agent-otel-content.js";-import { recordAgentSpanError } from "#tracing/agent-span-error.js";-import {- DIRECT_TOOL_CALL_ATTRIBUTE,- DIRECT_TOOL_CALL_VALUE,-} from "#tracing/agent-span-contract.js";-import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";+import { otelTelemetry, type Attributes, type SpanWriter } from "#compiled/@eve/tracing/index.js";+import { contentAttribute, mcpLifecycle, withErrorContent } from "#compiled/@eve/tracing/otel.js";+import { DIRECT_TOOL_CALL_ATTRIBUTE, DIRECT_TOOL_CALL_VALUE } from "#tracing/local/inspection.js";import { markAgentTraceContext } from "#tracing/eve/agent-trace-context.js";-import { withAgentToolSpanContext } from "#tracing/agent-tool-span-context.js";import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";-import { withErrorContent } from "#tracing/error-content-context.js";-import { suppressTracing } from "#tracing/suppress-tracing.js";+import { operationConversationId } from "#tracing/eve/conversation-context.js";+import { MCP_SERIALIZER } from "#tracing/eve/mcp.js";+import { eveOutputMapping } from "#tracing/eve/profile.js";const log = createLogger("invoke-tool");+// Built per call: the global tracer provider can be registered after this module loads.+const agentTelemetry = () =>+ otelTelemetry({ tracerName: "eve.agent", mapping: eveOutputMapping() });/** Where a direct tool call came from, so trace policy can classify it like a conversation. */export interface InvokeToolTraceOrigin {@@ -109,6 +100,7 @@ async function traced(): Promise<InvokeToolResult> {const runtime = getInstrumentationRuntime();if (runtime === undefined) return await run(silentObserver);+ const telemetry = agentTelemetry();const conversation = buildConversationContext({@@ -147,7 +139,7 @@ async function traced(}),);if (decision.action === "drop") {- return await otelContext.with(suppressTracing(base), () => run(events.observer(undefined)));+ return await telemetry.suppressed!(() => run(events.observer(undefined)));}const content = applyLiveDeliveryAudienceCeiling({@@ -164,7 +156,7 @@ async function traced(const spanName = `execute_tool ${input.toolName}`;const parent = withChannelAudience(base, conversation.audience);- const attributes: Attributes = {+ const attributes: Record<string, Attributes[string]> = {"agent.tool.is_framework": false,"gen_ai.agent.name": input.agentName,"gen_ai.operation.name": "execute_tool",@@ -172,30 +164,30 @@ async function traced("gen_ai.tool.name": input.toolName,"gen_ai.tool.type": "function",[DIRECT_TOOL_CALL_ATTRIBUTE]: DIRECT_TOOL_CALL_VALUE,- ...agentSpanNamingAttributes(spanName, "execute_tool"),- ...agentTraceIdentityAttributes({- rootSessionId: input.sessionId,- sessionId: input.sessionId,- traceSessionId: input.sessionId,- }),+ "operation.name": "execute_tool",+ "resource.name": spanName,+ "agent.run.id": input.sessionId,+ "agent.trace.schema.version": 4,+ "gen_ai.conversation.id": operationConversationId({ sessionId: input.sessionId }),};+ if (process.env.VERCEL_ENV !== undefined) attributes["vercel.session_id"] = input.sessionId;if (input.origin !== undefined) {attributes["eve.channel.kind"] = conversation.channel.kind;attributes["eve.channel.name"] = input.origin.channelName;}- const span = trace- .getTracer("eve.agent")- .startSpan(spanName, { attributes, kind: SpanKind.INTERNAL }, parent);- // As on a conversation's tool span: error text inside the call follows the output decision.- const active = markAgentTraceContext(- withAgentToolSpanContext(withErrorContent(trace.setSpan(parent, span), recordOutputs), {- recordInputs,- recordOutputs,- recordError: (error, errorType) =>- recordAgentSpanError(span, recordOutputs ? error : undefined, errorType),- setAttributes: (attributes) => setDefined(span, attributes),- }),+ const span = telemetry.startSpan(+ { type: "tool", operationId: input.callId, name: spanName, kind: "INTERNAL", attributes },+ { context: parent },);+ const capture = { emit: true, recordInputs, recordOutputs };+ // As on a conversation's tool span: error text inside the call follows the output decision.+ const host = markAgentTraceContext(withErrorContent(parent, recordOutputs));+ const mcp = mcpLifecycle({+ serializer: MCP_SERIALIZER,+ ...capture,+ write: (attributes) => setDefined(span, attributes),+ error: (error, errorType) => span.fail(recordOutputs ? error : undefined, errorType),+ });let failure: { readonly error: unknown } | undefined;const observer = events.observer({@@ -206,7 +198,11 @@ async function traced(span,});try {- const result = await otelContext.with(active, () => run(observer));+ const result = await telemetry.run(+ { type: "tool", reference: span.reference, capture, mcp },+ () => run(observer),+ host,+ );span.setAttribute("eve.tool.outcome", result.status);if (result.status === "completed" && recordOutputs) {const output = contentAttribute(result.output);@@ -214,18 +210,17 @@ async function traced(}if (result.status === "invalid-input") {// A model-issued call with invalid input never runs, so it has no span to compare to.- recordAgentSpanError(span, undefined, result.status);+ span.fail(undefined, result.status);} else if (result.status === "failed") {// The conversation's rule: the error itself when outputs are recorded, else a generic one.- recordAgentSpanError(- span,+ span.fail(recordOutputs ? failure?.error : undefined,failure === undefined ? result.status : undefined,);}return result;} catch (error) {- recordAgentSpanError(span, recordOutputs ? error : undefined);+ span.fail(recordOutputs ? error : undefined);throw error;} finally {span.end();@@ -260,7 +255,7 @@ function toolCallEvents(| {readonly failedWith: (error: unknown) => void;readonly recordInputs: boolean;- readonly span: Span;+ readonly span: SpanWriter;}| undefined,): InvokeToolObserver {@@ -311,7 +306,7 @@ function toolCallEvents(};}-function setDefined(span: Span, attributes: Attributes): void {+function setDefined(span: SpanWriter, attributes: Attributes): void {for (const [name, value] of Object.entries(attributes)) {if (value !== undefined) span.setAttribute(name, value);}diff --git a/packages/eve/src/tracing/eve/logging.integration.test.ts b/packages/eve/src/tracing/eve/logging.integration.test.tsnew file mode 100644index 000000000..6cdccfd7d--- /dev/null+++ b/packages/eve/src/tracing/eve/logging.integration.test.ts@@ -0,0 +1,43 @@+import { context, trace } from "@opentelemetry/api";+import {+ BasicTracerProvider,+ InMemorySpanExporter,+ SimpleSpanProcessor,+} from "@opentelemetry/sdk-trace-base";+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";+import { expect, it, vi } from "vitest";+import { withCapture } from "#compiled/@eve/tracing/otel.js";+import { createLogger, logError } from "#internal/logging.js";++it("keeps logger-to-span error details behind the active output policy", async () => {+ const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});+ const exporter = new InMemorySpanExporter();+ const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] });+ const manager = new AsyncLocalStorageContextManager().enable();+ context.setGlobalContextManager(manager);+ try {+ for (const allowed of [false, true]) {+ const span = provider.getTracer("logger").startSpan(`logger-${allowed}`);+ const active = withCapture(trace.setSpan(context.active(), span), {+ emit: true,+ recordInputs: false,+ recordOutputs: allowed,+ });+ context.with(active, () =>+ logError(createLogger("test"), "operation failed", new Error("sensitive payload")),+ );+ span.end();+ }+ const [privateSpan, publicSpan] = exporter.getFinishedSpans();+ expect(privateSpan?.status.code).toBe(2);+ expect(privateSpan?.status.message).toBeUndefined();+ expect(privateSpan?.events).toEqual([]);+ expect(publicSpan?.status.message).toContain("sensitive payload");+ expect(publicSpan?.events[0]?.attributes?.["exception.message"]).toContain("sensitive payload");+ } finally {+ context.disable();+ manager.disable();+ consoleError.mockRestore();+ await provider.shutdown();+ }+});diff --git a/packages/eve/src/tracing/eve/mcp.ts b/packages/eve/src/tracing/eve/mcp.tsnew file mode 100644index 000000000..7125069b4--- /dev/null+++ b/packages/eve/src/tracing/eve/mcp.ts@@ -0,0 +1,374 @@+import {+ context as otelContext,+ propagation,+ trace,+ type Context as OtelContext,+ type TextMapSetter,+} from "#compiled/@opentelemetry/api/index.js";++import {+ otelTelemetry,+ type CaptureDecision,+ type ContentSerializer,+} from "#compiled/@eve/tracing/index.js";+import {+ activeTraceOperation,+ contentAttribute,+ mcpLifecycle,+ truncateTelemetryText,+ type McpLifecycle,+ type McpUpdate,+} from "#compiled/@eve/tracing/otel.js";+import { eveOutputMapping } from "./profile.js";+function isObject(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++type Context = Pick<OtelContext, "getValue">;+const MAX_MCP_TRACE_REQUEST_BYTES = 1024 * 1024;+const MAX_MCP_TRACE_CONTEXT_BYTES = 8192;+const utf8Encoder = new TextEncoder();+const traceContextSetter: TextMapSetter<Record<string, string>> = {+ set(carrier, key, value) {+ carrier[key] = value;+ },+};++export function createMcpTraceFetch(input: {+ readonly connectionName: string;+ readonly fetcher?: typeof fetch;+ readonly getActiveContext?: () => Context;+ readonly getProtocolVersion: () => string | undefined;+ readonly injectContext?: McpTraceContextInjector;+ readonly filterCarrier?: (carrier: Record<string, string>) => void;+}): typeof fetch {+ const fetcher = input.fetcher ?? ((request, init) => globalThis.fetch(request, init));+ const getActiveContext = input.getActiveContext ?? (() => otelContext.active());+ const injectContext = input.injectContext ?? injectOpenTelemetryTraceContext;++ return async (request, init) => {+ const message = readJsonRpcRequest(init?.body);+ if (message === undefined) return await fetcher(request, init);++ const activeContext = getActiveContext();+ const propagated = injectMcpTraceContext(+ message,+ activeContext,+ injectContext,+ input.filterCarrier,+ );+ const annotateRequest =+ message.method === "tools/call" ||+ (message.method === "tools/list" && activeTraceOperation(activeContext)?.type === "mcp");+ if (annotateRequest) {+ activeTraceOperation(activeContext)?.mcp?.update(+ mcpRequestMetadata({+ connectionName: input.connectionName,+ message,+ params: isObject(message.params) ? message.params : {},+ protocolVersion:+ headerValue(init?.headers, "mcp-protocol-version") ?? input.getProtocolVersion(),+ }),+ );+ }++ const response = await fetcher(+ request,+ propagated === undefined ? init : { ...init, body: JSON.stringify(propagated) },+ );+ if (annotateRequest) {+ const sessionId = response.headers.get("mcp-session-id");+ if (sessionId !== null) {+ activeTraceOperation(activeContext)?.mcp?.update({+ sessionId: truncateTelemetryText(sessionId, 512),+ });+ }+ }+ return response;+ };+}++function injectMcpTraceContext(+ message: JsonRpcRequest,+ activeContext: Context,+ injectContext: McpTraceContextInjector = injectOpenTelemetryTraceContext,+ filterCarrier?: (carrier: Record<string, string>) => void,+): JsonRpcRequest | undefined {+ if (!withinTraceRequestLimit(JSON.stringify(message))) return undefined;++ if (message.params !== undefined && !isObject(message.params)) return undefined;+ const propagated: Record<string, string> = {};+ injectContext(activeContext, propagated);+ if (!withinTraceContextLimit(propagated)) return undefined;+ filterCarrier?.(propagated);++ const params = isObject(message.params) ? message.params : {};+ const existingMeta = isObject(params["_meta"]) ? params["_meta"] : {};+ return {+ ...message,+ params: {+ ...params,+ _meta: { ...existingMeta, ...propagated },+ },+ };+}++const noContent = () => undefined;+/** Records only JSON tool arguments and results, as MCP spans carry no model content. */+export const MCP_SERIALIZER: ContentSerializer = {+ json: contentAttribute,+ text: noContent,+ inputMessages: noContent,+ instructions: noContent,+ toolDefinitions: noContent,+ outputMessages: noContent,+ toolResults: noContent,+};++export function createMcpTracing() {+ const telemetry = otelTelemetry({ tracerName: "eve.mcp", mapping: eveOutputMapping() });+ function start(input: {+ method: "tools/list" | "tools/call";+ connectionName: string;+ toolName?: string;+ protocolVersion?: string;+ executionContext: OtelContext;+ capture: CaptureDecision;+ }) {+ const span = telemetry.startSpan(+ {+ type: "mcp",+ operationId: `${input.connectionName}:${input.method}`,+ name:+ input.method === "tools/call"+ ? `tools/call ${input.toolName ?? "unknown"}`+ : input.method,+ kind: "CLIENT",+ parent: trace.getSpan(input.executionContext)?.spanContext(),+ attributes: {+ "agent.connection.name": input.connectionName,+ "mcp.method.name": input.method,+ "network.protocol.name": "http",+ "network.transport": "tcp",+ "mcp.protocol.version": input.protocolVersion,+ "gen_ai.operation.name": input.method === "tools/call" ? "execute_tool" : undefined,+ "gen_ai.tool.name": input.toolName,+ },+ },+ { context: input.executionContext },+ );+ const semantic = mcpLifecycle({+ serializer: MCP_SERIALIZER,+ ...input.capture,+ write(attributes) {+ for (const [key, value] of Object.entries(attributes))+ if (value !== undefined) span.setAttribute(key, value);+ },+ error: span.fail,+ });+ return {+ ...semantic,+ end: span.end,+ run<T>(execute: () => T): T {+ return telemetry.run(+ { type: "mcp", reference: span.reference, capture: input.capture, mcp: semantic },+ execute,+ input.executionContext,+ );+ },+ };+ }+ return {+ async list<T>(input: {+ readonly connectionName: string;+ readonly execute: () => Promise<T>;+ readonly protocolVersion?: string;+ }): Promise<T> {+ const parent = otelContext.active();+ const span = start({+ method: "tools/list",+ connectionName: input.connectionName,+ protocolVersion: input.protocolVersion,+ executionContext: parent,+ capture: activeTraceOperation()?.capture ?? {+ emit: true,+ recordInputs: false,+ recordOutputs: false,+ },+ });+ try {+ return await span.run(async () => {+ try {+ return await input.execute();+ } catch (error) {+ const code = jsonRpcErrorCode(error);+ if (code !== undefined) {+ span.update({ statusCode: code });+ }+ span.error(error, errorType(error, code));+ throw error;+ }+ });+ } finally {+ span.end();+ }+ },+ async call<T>(input: {+ readonly arguments: unknown;+ readonly connectionName: string;+ readonly execute: () => Promise<T>;+ readonly protocolVersion?: string;+ readonly toolName: string;+ }): Promise<T> {+ const parent = otelContext.active();+ const existing = activeTraceOperation();+ if (existing?.type === "tool" && existing.mcp !== undefined) {+ existing.mcp.update({ ...input, method: "tools/call" });+ return await runMcpToolCall(input, undefined);+ }++ const span = start({+ method: "tools/call",+ connectionName: input.connectionName,+ toolName: truncateTelemetryText(input.toolName, 128),+ protocolVersion: input.protocolVersion,+ executionContext: parent,+ capture: existing?.capture ?? { emit: true, recordInputs: false, recordOutputs: false },+ });+ try {+ return await span.run(() => runMcpToolCall(input, span));+ } finally {+ span.end();+ }+ },+ };+}++function runMcpToolCall<T>(+ input: {+ readonly arguments: unknown;+ readonly execute: () => Promise<T>;+ },+ fallbackSpan: McpLifecycle | undefined,+): Promise<T> {+ if (fallbackSpan !== undefined) {+ fallbackSpan.arguments(input.arguments);+ }++ return input+ .execute()+ .then((result) => {+ if (isToolErrorResult(result)) {+ activeTraceOperation()?.mcp?.error(undefined, "tool_error");+ }+ if (fallbackSpan !== undefined) {+ fallbackSpan.result(result);+ }+ return result;+ })+ .catch((error: unknown) => {+ const code = jsonRpcErrorCode(error);+ if (code !== undefined) {+ activeTraceOperation()?.mcp?.update({ statusCode: code });+ }+ if (fallbackSpan !== undefined) {+ activeTraceOperation()?.mcp?.error(error, errorType(error, code));+ }+ throw error;+ });+}++function mcpRequestMetadata(input: {+ readonly connectionName: string;+ readonly message?: JsonRpcRequest;+ readonly method?: string;+ readonly params?: Record<string, unknown>;+ readonly protocolVersion?: string;+ readonly toolName?: string;+}): McpUpdate {+ const method = input.method ?? input.message?.method;+ const params = input.params ?? (isObject(input.message?.params) ? input.message.params : {});+ const toolName =+ input.toolName ?? (typeof params["name"] === "string" ? params["name"] : undefined);+ const requestId = input.message?.id;+ const meta = params["_meta"];+ const protocolVersion =+ input.protocolVersion ??+ (isObject(meta) && typeof meta["io.modelcontextprotocol/protocolVersion"] === "string"+ ? meta["io.modelcontextprotocol/protocolVersion"]+ : undefined);+ return {+ connectionName: input.connectionName,+ method: method ?? "unknown",+ toolName,+ protocolVersion,+ requestId:+ typeof requestId === "number" || typeof requestId === "string"+ ? String(requestId)+ : undefined,+ };+}++interface JsonRpcRequest extends Record<string, unknown> {+ readonly method: string;+ readonly params?: unknown;+}++type McpTraceContextInjector = (context: Context, carrier: Record<string, string>) => void;++function injectOpenTelemetryTraceContext(context: Context, carrier: Record<string, string>): void {+ propagation.inject(context as OtelContext, carrier, traceContextSetter);+}++function readJsonRpcRequest(body: RequestInit["body"]): JsonRpcRequest | undefined {+ if (typeof body !== "string" || !withinTraceRequestLimit(body)) return undefined;+ try {+ const value: unknown = JSON.parse(body);+ return isObject(value) && typeof value["method"] === "string"+ ? (value as JsonRpcRequest)+ : undefined;+ } catch {+ return undefined;+ }+}++function withinTraceRequestLimit(value: string): boolean {+ return (+ value.length <= MAX_MCP_TRACE_REQUEST_BYTES &&+ utf8Encoder.encode(value).byteLength <= MAX_MCP_TRACE_REQUEST_BYTES+ );+}++function withinTraceContextLimit(carrier: Readonly<Record<string, string>>): boolean {+ let bytes = 0;+ for (const [key, value] of Object.entries(carrier)) {+ if (key.length + value.length > MAX_MCP_TRACE_CONTEXT_BYTES) return false;+ bytes += utf8Encoder.encode(key).byteLength + utf8Encoder.encode(value).byteLength + 2;+ if (bytes > MAX_MCP_TRACE_CONTEXT_BYTES) return false;+ }+ return bytes > 0;+}++function headerValue(headers: RequestInit["headers"], name: string): string | undefined {+ if (headers === undefined) return undefined;+ try {+ return new Headers(headers).get(name) ?? undefined;+ } catch {+ return undefined;+ }+}++function isToolErrorResult(value: unknown): boolean {+ return isObject(value) && value["isError"] === true;+}++function jsonRpcErrorCode(error: unknown): string | undefined {+ if (!isObject(error)) return undefined;+ const code = error["code"];+ return typeof code === "number" || typeof code === "string" ? String(code) : undefined;+}++function errorType(error: unknown, code: string | undefined): string | undefined {+ if (code !== undefined) return code;+ return error instanceof Error ? error.name || "Error" : undefined;+}diff --git a/packages/eve/src/tracing/eve/metadata.ts b/packages/eve/src/tracing/eve/metadata.tsnew file mode 100644index 000000000..964cf8dcc--- /dev/null+++ b/packages/eve/src/tracing/eve/metadata.ts@@ -0,0 +1,71 @@+import type {+ AgentSessionTraceState,+ AgentTurnTraceState,+} from "#tracing/eve/agent-trace-state.js";+import { normalizeInstrumentationChannelKind } from "#internal/instrumentation.js";+import type { TurnInput } from "#compiled/@eve/tracing/index.js";+import { checkpointContent } from "#tracing/eve/operation-input.js";++export function eveActivationMetadata(input: {+ readonly session?: AgentSessionTraceState;+ readonly turn: AgentTurnTraceState;+ readonly sessionId: string;+}): Pick<TurnInput, "sequence" | "lineage" | "channel" | "attributes"> {+ const { session, turn, sessionId } = input;+ const lineage = turn.parentLineage ?? session?.parentLineage;+ const owns = lineage === undefined || turn.traceSessionId === sessionId;+ const delivery = turn.channelDelivery;+ const kind =+ delivery?.channelKind ??+ (!owns+ ? undefined+ : (session?.channelKind ??+ (session?.channelType === undefined+ ? undefined+ : normalizeInstrumentationChannelKind(session.channelType))));+ return {+ sequence: turn.sequence,+ lineage:+ lineage === undefined+ ? undefined+ : {+ agentName: turn.subagentName,+ parentCallId: lineage.callId,+ parentRunId: lineage.sessionId,+ },+ channel: {+ kind,+ origin:+ !owns || kind === undefined+ ? undefined+ : session?.scheduleId === undefined+ ? "channel"+ : "schedule",+ },+ attributes: {+ "agent.channel.audience": session?.channelAudience,+ "agent.session.title":+ owns && session?.decision?.action === "record" && session.decision.recordInputs+ ? session.title+ : undefined,+ "agent.schedule.id": lineage === undefined ? session?.scheduleId : undefined,+ "agent.principal.current.type": turn.currentPrincipal?.type,+ "agent.principal.current.id":+ turn.currentPrincipal && "id" in turn.currentPrincipal+ ? turn.currentPrincipal.id+ : undefined,+ "agent.principal.initiator.type": turn.initiatorPrincipal?.type,+ "agent.principal.initiator.id":+ turn.initiatorPrincipal && "id" in turn.initiatorPrincipal+ ? turn.initiatorPrincipal.id+ : undefined,+ "agent.channel.name": delivery?.channelName,+ "agent.channel.delivery.id": delivery?.deliveryId,+ "agent.channel.request.id": delivery?.requestId,+ "agent.channel.delivery.input":+ delivery?.inputAttribute === undefined+ ? undefined+ : JSON.stringify(checkpointContent(delivery.inputAttribute)),+ },+ };+}diff --git a/packages/eve/src/tracing/eve/operation-input.ts b/packages/eve/src/tracing/eve/operation-input.tsnew file mode 100644index 000000000..04cf3772d--- /dev/null+++ b/packages/eve/src/tracing/eve/operation-input.ts@@ -0,0 +1,63 @@+import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";++import { resolveConversationId } from "#shared/conversation-identity.js";+import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";+import {+ createAgentTracing,+ type AgentTelemetry,+ type AgentTracing,+ type TraceCheckpointer,+} from "#compiled/@eve/tracing/index.js";++export function traceSessionIdOf(scope: {+ readonly traceSessionId?: string;+ readonly rootSessionId?: string;+ readonly sessionId: string;+}): string {+ return scope.traceSessionId ?? scope.rootSessionId ?? scope.sessionId;+}++export function checkpointContent(value: string | undefined): unknown {+ if (value === undefined) return undefined;+ // Historical checkpoints contain truncated JSON; omit payloads, not the operation.+ try {+ return JSON.parse(value);+ } catch {+ return undefined;+ }+}++/** Identity of one eve turn within the shared tracing library. */+export function eveTurnIdentity(input: {+ readonly sessionId: string;+ readonly rootSessionId?: string;+ readonly turnId: string;+}) {+ return {+ conversationId: resolveConversationId(input.rootSessionId ?? input.sessionId),+ runId: input.sessionId,+ turnId: input.turnId,+ };+}++/** Content flags are recorded by eve's content processor, not withheld by the library. */+export function eveCapture(reference: SpanContext | undefined) {+ return {+ emit: ((reference?.traceFlags ?? 1) & 1) !== 0,+ recordInputs: true,+ recordOutputs: true,+ };+}++/** eve's agent tracing, checkpointed in its workflow-context trace state. */+export function createEveTracing(input: {+ readonly telemetry: AgentTelemetry;+ readonly stateStore: AgentTraceStateStore;+}): AgentTracing {+ const checkpointer: TraceCheckpointer = {+ get: (key) => input.stateStore.get("checkpoint", key),+ set: (key, value) => input.stateStore.set("checkpoint", key, value),+ delete: (key) => input.stateStore.delete("checkpoint", key),+ };+ return createAgentTracing({ telemetry: input.telemetry, checkpointer });+}diff --git a/packages/eve/src/tracing/eve/otel-configuration.ts b/packages/eve/src/tracing/eve/otel-configuration.tsnew file mode 100644index 000000000..9f170056e--- /dev/null+++ b/packages/eve/src/tracing/eve/otel-configuration.ts@@ -0,0 +1,84 @@+export interface SpanProcessor {+ forceFlush(): Promise<void>;+ onEnd(span: unknown): void;+ onStart(span: unknown, parentContext: unknown): void;+ shutdown(): Promise<void>;+}++export type SpanProcessorOrName = SpanProcessor | "auto";++export interface IdGenerator {+ generateSpanId(): string;+ generateTraceId(): string;+}++/**+ * A `SpanExporter`. Structural for the same reason the propagator is: the+ * instance comes from whichever `@opentelemetry/*` build the app installed, and+ * the host only ever hands it spans and waits for the callback.+ *+ * `code` is `ExportResultCode`: `0` succeeded, `1` failed.+ */+export interface SpanExporter {+ export(+ spans: readonly unknown[],+ resultCallback: (result: { code: number; error?: Error }) => void,+ ): void;+ forceFlush?(): Promise<void>;+ shutdown(): Promise<void>;+}++/**+ * A `TextMapPropagator`, or one of the names `@vercel/otel` resolves for you.+ * Structural rather than imported: the instance comes from whichever+ * `@opentelemetry/*` build the app installed, not from the host's.+ */+export type PropagatorOrName =+ | { inject(...args: never[]): void; extract(...args: never[]): unknown; fields(): string[] }+ | "auto"+ | "none"+ | "tracecontext"+ | "baggage";++/** A `Sampler`, or one of the names `@vercel/otel` resolves for you. */+export type SamplerOrName =+ | { shouldSample(...args: never[]): unknown; toString(): string }+ | "auto"+ | "always_off"+ | "always_on"+ | "parentbased_always_off"+ | "parentbased_always_on"+ | "parentbased_traceidratio"+ | "traceidratio";++/**+ * A `MetricReader`. Structural for the same reason the exporter is: the+ * instance comes from whichever `@opentelemetry/sdk-metrics` build the app+ * installed, and the host only ever hands it to `registerOTel`.+ */+export interface MetricReader {+ forceFlush(): Promise<void>;+ shutdown(): Promise<void>;+}++export interface Configuration {+ readonly attributes?: Readonly<Record<string, unknown>>;+ readonly autoDetectResources?: boolean;+ readonly idGenerator?: IdGenerator;+ readonly instrumentations?: readonly unknown[];+ readonly metricReaders?: readonly MetricReader[];+ readonly propagators?: readonly PropagatorOrName[];+ readonly serviceName?: string;+ readonly spanProcessors?: readonly SpanProcessorOrName[];+ readonly traceSampler?: SamplerOrName;+}++/** Resolved process-wide settings. Hosts discover and merge their own declarations. */+export interface OtelConfiguration {+ readonly instrumentations?: readonly unknown[];+ readonly metricReaders?: readonly MetricReader[];+ readonly propagators?: readonly PropagatorOrName[];+ readonly resource?: Readonly<Record<string, unknown>>;+ readonly sampler?: SamplerOrName;+ readonly spanProcessors: readonly SpanProcessorOrName[];+}diff --git a/packages/eve/src/tracing/eve/otel-declaration.test.ts b/packages/eve/src/tracing/eve/otel-declaration.test.tsindex fe46c0784..958592653 100644--- a/packages/eve/src/tracing/eve/otel-declaration.test.ts+++ b/packages/eve/src/tracing/eve/otel-declaration.test.ts@@ -5,7 +5,6 @@ import {agentRunsIntegration,collectOtelPipeline,isOtelDeclaration,- isOtelIntegration,managedOtelIntegration,otel,otelIntegration,@@ -47,16 +46,6 @@ describe("otel", () => {});describe("otelIntegration", () => {- it("passes declared processors through untouched", () => {- const first = processor();- const integration = otelIntegration({- spanProcessors: [first],- });-- expect(isOtelIntegration(integration)).toBe(true);- expect(integration.spanProcessors).toStrictEqual([first]);- });-it("wraps an exporter in a batching processor, after any declared ones", () => {const first = processor();const integration = otelIntegration({@@ -81,15 +70,6 @@ describe("otelIntegration", () => {});});-describe("agentRunsIntegration", () => {- it("declares the Agent Runs runtime processor", () => {- const integration = agentRunsIntegration();-- expect(integration.spanProcessors).toHaveLength(1);- expect(integration.spanProcessors[0]).not.toBe("auto");- });-});-describe("destination export policy", () => {it("does not redact content unless the export pipeline requests it", () => {let visibleAttributes: Readonly<Record<string, unknown>> | undefined;@@ -163,7 +143,7 @@ describe("collectOtelPipeline", () => {]);expect(collected.declared).toBe(true);- expect(collected.pipeline.spanProcessors).toStrictEqual([first, second, third]);+ expect(collected.configuration.spanProcessors).toStrictEqual([first, second, third]);});it("is declared by a destination alone, with no otel() beside it", () => {@@ -189,7 +169,7 @@ describe("collectOtelPipeline", () => {}),]);- expect(collected.pipeline).toMatchObject({+ expect(collected.configuration).toMatchObject({propagators: ["tracecontext"],resource: { "service.version": "abc" },sampler: "always_on",@@ -203,12 +183,6 @@ describe("collectOtelPipeline", () => {});});- it("captures complete spans whenever a destination is declared", () => {- const collected = collectOtelPipeline([otelIntegration()]);-- expect(collected.settings).toMatchObject({ recordInputs: true, recordOutputs: true });- });-// A process has one tracer provider, so letting the first declaration win// would silently discard the second — the failure this throw exists to stop.it("refuses a second otel() rather than picking one", () => {@@ -221,12 +195,6 @@ describe("collectOtelPipeline", () => {const instrumentation = { name: "test-instrumentation" };const collected = collectOtelPipeline([otel({ instrumentations: [instrumentation] })]);- expect(collected.pipeline.instrumentations).toStrictEqual([instrumentation]);- });-- it("defaults to undefined instrumentations when none are declared", () => {- const collected = collectOtelPipeline([otel()]);-- expect(collected.pipeline.instrumentations).toBeUndefined();+ expect(collected.configuration.instrumentations).toStrictEqual([instrumentation]);});});diff --git a/packages/eve/src/tracing/eve/otel-declaration.ts b/packages/eve/src/tracing/eve/otel-declaration.tsindex 50f2b626b..5f82409b6 100644--- a/packages/eve/src/tracing/eve/otel-declaration.ts+++ b/packages/eve/src/tracing/eve/otel-declaration.ts@@ -13,6 +13,9 @@ import type { JsonObject } from "#shared/json.js";import { batchSpanProcessor } from "#tracing/eve/batch-span-processor.js";import { contentFilteringProcessor } from "#tracing/eve/content-span-processor.js";import { vercelRuntimeSpanProcessor } from "#tracing/eve/vercel-runtime-span-exporter.js";+import type { OtelConfiguration } from "#tracing/eve/otel-configuration.js";+import { createLogger, formatError } from "#internal/logging.js";+const log = createLogger("tracing.export");import type { TraceCapturePolicy } from "#shared/trace-policy.js";export type {TraceCaptureContext,@@ -178,7 +181,16 @@ function createOtelIntegration(options: OtelIntegrationOptions): OtelIntegrationconst spanProcessors =options.traceExporter === undefined? declared- : [...declared, batchSpanProcessor(options.traceExporter)];+ : [+ ...declared,+ batchSpanProcessor(options.traceExporter, {+ diagnostic: (message, details) =>+ log.warn(message, {+ ...details,+ error: details.error === undefined ? undefined : formatError(details.error),+ }),+ }),+ ];return {[OTEL_INTEGRATION]: true,@@ -226,14 +238,6 @@ export function isOtelIntegration(value: unknown): value is OtelIntegration {}/** The one pipeline a process can register. @internal */-export interface OtelPipeline {- readonly instrumentations?: readonly unknown[];- readonly metricReaders?: readonly MetricReader[];- readonly propagators?: readonly PropagatorOrName[];- readonly resource?: Readonly<Record<string, unknown>>;- readonly sampler?: SamplerOrName;- readonly spanProcessors: readonly SpanProcessorOrName[];-}/** What the harness reads at turn time, as opposed to at registration. @internal */export interface OtelHarnessSettings {@@ -258,7 +262,7 @@ export interface CollectedOtel {* global tracer provider slot alone rather than register an empty pipeline.*/readonly declared: boolean;- readonly pipeline: OtelPipeline;+ readonly configuration: OtelConfiguration;readonly runtimeContextResolvers: readonly RuntimeContextResolver[];readonly settings: OtelHarnessSettings;}@@ -314,7 +318,7 @@ export function collectOtelPipeline(values: readonly unknown[]): CollectedOtel {}return {declared,- pipeline: {+ configuration: {instrumentations: options.instrumentations,metricReaders: metricReaders.length > 0 ? metricReaders : undefined,propagators: options.propagators,diff --git a/packages/eve/src/tracing/eve/otel-registration.scenario.test.ts b/packages/eve/src/tracing/eve/otel-registration.scenario.test.tsindex 694c77766..47f4d925c 100644--- a/packages/eve/src/tracing/eve/otel-registration.scenario.test.ts+++ b/packages/eve/src/tracing/eve/otel-registration.scenario.test.ts@@ -14,10 +14,8 @@ import {metrics as runtimeMetrics,trace as runtimeTrace,} from "#compiled/@opentelemetry/api/index.js";-import { registerOtelPipeline } from "#tracing/eve/otel-registration.js";-import { withErrorContent } from "#tracing/error-content-context.js";-import { withConversationId } from "#tracing/eve/conversation-context.js";-import { createLogger, logError } from "#internal/logging.js";+import { registerOtelPipeline } from "./otel-registration.js";+import { withConversationId } from "./conversation-context.js";const require = createRequire(import.meta.url);const authoredApi = require("@opentelemetry/api") as typeof import("@opentelemetry/api");@@ -37,7 +35,7 @@ afterEach(() => {(runtimeTrace as typeof runtimeTrace & { disable(): void }).disable();});-describe("registerOtelPipeline", () => {+describe("OTel registration", () => {it("samples using the real activation name and attributes without exporting probes", async () => {const exporter = new InMemorySpanExporter();const sampler = {@@ -56,7 +54,7 @@ describe("registerOtelPipeline", () => {toString: () => "activation-sampler",};const runtime = registerOtelPipeline({- pipeline: { sampler, spanProcessors: [new SimpleSpanProcessor(exporter)] },+ otel: { sampler, spanProcessors: [new SimpleSpanProcessor(exporter)] },serviceName: "researcher",});const operation = {@@ -84,7 +82,7 @@ describe("registerOtelPipeline", () => {async (sampler) => {vi.stubEnv("OTEL_TRACES_SAMPLER_ARG", "0");const runtime = registerOtelPipeline({- pipeline: { sampler, spanProcessors: [] },+ otel: { sampler, spanProcessors: [] },serviceName: "test",});expect(runtime.samplesTrace("a".repeat(32), { name: "invoke_agent test" })).toBe(false);@@ -92,31 +90,6 @@ describe("registerOtelPipeline", () => {},);- it("keeps logger-to-span error details behind the active output policy", async () => {- vi.spyOn(console, "error").mockImplementation(() => {});- const exporter = new InMemorySpanExporter();- const runtime = registerOtelPipeline({- pipeline: { spanProcessors: [new SimpleSpanProcessor(exporter)] },- serviceName: "test",- });- for (const allowed of [false, true]) {- const span = runtimeTrace.getTracer("eve.agent").startSpan(`logger-${allowed}`);- const active = withErrorContent(runtimeTrace.setSpan(COMPILED_ROOT_CONTEXT, span), allowed);- await runtimeContext.with(active, async () => {- logError(createLogger("test"), "operation failed", new Error("sensitive payload"));- });- span.end();- }- await runtime.forceFlush();- const [privateSpan, publicSpan] = exporter.getFinishedSpans();- expect(privateSpan?.status.code).toBe(2);- expect(privateSpan?.status.message).toBeUndefined();- expect(privateSpan?.events).toEqual([]);- expect(publicSpan?.status.message).toContain("sensitive payload");- expect(publicSpan?.events[0]?.attributes?.["exception.message"]).toContain("sensitive payload");- await runtime.shutdown();- });-it("delegates an authored tracer cached before registration", async () => {const authoredTracer = authoredApi.trace.getTracer("authored");expect(authoredTracer.startSpan("before-registration").isRecording()).toBe(false);@@ -124,7 +97,7 @@ describe("registerOtelPipeline", () => {const exporter = new InMemorySpanExporter();const processor = new SimpleSpanProcessor(exporter);const runtime = registerOtelPipeline({- pipeline: { spanProcessors: [processor] },+ otel: { spanProcessors: [processor] },serviceName: "weather",});const parent = runtimeTrace.getTracer("eve").startSpan("eve.parent");@@ -168,38 +141,19 @@ describe("registerOtelPipeline", () => {await runtime.shutdown();});- it("verifies tracer ownership when the sampler records nothing", () => {- expect(() =>- registerOtelPipeline({- pipeline: { sampler: "always_off", spanProcessors: [] },- serviceName: "weather",- }),- ).not.toThrow();- });-- it("reports the installed sampler's verdict for pre-allocated trace ids", () => {- const runtime = registerOtelPipeline({- pipeline: { sampler: "always_off", spanProcessors: [] },- serviceName: "weather",- });-- expect(runtime.samplesTrace(runtime.idGenerator.generateTraceId())).toBe(false);- });-it("passes the pre-allocated trace id to a custom sampler without exporting the probe", async () => {const seen: string[] = [];- const sampler: NonNullable<Parameters<typeof registerOtelPipeline>[0]["pipeline"]["sampler"]> =- {- shouldSample: (_context: unknown, traceId: string) => {- seen.push(traceId);- return { decision: traceId.startsWith("a") ? 2 : 0 };- },- toString: () => "test-sampler",- };+ const sampler: NonNullable<Parameters<typeof registerOtelPipeline>[0]["otel"]["sampler"]> = {+ shouldSample: (_context: unknown, traceId: string) => {+ seen.push(traceId);+ return { decision: traceId.startsWith("a") ? 2 : 0 };+ },+ toString: () => "test-sampler",+ };const exporter = new InMemorySpanExporter();const processor = new SimpleSpanProcessor(exporter);const runtime = registerOtelPipeline({- pipeline: {+ otel: {sampler,spanProcessors: [processor],},@@ -231,7 +185,7 @@ describe("registerOtelPipeline", () => {expect(() =>registerOtelPipeline({- pipeline: {+ otel: {spanProcessors: [{ forceFlush: async () => {}, onEnd: () => {}, onStart: () => {}, shutdown },],@@ -260,7 +214,7 @@ describe("registerOtelPipeline", () => {expect(() =>registerOtelPipeline({- pipeline: { spanProcessors: [] },+ otel: { spanProcessors: [] },serviceName: "weather",}),).toThrow(/another runtime already owns the global tracer provider/u);@@ -284,7 +238,7 @@ describe("registerOtelPipeline", () => {shutdown: vi.fn(async () => {}),};const runtime = registerOtelPipeline({- pipeline: { metricReaders: [reader], spanProcessors: [] },+ otel: { metricReaders: [reader], spanProcessors: [] },serviceName: "weather",});@@ -305,7 +259,7 @@ describe("registerOtelPipeline", () => {setTracerProvider: vi.fn(),};const runtime = registerOtelPipeline({- pipeline: { instrumentations: [instrumentation], spanProcessors: [] },+ otel: { instrumentations: [instrumentation], spanProcessors: [] },serviceName: "weather",});expect(instrumentation.enable).toHaveBeenCalledOnce();@@ -314,23 +268,6 @@ describe("registerOtelPipeline", () => {await runtime.shutdown();expect(instrumentation.disable).toHaveBeenCalledOnce();});-- it("does not export the private registration span", async () => {- const exporter = new InMemorySpanExporter();- const processor = new SimpleSpanProcessor(exporter);- const shutdown = vi.spyOn(processor, "shutdown");- const runtime = registerOtelPipeline({- pipeline: { spanProcessors: [processor] },- serviceName: "weather",- });-- trace.getTracer("test").startSpan("user.work").end();- await processor.forceFlush();-- expect(exporter.getFinishedSpans().map((span) => span.name)).toEqual(["user.work"]);- await runtime.shutdown();- expect(shutdown).toHaveBeenCalledOnce();- });});function currentTracerDelegate(): unknown {diff --git a/packages/eve/src/tracing/eve/otel-registration.test.ts b/packages/eve/src/tracing/eve/otel-registration.test.tsindex 2fcd9f4f9..3b4f33df3 100644--- a/packages/eve/src/tracing/eve/otel-registration.test.ts+++ b/packages/eve/src/tracing/eve/otel-registration.test.ts@@ -1,12 +1,12 @@import { beforeEach, describe, expect, it, vi } from "vitest";-import { registerOtelPipeline } from "#tracing/eve/otel-registration.js";+import { registerOtelPipeline } from "./otel-registration.js";const { registerOTel } = vi.hoisted(() => ({ registerOTel: vi.fn() }));vi.mock("#compiled/@vercel/otel/index.js", () => ({ registerOTel }));-describe("registerOtelPipeline", () => {+describe("OTel registration", () => {beforeEach(() => {delete (globalThis as Record<symbol, unknown>)[Symbol.for("eve.otel.replay-deduplication")];});@@ -16,7 +16,7 @@ describe("registerOtelPipeline", () => {expect(() =>registerOtelPipeline({- pipeline: {+ otel: {propagators: ["tracecontext"],resource: { "service.version": "abc" },sampler: "always_on",@@ -42,7 +42,7 @@ describe("registerOtelPipeline", () => {registerOTel.mockImplementation(() => undefined);expect(() =>- registerOtelPipeline({ pipeline: { spanProcessors: [] }, serviceName: "weather" }),+ registerOtelPipeline({ otel: { spanProcessors: [] }, serviceName: "weather" }),).toThrow(/already owns the global tracer provider/u);const configuration = registerOTel.mock.calls.at(-1)?.[0] as Record<string, unknown>;@@ -51,43 +51,11 @@ describe("registerOtelPipeline", () => {expect(configuration["propagators"]).toEqual(["auto", expect.any(Object)]);});- it("filters the private registration span from authored processors", () => {- const downstream = {- forceFlush: vi.fn(async () => {}),- onEnd: vi.fn(),- onStart: vi.fn(),- shutdown: vi.fn(async () => {}),- };- registerOTel.mockImplementation(() => undefined);-- expect(() =>- registerOtelPipeline({- pipeline: { spanProcessors: [downstream] },- serviceName: "weather",- }),- ).toThrow();-- const configuration = registerOTel.mock.calls.at(-1)?.[0] as {- spanProcessors: {- onEnd(span: unknown): void;- onStart(span: unknown, parentContext: unknown): void;- }[];- };- const processor = configuration.spanProcessors[0]!;- processor.onStart({ name: "eve.otel.registration" }, {});- processor.onEnd({ name: "eve.otel.registration" });- processor.onStart({ name: "agent.turn" }, {});- processor.onEnd({ name: "agent.turn" });-- expect(downstream.onStart).toHaveBeenCalledExactlyOnceWith({ name: "agent.turn" }, {});- expect(downstream.onEnd).toHaveBeenCalledExactlyOnceWith({ name: "agent.turn" });- });-it("passes Vercel's automatic processor through", () => {registerOTel.mockImplementation(() => undefined);expect(() =>- registerOtelPipeline({ pipeline: { spanProcessors: ["auto"] }, serviceName: "weather" }),+ registerOtelPipeline({ otel: { spanProcessors: ["auto"] }, serviceName: "weather" }),).toThrow();const configuration = registerOTel.mock.calls.at(-1)?.[0] as {@@ -113,7 +81,7 @@ describe("registerOtelPipeline", () => {expect(() =>registerOtelPipeline({- pipeline: { spanProcessors: [first, second] },+ otel: { spanProcessors: [first, second] },serviceName: "weather",}),).toThrow();@@ -153,7 +121,7 @@ describe("registerOtelPipeline", () => {registerOTel.mockImplementation(() => undefined);expect(() =>registerOtelPipeline({- pipeline: { spanProcessors: [downstream] },+ otel: { spanProcessors: [downstream] },serviceName: "weather",}),).toThrow();@@ -176,11 +144,43 @@ describe("registerOtelPipeline", () => {expect(downstream.onEnd.mock.calls).toEqual([[parent], [child]]);});+ it("filters the private registration span from authored processors", () => {+ const downstream = {+ forceFlush: vi.fn(async () => {}),+ onEnd: vi.fn(),+ onStart: vi.fn(),+ shutdown: vi.fn(async () => {}),+ };+ registerOTel.mockImplementation(() => undefined);++ expect(() =>+ registerOtelPipeline({+ otel: { spanProcessors: [downstream] },+ serviceName: "weather",+ }),+ ).toThrow();++ const configuration = registerOTel.mock.calls.at(-1)?.[0] as {+ spanProcessors: {+ onEnd(span: unknown): void;+ onStart(span: unknown, parentContext: unknown): void;+ }[];+ };+ const processor = configuration.spanProcessors[0]!;+ processor.onStart({ name: "eve.otel.registration" }, {});+ processor.onEnd({ name: "eve.otel.registration" });+ processor.onStart({ name: "agent.turn" }, {});+ processor.onEnd({ name: "agent.turn" });++ expect(downstream.onStart).toHaveBeenCalledExactlyOnceWith({ name: "agent.turn" }, {});+ expect(downstream.onEnd).toHaveBeenCalledExactlyOnceWith({ name: "agent.turn" });+ });+it("throws when the registration never reached a processor", () => {registerOTel.mockImplementation(() => undefined);expect(() =>- registerOtelPipeline({ pipeline: { spanProcessors: [] }, serviceName: "weather" }),+ registerOtelPipeline({ otel: { spanProcessors: [] }, serviceName: "weather" }),).toThrow(/another runtime already owns/u);});@@ -261,7 +261,7 @@ function filteringProcessor() {registerOTel.mockImplementation(() => undefined);expect(() =>registerOtelPipeline({- pipeline: { spanProcessors: [downstream] },+ otel: { spanProcessors: [downstream] },serviceName: "weather",}),).toThrow();diff --git a/packages/eve/src/tracing/eve/otel-registration.ts b/packages/eve/src/tracing/eve/otel-registration.tsindex 850b55805..071054cd0 100644--- a/packages/eve/src/tracing/eve/otel-registration.ts+++ b/packages/eve/src/tracing/eve/otel-registration.ts@@ -7,21 +7,18 @@ import {trace,SpanKind,type Context,+ type TracerProvider,} from "#compiled/@opentelemetry/api/index.js";-import {- registerOTel,- type Configuration,- type SpanProcessor,- type SpanProcessorOrName,-} from "#compiled/@vercel/otel/index.js";+import { registerOTel } from "#compiled/@vercel/otel/index.js";+import type { Configuration, SpanProcessor, SpanProcessorOrName } from "./otel-configuration.js";++import { AgentSpanIdGenerator } from "#compiled/@eve/tracing/index.js";+import type { OtelConfiguration } from "./otel-configuration.js";+import { type Attributes } from "#compiled/@eve/tracing/index.js";+import { invocationName } from "#compiled/@eve/tracing/otel.js";+export type SamplingOperation = { readonly name: string; readonly attributes?: Attributes };-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";import { conversationIdFromContext } from "#tracing/eve/conversation-context.js";-import type { OtelPipeline } from "#tracing/eve/otel-declaration.js";-import {- agentInvocationSpanName,- type AgentSamplingOperation,-} from "#tracing/agent-span-contract.js";const REGISTRATION_SPAN_NAME = "eve.otel.registration";const REPLAY_DEDUPLICATION_LIMIT = 100_000;@@ -163,10 +160,10 @@ class PrivateSpanFilteringProcessor implements SpanProcessor {* propagator, then verifies both through the global APIs.*/export function registerOtelPipeline(input: {- readonly pipeline: OtelPipeline;+ readonly otel: OtelConfiguration;readonly serviceName: string;}): RegisteredOtelPipeline {- const { pipeline } = input;+ const pipeline = input.otel;// A tracer cached before registration retains this private proxy even after// the vendored API takes the global slot. Delegate it only after ownership is proven.const optionalPeerTracerProxy = captureOptionalPeerTracerProxy();@@ -218,6 +215,7 @@ export function registerOtelPipeline(input: {// global is a no-op provider with no lifecycle methods.const meterProvider = runtimeMeterProvider();return {+ provider: trace.getTracerProvider() as TracerProvider,forceFlush: async () => {await Promise.all([provider.forceFlush!(), meterProvider.forceFlush?.()]);},@@ -260,17 +258,19 @@ function replayDeduplicationRegistry(): Set<string> {/** Lifecycle retained from the providers that own every destination. */export interface RegisteredOtelPipeline {+ readonly provider: TracerProvider;readonly forceFlush: () => Promise<void>;readonly idGenerator: AgentSpanIdGenerator;/** Whether the installed sampler would record a trace with this id. */- readonly samplesTrace: (traceId: string, operation?: AgentSamplingOperation) => boolean;+ readonly samplesTrace: (traceId: string, operation?: SamplingOperation) => boolean;readonly shutdown: () => Promise<void>;}+export const registerOtel = registerOtelPipeline;function samplerAdmitsTrace(idGenerator: AgentSpanIdGenerator,traceId: string,- operation: AgentSamplingOperation = { name: agentInvocationSpanName(undefined) },+ operation: SamplingOperation = { name: invocationName(undefined) },): boolean {const probe = idGenerator.withTraceId(traceId, () =>trace.getTracer("eve.registration").startSpan(operation.name, {@@ -296,20 +296,6 @@ interface ProxyTracerProvider {getDelegate(): unknown;}-interface OptionalPeerTracerProxy {- setDelegate(delegate: unknown): void;-}--function captureOptionalPeerTracerProxy(): OptionalPeerTracerProxy | undefined {- try {- const api = require("@opentelemetry/api") as typeof import("@opentelemetry/api");- const provider = api.trace.getTracerProvider();- return provider instanceof api.ProxyTracerProvider ? provider : undefined;- } catch {- return undefined;- }-}-function rollbackRegistration(input: {readonly ownsPropagator: boolean;readonly ownsTracer: boolean;@@ -417,3 +403,17 @@ function addBounded(values: Set<string>, value: string): void {function isSpanProcessor(processor: SpanProcessorOrName): processor is SpanProcessor {return processor !== "auto";}++interface OptionalPeerTracerProxy {+ setDelegate(delegate: unknown): void;+}++function captureOptionalPeerTracerProxy(): OptionalPeerTracerProxy | undefined {+ try {+ const api = require("@opentelemetry/api") as typeof import("@opentelemetry/api");+ const provider = api.trace.getTracerProvider();+ return provider instanceof api.ProxyTracerProvider ? provider : undefined;+ } catch {+ return undefined;+ }+}diff --git a/packages/eve/src/tracing/eve/policy.ts b/packages/eve/src/tracing/eve/policy.tsnew file mode 100644index 000000000..88164cf82--- /dev/null+++ b/packages/eve/src/tracing/eve/policy.ts@@ -0,0 +1,115 @@+import { contextStorage } from "#context/container.js";+import { SessionTraceSeedKey } from "#context/keys.js";+import { withoutInstrumentationContent } from "#instrumentation/content.js";+import { instrumentationEventForTraceDecision } from "#instrumentation/content-policy.js";+import type {+ InstrumentationEvent,+ InstrumentationAttemptScope,+} from "#instrumentation/lifecycle.js";+import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";+import { isSampledTrace, resolveTracePolicyDecision } from "#shared/trace-policy.js";+import {+ applyLiveDeliveryAudienceCeiling,+ resolveForwardedTraceSeed,+} from "#shared/forwarded-trace-policy.js";+import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";+import { normalizeChannelAudience, type ChannelAudience } from "#shared/channel-audience.js";+import type { ConversationEnvironment } from "#shared/conversation-context.js";++export function createEveCapturePolicy(input: {+ readonly stateStore: AgentTraceStateStore;+ readonly environment: ConversationEnvironment;+ readonly recordInputs: boolean;+ readonly recordOutputs: boolean;+}) {+ return {+ async projectEvent(event: InstrumentationEvent): Promise<InstrumentationEvent> {+ const session = await input.stateStore.get(+ "session",+ "scope" in event ? event.scope.sessionId : event.sessionId,+ );+ const audience = audienceForEvent(event, session?.channelAudience);+ const eventSeed = "traceSeed" in event ? event.traceSeed : undefined;+ const contextSeed = contextStorage.getStore()?.get(SessionTraceSeedKey);+ const contextTraceState = resolveForwardedTraceSeed(contextSeed);+ const eventTraceState = resolveForwardedTraceSeed(+ eventSeed,+ contextTraceState?.forwardedTracePolicy,+ );+ const decisionForTrace = (trace: { readonly traceFlags: number } | undefined) =>+ trace === undefined+ ? undefined+ : resolveTracePolicyDecision(isSampledTrace(trace), {+ audience,+ environment: input.environment,+ });+ const decision =+ eventTraceState?.decision ??+ contextTraceState?.decision ??+ readInstrumentationDecision(session?.decision) ??+ decisionForTrace(eventSeed) ??+ decisionForTrace(contextSeed) ??+ decisionForTrace(session?.context);+ if (decision === undefined) return withoutInstrumentationContent(event);+ const normalizedEvent =+ eventTraceState === undefined || !("traceSeed" in event) || event.traceSeed === undefined+ ? event+ : {+ ...event,+ traceSeed: {+ ...event.traceSeed,+ decision: eventTraceState.decision,+ traceFlags: eventTraceState.traceFlags,+ },+ };+ return instrumentationEventForTraceDecision(+ normalizedEvent,+ applyLiveDeliveryAudienceCeiling(+ decision.action === "drop"+ ? decision+ : {+ action: "record",+ recordInputs: input.recordInputs && decision.recordInputs,+ recordOutputs: input.recordOutputs && decision.recordOutputs,+ },+ audience,+ eventTraceState?.forwardedTracePolicy ?? contextTraceState?.forwardedTracePolicy,+ input.environment,+ ),+ { audience, environment: input.environment },+ { applyAudienceCeiling: false },+ );+ },+ async forOperation(scope: InstrumentationAttemptScope) {+ const session = await input.stateStore.get("session", scope.sessionId);+ const seed = resolveForwardedTraceSeed(contextStorage.getStore()?.get(SessionTraceSeedKey));+ const decision = seed?.decision ?? session?.decision;+ const effective =+ decision === undefined+ ? undefined+ : applyLiveDeliveryAudienceCeiling(+ decision,+ normalizeChannelAudience(scope.channelAudience),+ seed?.forwardedTracePolicy,+ input.environment,+ );+ return {+ recordInputs:+ input.recordInputs && effective?.action === "record" && effective.recordInputs,+ recordOutputs:+ input.recordOutputs && effective?.action === "record" && effective.recordOutputs,+ };+ },+ };+}++function audienceForEvent(+ event: InstrumentationEvent,+ sessionAudience: ChannelAudience | undefined,+): ChannelAudience {+ if ("delivery" in event) return normalizeChannelAudience(event.delivery.channelAudience);+ if ("scope" in event && event.scope.channelAudience !== undefined)+ return normalizeChannelAudience(event.scope.channelAudience);+ if (event.type === "session.started") return normalizeChannelAudience(event.channelAudience);+ return normalizeChannelAudience(sessionAudience);+}diff --git a/packages/eve/src/tracing/eve/profile.ts b/packages/eve/src/tracing/eve/profile.tsnew file mode 100644index 000000000..58269bdb5--- /dev/null+++ b/packages/eve/src/tracing/eve/profile.ts@@ -0,0 +1,51 @@+import type { Attributes, MappingContext, OutputMapping } from "#compiled/@eve/tracing/index.js";+import { currentTraceSessionId } from "#tracing/eve/agent-trace-context-store.js";++/** eve publishes tool call kind and outcome under its schema-4 `agent.action.*` names. */+const TOOL_ATTRIBUTES: Readonly<Record<string, string>> = {+ "agent.tool.kind": "agent.action.kind",+ "agent.tool.outcome": "agent.action.outcome",+};++export function eveOutputMapping(options?: {+ readonly resolve: (span: MappingContext) => {+ readonly traceSessionId?: string;+ readonly platform: "vercel" | "other";+ };+}): OutputMapping {+ return {+ attributes(span, attributes) {+ const output: Record<string, Attributes[string]> = {};+ for (const [key, value] of Object.entries(attributes)) {+ if (key === "agent.connection.name") output["eve.connection.name"] = value;+ else if (span.type === "tool" && TOOL_ATTRIBUTES[key] !== undefined)+ output[TOOL_ATTRIBUTES[key]] = value;+ else if (span.type === "approval" && key === "gen_ai.tool.name")+ output["agent.action.name"] = value;+ else if (span.type === "channelRequest" && key === "agent.channel.name")+ output["eve.channel.name"] = value;+ else if (span.type === "channelRequest" && key === "agent.channel.kind")+ output["eve.channel.kind"] = value;+ else output[key] = value;+ }+ if (span.type === "tool" && attributes["agent.tool.kind"] !== undefined)+ output["agent.action.name"] = attributes["gen_ai.tool.name"];+ if (attributes["agent.run.id"] !== undefined) {+ const context = options?.resolve(span) ?? {+ platform: process.env.VERCEL_ENV === undefined ? "other" : "vercel",+ traceSessionId:+ typeof attributes["agent.run.id"] === "string"+ ? currentTraceSessionId(attributes["agent.run.id"])+ : undefined,+ };+ if (context?.platform === "vercel" && context.traceSessionId !== undefined)+ output["vercel.session_id"] = context.traceSessionId;+ }+ return output;+ },+ link: (_span, link) => ({+ "eve.link.type":+ link.relationship === "execution.delivery" ? "workflow.delivery" : link.relationship,+ }),+ };+}diff --git a/packages/eve/src/tracing/eve/session.ts b/packages/eve/src/tracing/eve/session.tsnew file mode 100644index 000000000..3f040d0c2--- /dev/null+++ b/packages/eve/src/tracing/eve/session.ts@@ -0,0 +1,355 @@+import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";++import {+ type InstrumentationSessionStartedEvent,+ type InstrumentationTraceContext,+ type InstrumentationTraceSeed,+ type InstrumentationTurnStartedEvent,+ type InstrumentationTurnTerminalEvent,+ type InstrumentationSessionTransitionEvent,+} from "#instrumentation/lifecycle.js";+import { normalizeChannelAudience } from "#shared/channel-audience.js";+import type { ChannelAudience } from "#shared/channel-audience.js";+import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";+import {+ isSampledTrace,+ resolveTracePolicy,+ resolveTracePolicyDecision,+} from "#shared/trace-policy.js";+import type {+ AgentSessionTraceState,+ AgentTraceStateStore,+} from "#tracing/eve/agent-trace-state.js";+import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";+import { eveActivationMetadata } from "#tracing/eve/metadata.js";+import { eveCapture, eveTurnIdentity } from "#tracing/eve/operation-input.js";+import type { AgentTurnTraceState } from "#tracing/eve/agent-trace-state.js";+import { applyPrincipalTraceDecision } from "#instrumentation/principal-summary.js";+import { normalizeInstrumentationChannelKind } from "#internal/instrumentation.js";+import type { ConversationEnvironment } from "#shared/conversation-context.js";+import { traceSessionIdOf } from "#tracing/eve/operation-input.js";+import { ROOT_CONTEXT } from "#compiled/@opentelemetry/api/index.js";+import { withChannelAudience } from "#tracing/eve/channel-audience-context.js";+import { withOperationConversation } from "#tracing/eve/conversation-context.js";+import type {+ AgentSpanIdGenerator,+ AgentTracing,+ ExecutionContext,+ TraceLink,+ TurnOperation,+} from "#compiled/@eve/tracing/index.js";++interface EveSessionTracingInput {+ readonly tracing: AgentTracing;+ readonly idGenerator: AgentSpanIdGenerator;+ readonly environment: ConversationEnvironment;+ readonly frameworkVersion: string;+ readonly stateStore: AgentTraceStateStore;+ readonly tracePolicy?: TraceCapturePolicy;+}++type SessionMetadata = Omit<InstrumentationSessionStartedEvent, "idempotencyKey" | "type">;++interface EveSessionTracing {+ readonly ensureSessionContext: (event: SessionMetadata) => Promise<AgentSessionTraceState>;+ readonly prepareSessionTrace: (+ event: InstrumentationSessionStartedEvent,+ ) => Promise<InstrumentationTraceSeed>;+ readonly prepareTurnTrace: (+ event: InstrumentationTurnStartedEvent,+ ) => Promise<InstrumentationTraceSeed>;+ /** Hydrates a turn this process or an earlier one started. */+ turnFor(+ sessionId: string,+ turnId: string,+ context?: ExecutionContext,+ ): Promise<TurnOperation | undefined>;+ /** Whether the trace policy dropped this turn, so its work must not record spans. */+ dropped(sessionId: string, turnId: string): Promise<boolean>;+ turnTerminal(event: InstrumentationTurnTerminalEvent): Promise<void>;+ sessionTransition(event: InstrumentationSessionTransitionEvent): Promise<void>;+}++export function createEveSessionTracing(input: EveSessionTracingInput): EveSessionTracing {+ const ensureSessionContext = async (event: SessionMetadata): Promise<AgentSessionTraceState> => {+ let state = await input.stateStore.get("session", event.sessionId);+ if (state === undefined) {+ const channelAudience = normalizeChannelAudience(event.channelAudience);+ const decision = resolveSessionTraceDecision(+ event,+ channelAudience,+ input.environment,+ input.tracePolicy,+ );+ state = {+ agentName: event.agentName,+ channelAudience,+ channelKind: event.channelKind,+ channelType: event.channelType,+ decision,+ context: initialSessionContext(input, event, decision),+ parentLineage: event.parentLineage,+ rootSessionId: event.rootSessionId,+ traceSessionId: traceSessionIdOf(event),+ scheduleId: event.scheduleId,+ title: event.title,+ };+ await input.stateStore.set("session", event.sessionId, state);+ }+ return state;+ };++ const prepareSessionTrace = async (+ event: InstrumentationSessionStartedEvent,+ ): Promise<InstrumentationTraceSeed> => {+ const session = await ensureSessionContext(event);+ return portableSpanContext(session.context, session.decision);+ };++ const turnKey = (sessionId: string, turnId: string) => JSON.stringify([sessionId, turnId]);++ async function turnFor(sessionId: string, turnId: string, context?: ExecutionContext) {+ const turn = await input.stateStore.get("turn", turnKey(sessionId, turnId));+ if (turn === undefined || !isSampledTrace(turn.context)) return undefined;+ const session = await input.stateStore.get("session", sessionId);+ return input.tracing.resume({+ identity: eveTurnIdentity({ sessionId, rootSessionId: turn.rootSessionId, turnId }),+ context: context ?? hostContext(session?.channelAudience, { sessionId, ...turn }),+ });+ }++ async function dropped(sessionId: string, turnId: string): Promise<boolean> {+ const turn = await input.stateStore.get("turn", turnKey(sessionId, turnId));+ return turn !== undefined && !isSampledTrace(turn.context);+ }++ const prepareTurnTrace = async (+ event: InstrumentationTurnStartedEvent,+ ): Promise<InstrumentationTraceSeed> => {+ const prepared = await input.stateStore.get("turn", turnKey(event.sessionId, event.turnId));+ if (prepared !== undefined) {+ const session = await input.stateStore.get("session", event.sessionId);+ return portableSpanContext(prepared.context, session?.decision);+ }++ const session = await ensureSessionContext(event);+ const useInitialContext = event.sequence === 0;+ const caller = useInitialContext ? event.parentTraceContext : undefined;+ const emit = session.decision?.action === "record";+ // A local caller's first child turn joins the caller's trace beneath its call.+ const nested = caller !== undefined && !("isRemote" in caller && caller.isRemote === true);+ const reserved = useInitialContext+ ? {+ ...session.context,+ traceId: nested ? caller.traceId : session.context.traceId,+ isRemote: false,+ }+ : {+ traceId: input.idGenerator.deriveTraceId(`turn:${event.idempotencyKey}`),+ spanId: input.idGenerator.deriveSpanId(`turn:${event.idempotencyKey}`),+ traceFlags: emit ? 1 : 0,+ isRemote: false,+ };+ const agentName =+ session.agentName ?? (event.parentLineage ?? session.parentLineage)?.subagentName;+ let turn: AgentTurnTraceState = {+ agentName,+ caller: caller === undefined ? undefined : adoptedSpanContext(caller),+ context: reserved,+ currentPrincipal: applyPrincipalTraceDecision(event.currentPrincipal, session.decision),+ initiatorPrincipal: applyPrincipalTraceDecision(event.initiatorPrincipal, session.decision),+ parentLineage: event.parentLineage ?? session.parentLineage,+ rootSessionId: event.rootSessionId,+ traceSessionId: traceSessionIdOf(event),+ sequence: event.sequence,+ startTimeMs: Date.now(),+ subagentName: (event.parentLineage ?? session.parentLineage)?.subagentName,+ };+ if (isSampledTrace(reserved)) {+ const operation = await input.tracing.turn({+ ...eveActivationMetadata({ session, turn, sessionId: event.sessionId }),+ agentName,+ identity: eveTurnIdentity({ ...event, sessionId: event.sessionId }),+ framework: { name: "eve", version: input.frameworkVersion },+ capture: eveCapture(reserved),+ reference: reserved,+ parent: nested ? adoptedSpanContext(caller) : undefined,+ links: activationLinks(turn),+ startTimeMs: turn.startTimeMs,+ context: hostContext(session.channelAudience, event),+ });+ turn = { ...turn, context: { ...reserved, traceFlags: operation.reference.traceFlags } };+ }+ await input.stateStore.set("turn", turnKey(event.sessionId, event.turnId), turn);+ return portableSpanContext(turn.context, session.decision);+ };++ async function recordTerminal(+ sessionId: string,+ turnId: string,+ terminal: NonNullable<AgentTurnTraceState["terminal"]>,+ error?: unknown,+ ) {+ await input.stateStore.update("turn", turnKey(sessionId, turnId), (turn) => ({+ ...turn,+ terminal:+ terminal.outcome !== "failed"+ ? terminal+ : {+ outcome: "failed",+ errorName: error instanceof Error ? error.name : undefined,+ errorMessage: error instanceof Error ? error.message : undefined,+ },+ }));+ }++ return {+ ensureSessionContext,+ prepareSessionTrace,+ prepareTurnTrace,+ turnFor,+ dropped,+ async turnTerminal(event) {+ await recordTerminal(+ event.sessionId,+ event.turnId,+ {+ outcome:+ event.type === "turn.failed"+ ? "failed"+ : event.type === "turn.cancelled"+ ? "cancelled"+ : "completed",+ },+ event.type === "turn.failed" ? event.error : undefined,+ );+ },+ async sessionTransition(event) {+ if (event.turnId === undefined) return;+ if (event.type === "session.failed")+ await recordTerminal(event.sessionId, event.turnId, { outcome: "failed" }, event.error);+ const turn = await input.stateStore.get("turn", turnKey(event.sessionId, event.turnId));+ if (turn === undefined) return;+ const session = await input.stateStore.get("session", event.sessionId);+ const operation = await turnFor(event.sessionId, event.turnId);+ if (operation !== undefined) {+ const metadata = eveActivationMetadata({ session, turn, sessionId: event.sessionId });+ operation.attributes(metadata.attributes ?? {});+ operation.links(activationLinks(turn) ?? []);+ const terminal = turn.terminal;+ // A turn the session left without a terminal event ends with no outcome, as on main.+ if (terminal === undefined) await operation.complete({ outcomeUnknown: true });+ else if (terminal.outcome === "failed") {+ const error = new Error(terminal.errorMessage);+ error.name = terminal.errorName ?? "Error";+ await operation.complete({+ outcome: "failed",+ failed: true,+ error,+ errorType: terminal.errorName,+ });+ } else await operation.complete({ outcome: terminal.outcome });+ }+ await input.stateStore.delete("turn", turnKey(event.sessionId, event.turnId));+ },+ };+}++/** Spans this process starts inherit the channel audience and the operation's conversation. */+function hostContext(+ audience: ChannelAudience | undefined,+ operation: { readonly sessionId: string; readonly rootSessionId?: string },+) {+ return withOperationConversation(withChannelAudience(ROOT_CONTEXT, audience), operation);+}++function activationLinks(turn: AgentTurnTraceState): TraceLink[] | undefined {+ const links: TraceLink[] = [];+ // A local caller is the turn's parent; only a remote one is linked.+ if (turn.caller?.isRemote === true)+ links.push({ context: turn.caller, relationship: "agent.dispatch" });+ if (turn.channelDelivery?.requestTraceContext !== undefined)+ links.push({+ context: turn.channelDelivery.requestTraceContext,+ relationship: "channel.request",+ });+ return links.length === 0 ? undefined : links;+}++function portableSpanContext(+ spanContext: SpanContext,+ decision?: InstrumentationTraceSeed["decision"],+): InstrumentationTraceSeed {+ return {+ decision,+ spanId: spanContext.spanId,+ traceFlags: spanContext.traceFlags,+ traceId: spanContext.traceId,+ };+}++function adoptedSpanContext(handed: InstrumentationTraceContext): SpanContext {+ return {+ isRemote: "isRemote" in handed && handed.isRemote === true,+ spanId: handed.spanId,+ traceFlags: handed.traceFlags,+ traceId: handed.traceId,+ };+}++function initialSessionContext(+ input: EveSessionTracingInput,+ event: SessionMetadata,+ decision: ReturnType<typeof resolveTracePolicy>,+): SpanContext {+ const handed = event.traceSeed;+ if (handed !== undefined) {+ return {+ ...adoptedSpanContext(handed),+ traceFlags: decision.action === "drop" ? 0 : handed.traceFlags,+ };+ }+ const key = `session:${event.sessionId}`;+ const parent = event.parentTraceContext;+ const local = parent !== undefined && !("isRemote" in parent && parent.isRemote === true);+ return {+ traceId: local ? parent.traceId : input.idGenerator.deriveTraceId(key),+ spanId: input.idGenerator.deriveSpanId(key),+ traceFlags: decision.action === "record" ? 1 : 0,+ };+}++function resolveSessionTraceDecision(+ event: SessionMetadata,+ audience: ChannelAudience,+ environment: ConversationEnvironment,+ policy: TraceCapturePolicy | undefined,+): ReturnType<typeof resolveTracePolicy> {+ const content = { audience, environment };+ if (event.parentTraceContext !== undefined && !isSampledTrace(event.parentTraceContext)) {+ return { action: "drop" };+ }+ if (event.traceSeed?.decision !== undefined) {+ return readInstrumentationDecision(event.traceSeed.decision) ?? { action: "drop" };+ }+ if (event.traceSeed !== undefined) {+ return resolveTracePolicyDecision(isSampledTrace(event.traceSeed), content);+ }+ if (event.parentTraceContext !== undefined) {+ return resolveTracePolicyDecision(isSampledTrace(event.parentTraceContext), content);+ }+ if (event.agentName === undefined) {+ return policy === undefined ? resolveTracePolicyDecision(true, content) : { action: "drop" };+ }+ // The tool loop can evaluate the same policy before this first-session+ // preparation path; the persisted decision removes that window on replay.+ return resolveTracePolicy(policy, {+ agentName: event.agentName,+ audience,+ channel: {+ kind: normalizeInstrumentationChannelKind(event.channelKind ?? event.channelType),+ },+ environment,+ principalType: "unknown",+ });+}diff --git a/packages/eve/src/tracing/eve/telemetry-budget.test.ts b/packages/eve/src/tracing/eve/telemetry-budget.test.tsindex 0eb8de393..a9d498990 100644--- a/packages/eve/src/tracing/eve/telemetry-budget.test.ts+++ b/packages/eve/src/tracing/eve/telemetry-budget.test.ts@@ -3,7 +3,7 @@ import {boundedPrincipalId,telemetryByteLength,truncateTelemetryText,-} from "#tracing/telemetry-budget.js";+} from "#compiled/@eve/tracing/otel.js";describe("boundedPrincipalId", () => {it.each(["x".repeat(1024), String.fromCodePoint(0x1f600).repeat(256)])(diff --git a/packages/eve/src/tracing/eve/test-data/remote-local-agent-trace.otlp.json b/packages/eve/src/tracing/eve/test-data/remote-local-agent-trace.otlp.jsonindex d8c9a0766..e61754306 100644--- a/packages/eve/src/tracing/eve/test-data/remote-local-agent-trace.otlp.json+++ b/packages/eve/src/tracing/eve/test-data/remote-local-agent-trace.otlp.json@@ -45,51 +45,45 @@"endTimeUnixNano": "1700000000001000000","attributes": [{- "key": "agent.framework.name",- "value": {- "stringValue": "eve"- }- },- {- "key": "agent.framework.version",+ "key": "agent.run.id","value": {- "stringValue": "test"+ "stringValue": "remote-session"}},{- "key": "agent.name",+ "key": "agent.trace.schema.version","value": {- "stringValue": "remote"+ "intValue": 4}},{- "key": "agent.channel.audience",+ "key": "gen_ai.conversation.id","value": {- "stringValue": "public"+ "stringValue": "caller-root"}},{- "key": "agent.principal.current.id",+ "key": "agent.framework.name","value": {- "stringValue": "current-user"+ "stringValue": "eve"}},{- "key": "agent.principal.current.type",+ "key": "agent.framework.version","value": {- "stringValue": "service"+ "stringValue": "test"}},{- "key": "agent.principal.initiator.id",+ "key": "operation.name","value": {- "stringValue": "initiator-user"+ "stringValue": "invoke_agent"}},{- "key": "agent.principal.initiator.type",+ "key": "resource.name","value": {- "stringValue": "user"+ "stringValue": "invoke_agent remote"}},{@@ -105,87 +99,93 @@}},{- "key": "agent.parent_call.id",+ "key": "agent.name","value": {- "stringValue": "remote-call"+ "stringValue": "remote"}},{- "key": "agent.parent_run.id",+ "key": "gen_ai.agent.name","value": {- "stringValue": "caller-root"+ "stringValue": "remote"}},{- "key": "agent.run.type",+ "key": "gen_ai.operation.name","value": {- "stringValue": "subagent"+ "stringValue": "invoke_agent"}},{- "key": "agent.trace.content.input",+ "key": "agent.turn.id","value": {- "boolValue": true+ "stringValue": "turn_0"}},{- "key": "agent.trace.content.output",+ "key": "agent.turn.sequence","value": {- "boolValue": true+ "intValue": 0}},{- "key": "agent.turn.id",+ "key": "agent.run.type","value": {- "stringValue": "turn_0"+ "stringValue": "subagent"}},{- "key": "agent.turn.sequence",+ "key": "agent.parent_call.id","value": {- "intValue": 0+ "stringValue": "remote-call"}},{- "key": "gen_ai.agent.name",+ "key": "agent.parent_run.id","value": {- "stringValue": "remote"+ "stringValue": "caller-root"}},{- "key": "gen_ai.operation.name",+ "key": "agent.trace.content.input","value": {- "stringValue": "invoke_agent"+ "boolValue": true}},{- "key": "operation.name",+ "key": "agent.trace.content.output","value": {- "stringValue": "invoke_agent"+ "boolValue": true}},{- "key": "resource.name",+ "key": "agent.channel.audience","value": {- "stringValue": "invoke_agent remote"+ "stringValue": "public"}},{- "key": "agent.run.id",+ "key": "agent.principal.current.type","value": {- "stringValue": "remote-session"+ "stringValue": "service"}},{- "key": "agent.trace.schema.version",+ "key": "agent.principal.current.id","value": {- "intValue": 4+ "stringValue": "current-user"}},{- "key": "gen_ai.conversation.id",+ "key": "agent.principal.initiator.type","value": {- "stringValue": "caller-root"+ "stringValue": "user"+ }+ },+ {+ "key": "agent.principal.initiator.id",+ "value": {+ "stringValue": "initiator-user"}},{@@ -248,51 +248,45 @@"endTimeUnixNano": "1700000000001000000","attributes": [{- "key": "agent.framework.name",- "value": {- "stringValue": "eve"- }- },- {- "key": "agent.framework.version",+ "key": "agent.run.id","value": {- "stringValue": "test"+ "stringValue": "remote-session"}},{- "key": "agent.name",+ "key": "agent.trace.schema.version","value": {- "stringValue": "remote"+ "intValue": 4}},{- "key": "agent.channel.audience",+ "key": "gen_ai.conversation.id","value": {- "stringValue": "public"+ "stringValue": "caller-root"}},{- "key": "agent.principal.current.id",+ "key": "agent.framework.name","value": {- "stringValue": "current-user"+ "stringValue": "eve"}},{- "key": "agent.principal.current.type",+ "key": "agent.framework.version","value": {- "stringValue": "service"+ "stringValue": "test"}},{- "key": "agent.principal.initiator.id",+ "key": "operation.name","value": {- "stringValue": "initiator-user"+ "stringValue": "invoke_agent"}},{- "key": "agent.principal.initiator.type",+ "key": "resource.name","value": {- "stringValue": "user"+ "stringValue": "invoke_agent remote"}},{@@ -308,87 +302,93 @@}},{- "key": "agent.parent_call.id",+ "key": "agent.name","value": {- "stringValue": "remote-call"+ "stringValue": "remote"}},{- "key": "agent.parent_run.id",+ "key": "gen_ai.agent.name","value": {- "stringValue": "caller-root"+ "stringValue": "remote"}},{- "key": "agent.run.type",+ "key": "gen_ai.operation.name","value": {- "stringValue": "subagent"+ "stringValue": "invoke_agent"}},{- "key": "agent.trace.content.input",+ "key": "agent.turn.id","value": {- "boolValue": true+ "stringValue": "turn_1"}},{- "key": "agent.trace.content.output",+ "key": "agent.turn.sequence","value": {- "boolValue": true+ "intValue": 1}},{- "key": "agent.turn.id",+ "key": "agent.run.type","value": {- "stringValue": "turn_1"+ "stringValue": "subagent"}},{- "key": "agent.turn.sequence",+ "key": "agent.parent_call.id","value": {- "intValue": 1+ "stringValue": "remote-call"}},{- "key": "gen_ai.agent.name",+ "key": "agent.parent_run.id","value": {- "stringValue": "remote"+ "stringValue": "caller-root"}},{- "key": "gen_ai.operation.name",+ "key": "agent.trace.content.input","value": {- "stringValue": "invoke_agent"+ "boolValue": true}},{- "key": "operation.name",+ "key": "agent.trace.content.output","value": {- "stringValue": "invoke_agent"+ "boolValue": true}},{- "key": "resource.name",+ "key": "agent.channel.audience","value": {- "stringValue": "invoke_agent remote"+ "stringValue": "public"}},{- "key": "agent.run.id",+ "key": "agent.principal.current.type","value": {- "stringValue": "remote-session"+ "stringValue": "service"}},{- "key": "agent.trace.schema.version",+ "key": "agent.principal.current.id","value": {- "intValue": 4+ "stringValue": "current-user"}},{- "key": "gen_ai.conversation.id",+ "key": "agent.principal.initiator.type","value": {- "stringValue": "caller-root"+ "stringValue": "user"+ }+ },+ {+ "key": "agent.principal.initiator.id",+ "value": {+ "stringValue": "initiator-user"}},{@@ -437,135 +437,135 @@"endTimeUnixNano": "1700000000001000000","attributes": [{- "key": "agent.framework.name",+ "key": "agent.run.id","value": {- "stringValue": "eve"+ "stringValue": "local-session"}},{- "key": "agent.framework.version",+ "key": "agent.trace.schema.version","value": {- "stringValue": "test"+ "intValue": 4}},{- "key": "agent.name",+ "key": "gen_ai.conversation.id","value": {- "stringValue": "local"+ "stringValue": "caller-root"}},{- "key": "agent.channel.audience",+ "key": "agent.framework.name","value": {- "stringValue": "public"+ "stringValue": "eve"}},{- "key": "agent.principal.current.id",+ "key": "agent.framework.version","value": {- "stringValue": "current-user"+ "stringValue": "test"}},{- "key": "agent.principal.current.type",+ "key": "operation.name","value": {- "stringValue": "service"+ "stringValue": "invoke_agent"}},{- "key": "agent.principal.initiator.id",+ "key": "resource.name","value": {- "stringValue": "initiator-user"+ "stringValue": "invoke_agent local"}},{- "key": "agent.principal.initiator.type",+ "key": "agent.name","value": {- "stringValue": "user"+ "stringValue": "local"}},{- "key": "agent.parent_call.id",+ "key": "gen_ai.agent.name","value": {- "stringValue": "local-call"+ "stringValue": "local"}},{- "key": "agent.parent_run.id",+ "key": "gen_ai.operation.name","value": {- "stringValue": "remote-session"+ "stringValue": "invoke_agent"}},{- "key": "agent.run.type",+ "key": "agent.turn.id","value": {- "stringValue": "subagent"+ "stringValue": "turn_0"}},{- "key": "agent.trace.content.input",+ "key": "agent.turn.sequence","value": {- "boolValue": true+ "intValue": 0}},{- "key": "agent.trace.content.output",+ "key": "agent.run.type","value": {- "boolValue": true+ "stringValue": "subagent"}},{- "key": "agent.turn.id",+ "key": "agent.parent_call.id","value": {- "stringValue": "turn_0"+ "stringValue": "local-call"}},{- "key": "agent.turn.sequence",+ "key": "agent.parent_run.id","value": {- "intValue": 0+ "stringValue": "remote-session"}},{- "key": "gen_ai.agent.name",+ "key": "agent.trace.content.input","value": {- "stringValue": "local"+ "boolValue": true}},{- "key": "gen_ai.operation.name",+ "key": "agent.trace.content.output","value": {- "stringValue": "invoke_agent"+ "boolValue": true}},{- "key": "operation.name",+ "key": "agent.channel.audience","value": {- "stringValue": "invoke_agent"+ "stringValue": "public"}},{- "key": "resource.name",+ "key": "agent.principal.current.type","value": {- "stringValue": "invoke_agent local"+ "stringValue": "service"}},{- "key": "agent.run.id",+ "key": "agent.principal.current.id","value": {- "stringValue": "local-session"+ "stringValue": "current-user"}},{- "key": "agent.trace.schema.version",+ "key": "agent.principal.initiator.type","value": {- "intValue": 4+ "stringValue": "user"}},{- "key": "gen_ai.conversation.id",+ "key": "agent.principal.initiator.id","value": {- "stringValue": "caller-root"+ "stringValue": "initiator-user"}},{diff --git a/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.test.ts b/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.test.tsindex 6b766492e..de037728c 100644--- a/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.test.ts+++ b/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.test.ts@@ -1,6 +1,6 @@import { afterEach, describe, expect, it, vi } from "vitest";-import { vercelRuntimeSpanExporter } from "#tracing/eve/vercel-runtime-span-exporter.js";+import { vercelRuntimeSpanExporter } from "./vercel-runtime-span-exporter.js";const { serializeRequest } = vi.hoisted(() => ({serializeRequest: vi.fn(() => new TextEncoder().encode('{"resourceSpans":[]}')),@@ -17,7 +17,7 @@ afterEach(() => {serializeRequest.mockClear();});-describe("vercelRuntimeSpanExporter", () => {+describe("Vercel runtime span export", () => {it("reports through the current Vercel request context", () => {const reportSpans = vi.fn();(globalThis as Record<symbol, unknown>)[REQUEST_CONTEXT] = {diff --git a/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.ts b/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.tsindex 49d1519ba..c9f27707c 100644--- a/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.ts+++ b/packages/eve/src/tracing/eve/vercel-runtime-span-exporter.ts@@ -1,6 +1,6 @@import { JsonTraceSerializer } from "#compiled/@opentelemetry/otlp-transformer/index.js";import { TraceFlags } from "#compiled/@opentelemetry/api/index.js";-import type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js";+import type { SpanExporter, SpanProcessor } from "./otel-configuration.js";import { createLogger, formatError } from "#internal/logging.js";@@ -29,7 +29,9 @@ export function vercelRuntimeSpanExporter(): SpanExporter {}try {- const serialized = JsonTraceSerializer.serializeRequest([...spans]);+ const serialized = JsonTraceSerializer.serializeRequest([...spans] as Parameters<+ typeof JsonTraceSerializer.serializeRequest+ >[0]);if (serialized === undefined) throw new Error("Failed to serialize spans.");telemetry.reportSpans(JSON.parse(new TextDecoder().decode(serialized)) as unknown);resultCallback({ code: 0 });diff --git a/packages/eve/src/tracing/local-instrumentation-runtime-conflict.scenario.test.ts b/packages/eve/src/tracing/local-instrumentation-runtime-conflict.scenario.test.tsdeleted file mode 100644index 73cf2b6b0..000000000--- a/packages/eve/src/tracing/local-instrumentation-runtime-conflict.scenario.test.ts+++ /dev/null@@ -1,30 +0,0 @@-import { mkdtemp, rm } from "node:fs/promises";-import { tmpdir } from "node:os";-import { join } from "node:path";--import { trace } from "@opentelemetry/api";-import { BasicTracerProvider } from "@opentelemetry/sdk-trace-base";-import { afterEach, describe, expect, it } from "vitest";--import { installLocalInstrumentationRuntime } from "#tracing/local/instrumentation-runtime.js";--let appRoot: string | undefined;--afterEach(async () => {- if (appRoot !== undefined) await rm(appRoot, { force: true, recursive: true });-});--describe("local instrumentation runtime ownership", () => {- it("fails loudly when another global tracer provider already exists", async () => {- appRoot = await mkdtemp(join(tmpdir(), "eve-local-traces-conflict-"));- expect(trace.setGlobalTracerProvider(new BasicTracerProvider())).toBe(true);-- expect(() =>- installLocalInstrumentationRuntime({- appRoot: appRoot!,- frameworkVersion: "test",- serviceName: "test-agent",- }),- ).toThrow(/another runtime already owns the global tracer provider/u);- });-});diff --git a/packages/eve/src/tracing/local/agent-trace-retention.test.ts b/packages/eve/src/tracing/local/agent-trace-retention.test.tsindex 7d7dd9f5b..213777e0c 100644--- a/packages/eve/src/tracing/local/agent-trace-retention.test.ts+++ b/packages/eve/src/tracing/local/agent-trace-retention.test.ts@@ -4,8 +4,8 @@ import { serializeContext } from "#context/serialize.js";import {ContextAgentTraceStateStore,pruneAgentTraceState,-} from "#tracing/agent-trace-context-store.js";-import { AGENT_TRACE_CONTEXT_KEY } from "#tracing/agent-trace-context-codec.js";+} from "#tracing/eve/agent-trace-context-store.js";+import { AGENT_TRACE_CONTEXT_KEY } from "#tracing/eve/agent-trace-context-store.js";import { AgentTraceSpanProcessor } from "#tracing/local/agent-trace-span-processor.js";const anchor = {@@ -13,12 +13,9 @@ const anchor = {callId: "call",kind: "tool-call" as const,name: "workflow",- parent: { spanId: "1".repeat(16), traceFlags: 1, traceId: "2".repeat(32) },+ context: { spanId: "3".repeat(16), traceFlags: 1, traceId: "2".repeat(32) },rootSessionId: "session",- traceSessionId: "session",sessionId: "session",- spanId: "3".repeat(16),- startTimeMs: 1,stepIndex: 0,turnId: "turn",};@@ -28,7 +25,7 @@ describe("trace retention by live work", () => {const warn = vi.spyOn(console, "warn").mockImplementation(() => {});const context = new ContextContainer();contextStorage.run(context, () =>- new ContextAgentTraceStateStore().setActionAnchor("key", anchor),+ new ContextAgentTraceStateStore().set("anchor", "key", anchor),);const before = serializeContext(context);expect(() =>@@ -49,7 +46,7 @@ describe("trace retention by live work", () => {contextStorage.run(context, () => {const store = new ContextAgentTraceStateStore();for (let index = 0; index < 1000; index++) {- store.setActionAnchor(`key-${index}`, {+ store.set("anchor", `key-${index}`, {...anchor,callId: `call-${index}`,turnId: `turn-${index}`,@@ -57,7 +54,7 @@ describe("trace retention by live work", () => {pruneAgentTraceState(context, "session", undefined);}});- expect(serializeContext(context)[AGENT_TRACE_CONTEXT_KEY]).toMatchObject({ actionAnchors: {} });+ expect(serializeContext(context)[AGENT_TRACE_CONTEXT_KEY]).toMatchObject({ entries: {} });});it("unpins completed activation traces only after the writer flushes", async () => {diff --git a/packages/eve/src/tracing/local/agent-trace-span-processor.ts b/packages/eve/src/tracing/local/agent-trace-span-processor.tsindex 4642f93c7..5286ed521 100644--- a/packages/eve/src/tracing/local/agent-trace-span-processor.ts+++ b/packages/eve/src/tracing/local/agent-trace-span-processor.ts@@ -1,5 +1,5 @@import type { SpanProcessor } from "#compiled/@vercel/otel/index.js";-import { isAgentActivationSpan, isDirectToolCallSpan } from "#tracing/agent-span-contract.js";+import { isAgentActivationSpan, isDirectToolCallSpan } from "#tracing/local/inspection.js";const REMEMBERED_TRACE_LIMIT = 2048;diff --git a/packages/eve/src/tracing/local/inspection.ts b/packages/eve/src/tracing/local/inspection.tsnew file mode 100644index 000000000..24c033a5e--- /dev/null+++ b/packages/eve/src/tracing/local/inspection.ts@@ -0,0 +1,33 @@+interface SpanRecord {+ readonly name: string;+ readonly attributes: Readonly<Record<string, unknown>>;+}++/** Marks the `execute_tool` span of a tool call made outside any conversation. */+export const DIRECT_TOOL_CALL_ATTRIBUTE = "eve.tool.invocation";+export const DIRECT_TOOL_CALL_VALUE = "direct";++/**+ * A direct tool call has no turn, so its own span is what starts and ends+ * its claim on the trace, the way an activation span does for a turn.+ */+export function isDirectToolCallSpan(span: Pick<SpanRecord, "attributes">): boolean {+ return span.attributes[DIRECT_TOOL_CALL_ATTRIBUTE] === DIRECT_TOOL_CALL_VALUE;+}++export function isAgentActivationSpan(span: SpanRecord): boolean {+ return (+ span.name === "agent.turn" ||+ (span.attributes["gen_ai.operation.name"] === "invoke_agent" &&+ span.attributes["agent.invocation.role"] !== "caller" &&+ typeof span.attributes["agent.turn.id"] === "string")+ );+}++export function agentTurnIdentity(span: SpanRecord): string | undefined {+ const sessionId = span.attributes["agent.run.id"];+ const turnId = span.attributes["agent.turn.id"];+ return typeof turnId === "string"+ ? `${typeof sessionId === "string" ? sessionId : ""}\0${turnId}`+ : undefined;+}diff --git a/packages/eve/src/tracing/local/trace-reader.ts b/packages/eve/src/tracing/local/trace-reader.tsindex 16dbc8e99..f8a46d475 100644--- a/packages/eve/src/tracing/local/trace-reader.ts+++ b/packages/eve/src/tracing/local/trace-reader.ts@@ -15,7 +15,7 @@ import {resolveLocalTraceSchemaDirectory,resolveLocalTraceSegmentsDirectory,} from "#tracing/local/trace-span-processor.js";-import { isAgentActivationSpan } from "#tracing/agent-span-contract.js";+import { isAgentActivationSpan } from "#tracing/local/inspection.js";const TRACE_ID_PATTERN = /^[0-9a-f]{32}$/u;const SPAN_FILE_PATTERN = /^[0-9a-f]{16}\.otlp\.json$/u;diff --git a/packages/eve/src/tracing/local/trace-summary.ts b/packages/eve/src/tracing/local/trace-summary.tsindex f1beb1dc8..a6ff7fba4 100644--- a/packages/eve/src/tracing/local/trace-summary.ts+++ b/packages/eve/src/tracing/local/trace-summary.ts@@ -1,5 +1,5 @@import type { LocalTraceSpan } from "#tracing/local/trace-reader.js";-import { AGENT_USAGE_ATTRIBUTES } from "#tracing/agent-span-contract.js";+import { USAGE_FIELDS as AGENT_USAGE_ATTRIBUTES } from "#compiled/@eve/tracing/otel.js";import {localTraceOperations,traceModelName,diff --git a/packages/eve/src/tracing/local/traces.test.ts b/packages/eve/src/tracing/local/traces.test.tsindex 9dae51387..cc267f6e8 100644--- a/packages/eve/src/tracing/local/traces.test.ts+++ b/packages/eve/src/tracing/local/traces.test.ts@@ -7,7 +7,7 @@ import {createLocalTracesProcessor,resolveLocalTracesExportPolicy,} from "#tracing/local/traces.js";-import { resolveTracePolicy } from "#tracing/sampled-trace.js";+import { resolveTracePolicy } from "#shared/trace-policy.js";import { localTraces } from "#public/instrumentation/otel.js";vi.mock("#tracing/local/trace-span-processor.js", () => ({@@ -60,14 +60,6 @@ describe("createLocalTracesProcessor", () => {await expect(spool.releaseConversation("session-one")).resolves.toBe(false);});- it("is a span processor, so it composes wherever one goes", () => {- const spool = createLocalTracesProcessor({ appRoot: "/tmp/eve-local-traces-test" });- expect(typeof spool.onStart).toBe("function");- expect(typeof spool.onEnd).toBe("function");- expect(typeof spool.forceFlush).toBe("function");- expect(typeof spool.shutdown).toBe("function");- });-it("is inert outside a development worker", async () => {vi.stubEnv("EVE_DEV_WORKER_APP_ROOT", undefined);const [processor] = localTraces().spanProcessors;diff --git a/packages/eve/src/tracing/sampled-trace.ts b/packages/eve/src/tracing/sampled-trace.tsdeleted file mode 100644index 86a634f6d..000000000--- a/packages/eve/src/tracing/sampled-trace.ts+++ /dev/null@@ -1,9 +0,0 @@-export { resolveTracePolicy, resolveTracePolicyDecision } from "#shared/trace-policy.js";--// W3C sampled flag; written as a literal so this module stays out of the-// vendored OTel chunk, which the workflow driver bundle cannot include.-const TRACE_FLAGS_SAMPLED = 0x01;--export function isSampledTrace(context: { readonly traceFlags: number }): boolean {- return (context.traceFlags & TRACE_FLAGS_SAMPLED) === TRACE_FLAGS_SAMPLED;-}diff --git a/packages/eve/src/tracing/suppress-tracing.ts b/packages/eve/src/tracing/suppress-tracing.tsdeleted file mode 100644index b21a13abb..000000000--- a/packages/eve/src/tracing/suppress-tracing.ts+++ /dev/null@@ -1,8 +0,0 @@-import { createContextKey, type Context } from "#compiled/@opentelemetry/api/index.js";--// Standard OpenTelemetry SDK suppression key, kept behind an eve-owned wrapper.-const SUPPRESS_TRACING_KEY = createContextKey("OpenTelemetry SDK Context Key SUPPRESS_TRACING");--export function suppressTracing(context: Context): Context {- return context.setValue(SUPPRESS_TRACING_KEY, true);-}diff --git a/packages/eve/src/tracing/telemetry-budget.ts b/packages/eve/src/tracing/telemetry-budget.tsdeleted file mode 100644index 2d858c659..000000000--- a/packages/eve/src/tracing/telemetry-budget.ts+++ /dev/null@@ -1,28 +0,0 @@-const TELEMETRY_PRINCIPAL_ID_BYTES = 1024;--export function boundedPrincipalId(value: unknown): string | undefined {- return typeof value === "string" &&- value.length > 0 &&- value.length <= TELEMETRY_PRINCIPAL_ID_BYTES &&- telemetryByteLength(value) <= TELEMETRY_PRINCIPAL_ID_BYTES- ? value- : undefined;-}--const encoder = new TextEncoder();-const decoder = new TextDecoder();-const MARKER = "... [truncated]";--export function telemetryByteLength(text: string): number {- return encoder.encode(text).length;-}--export function truncateTelemetryText(text: string, maxBytes: number): string {- const prefix = text.slice(0, maxBytes + 1);- const bytes = encoder.encode(prefix);- if (prefix.length === text.length && bytes.length <= maxBytes) return text;- const marker = MARKER.slice(0, maxBytes);- let end = Math.max(0, maxBytes - marker.length);- while (end > 0 && (bytes[end]! & 0xc0) === 0x80) end -= 1;- return decoder.decode(bytes.subarray(0, end)) + marker;-}diff --git a/packages/eve/src/tracing/test-data/remote-local-agent-trace.otlp.json b/packages/eve/src/tracing/test-data/remote-local-agent-trace.otlp.jsonnew file mode 100644index 000000000..d8c9a0766--- /dev/null+++ b/packages/eve/src/tracing/test-data/remote-local-agent-trace.otlp.json@@ -0,0 +1,600 @@+{+ "resourceSpans": [+ {+ "resource": {+ "attributes": [+ {+ "key": "service.name",+ "value": {+ "stringValue": "eve-trace-contract"+ }+ },+ {+ "key": "telemetry.sdk.language",+ "value": {+ "stringValue": "nodejs"+ }+ },+ {+ "key": "telemetry.sdk.name",+ "value": {+ "stringValue": "opentelemetry"+ }+ },+ {+ "key": "telemetry.sdk.version",+ "value": {+ "stringValue": "2.6.1"+ }+ }+ ],+ "droppedAttributesCount": 0+ },+ "scopeSpans": [+ {+ "scope": {+ "name": "eve.agent"+ },+ "spans": [+ {+ "traceId": "b31314b8db6855d6662fc41a0ea318e2",+ "spanId": "b31314b8db6855d6",+ "name": "invoke_agent remote",+ "kind": 1,+ "startTimeUnixNano": "1700000000000000000",+ "endTimeUnixNano": "1700000000001000000",+ "attributes": [+ {+ "key": "agent.framework.name",+ "value": {+ "stringValue": "eve"+ }+ },+ {+ "key": "agent.framework.version",+ "value": {+ "stringValue": "test"+ }+ },+ {+ "key": "agent.name",+ "value": {+ "stringValue": "remote"+ }+ },+ {+ "key": "agent.channel.audience",+ "value": {+ "stringValue": "public"+ }+ },+ {+ "key": "agent.principal.current.id",+ "value": {+ "stringValue": "current-user"+ }+ },+ {+ "key": "agent.principal.current.type",+ "value": {+ "stringValue": "service"+ }+ },+ {+ "key": "agent.principal.initiator.id",+ "value": {+ "stringValue": "initiator-user"+ }+ },+ {+ "key": "agent.principal.initiator.type",+ "value": {+ "stringValue": "user"+ }+ },+ {+ "key": "agent.channel.kind",+ "value": {+ "stringValue": "http"+ }+ },+ {+ "key": "agent.session.origin",+ "value": {+ "stringValue": "channel"+ }+ },+ {+ "key": "agent.parent_call.id",+ "value": {+ "stringValue": "remote-call"+ }+ },+ {+ "key": "agent.parent_run.id",+ "value": {+ "stringValue": "caller-root"+ }+ },+ {+ "key": "agent.run.type",+ "value": {+ "stringValue": "subagent"+ }+ },+ {+ "key": "agent.trace.content.input",+ "value": {+ "boolValue": true+ }+ },+ {+ "key": "agent.trace.content.output",+ "value": {+ "boolValue": true+ }+ },+ {+ "key": "agent.turn.id",+ "value": {+ "stringValue": "turn_0"+ }+ },+ {+ "key": "agent.turn.sequence",+ "value": {+ "intValue": 0+ }+ },+ {+ "key": "gen_ai.agent.name",+ "value": {+ "stringValue": "remote"+ }+ },+ {+ "key": "gen_ai.operation.name",+ "value": {+ "stringValue": "invoke_agent"+ }+ },+ {+ "key": "operation.name",+ "value": {+ "stringValue": "invoke_agent"+ }+ },+ {+ "key": "resource.name",+ "value": {+ "stringValue": "invoke_agent remote"+ }+ },+ {+ "key": "agent.run.id",+ "value": {+ "stringValue": "remote-session"+ }+ },+ {+ "key": "agent.trace.schema.version",+ "value": {+ "intValue": 4+ }+ },+ {+ "key": "gen_ai.conversation.id",+ "value": {+ "stringValue": "caller-root"+ }+ },+ {+ "key": "vercel.session_id",+ "value": {+ "stringValue": "remote-session"+ }+ },+ {+ "key": "agent.turn.outcome",+ "value": {+ "stringValue": "completed"+ }+ }+ ],+ "droppedAttributesCount": 0,+ "events": [+ {+ "attributes": [],+ "name": "turn.started",+ "timeUnixNano": "1700000000000000000",+ "droppedAttributesCount": 0+ },+ {+ "attributes": [],+ "name": "turn.completed",+ "timeUnixNano": "1700000000000000000",+ "droppedAttributesCount": 0+ }+ ],+ "droppedEventsCount": 0,+ "status": {+ "code": 0+ },+ "links": [+ {+ "attributes": [+ {+ "key": "eve.link.type",+ "value": {+ "stringValue": "agent.dispatch"+ }+ }+ ],+ "spanId": "aaaaaaaaaaaaaaaa",+ "traceId": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+ "droppedAttributesCount": 0,+ "flags": 769+ }+ ],+ "droppedLinksCount": 0,+ "flags": 257+ },+ {+ "traceId": "86d795248ef3bc2325f68a0cc80f27d7",+ "spanId": "86d795248ef3bc23",+ "name": "invoke_agent remote",+ "kind": 1,+ "startTimeUnixNano": "1700000000000000000",+ "endTimeUnixNano": "1700000000001000000",+ "attributes": [+ {+ "key": "agent.framework.name",+ "value": {+ "stringValue": "eve"+ }+ },+ {+ "key": "agent.framework.version",+ "value": {+ "stringValue": "test"+ }+ },+ {+ "key": "agent.name",+ "value": {+ "stringValue": "remote"+ }+ },+ {+ "key": "agent.channel.audience",+ "value": {+ "stringValue": "public"+ }+ },+ {+ "key": "agent.principal.current.id",+ "value": {+ "stringValue": "current-user"+ }+ },+ {+ "key": "agent.principal.current.type",+ "value": {+ "stringValue": "service"+ }+ },+ {+ "key": "agent.principal.initiator.id",+ "value": {+ "stringValue": "initiator-user"+ }+ },+ {+ "key": "agent.principal.initiator.type",+ "value": {+ "stringValue": "user"+ }+ },+ {+ "key": "agent.channel.kind",+ "value": {+ "stringValue": "http"+ }+ },+ {+ "key": "agent.session.origin",+ "value": {+ "stringValue": "channel"+ }+ },+ {+ "key": "agent.parent_call.id",+ "value": {+ "stringValue": "remote-call"+ }+ },+ {+ "key": "agent.parent_run.id",+ "value": {+ "stringValue": "caller-root"+ }+ },+ {+ "key": "agent.run.type",+ "value": {+ "stringValue": "subagent"+ }+ },+ {+ "key": "agent.trace.content.input",+ "value": {+ "boolValue": true+ }+ },+ {+ "key": "agent.trace.content.output",+ "value": {+ "boolValue": true+ }+ },+ {+ "key": "agent.turn.id",+ "value": {+ "stringValue": "turn_1"+ }+ },+ {+ "key": "agent.turn.sequence",+ "value": {+ "intValue": 1+ }+ },+ {+ "key": "gen_ai.agent.name",+ "value": {+ "stringValue": "remote"+ }+ },+ {+ "key": "gen_ai.operation.name",+ "value": {+ "stringValue": "invoke_agent"+ }+ },+ {+ "key": "operation.name",+ "value": {+ "stringValue": "invoke_agent"+ }+ },+ {+ "key": "resource.name",+ "value": {+ "stringValue": "invoke_agent remote"+ }+ },+ {+ "key": "agent.run.id",+ "value": {+ "stringValue": "remote-session"+ }+ },+ {+ "key": "agent.trace.schema.version",+ "value": {+ "intValue": 4+ }+ },+ {+ "key": "gen_ai.conversation.id",+ "value": {+ "stringValue": "caller-root"+ }+ },+ {+ "key": "vercel.session_id",+ "value": {+ "stringValue": "remote-session"+ }+ },+ {+ "key": "agent.turn.outcome",+ "value": {+ "stringValue": "completed"+ }+ }+ ],+ "droppedAttributesCount": 0,+ "events": [+ {+ "attributes": [],+ "name": "turn.started",+ "timeUnixNano": "1700000000000000000",+ "droppedAttributesCount": 0+ },+ {+ "attributes": [],+ "name": "turn.completed",+ "timeUnixNano": "1700000000000000000",+ "droppedAttributesCount": 0+ }+ ],+ "droppedEventsCount": 0,+ "status": {+ "code": 0+ },+ "links": [],+ "droppedLinksCount": 0,+ "flags": 257+ },+ {+ "traceId": "b31314b8db6855d6662fc41a0ea318e2",+ "spanId": "d8f3a7d23b288e65",+ "parentSpanId": "b31314b8db6855d6",+ "name": "invoke_agent local",+ "kind": 1,+ "startTimeUnixNano": "1700000000000000000",+ "endTimeUnixNano": "1700000000001000000",+ "attributes": [+ {+ "key": "agent.framework.name",+ "value": {+ "stringValue": "eve"+ }+ },+ {+ "key": "agent.framework.version",+ "value": {+ "stringValue": "test"+ }+ },+ {+ "key": "agent.name",+ "value": {+ "stringValue": "local"+ }+ },+ {+ "key": "agent.channel.audience",+ "value": {+ "stringValue": "public"+ }+ },+ {+ "key": "agent.principal.current.id",+ "value": {+ "stringValue": "current-user"+ }+ },+ {+ "key": "agent.principal.current.type",+ "value": {+ "stringValue": "service"+ }+ },+ {+ "key": "agent.principal.initiator.id",+ "value": {+ "stringValue": "initiator-user"+ }+ },+ {+ "key": "agent.principal.initiator.type",+ "value": {+ "stringValue": "user"+ }+ },+ {+ "key": "agent.parent_call.id",+ "value": {+ "stringValue": "local-call"+ }+ },+ {+ "key": "agent.parent_run.id",+ "value": {+ "stringValue": "remote-session"+ }+ },+ {+ "key": "agent.run.type",+ "value": {+ "stringValue": "subagent"+ }+ },+ {+ "key": "agent.trace.content.input",+ "value": {+ "boolValue": true+ }+ },+ {+ "key": "agent.trace.content.output",+ "value": {+ "boolValue": true+ }+ },+ {+ "key": "agent.turn.id",+ "value": {+ "stringValue": "turn_0"+ }+ },+ {+ "key": "agent.turn.sequence",+ "value": {+ "intValue": 0+ }+ },+ {+ "key": "gen_ai.agent.name",+ "value": {+ "stringValue": "local"+ }+ },+ {+ "key": "gen_ai.operation.name",+ "value": {+ "stringValue": "invoke_agent"+ }+ },+ {+ "key": "operation.name",+ "value": {+ "stringValue": "invoke_agent"+ }+ },+ {+ "key": "resource.name",+ "value": {+ "stringValue": "invoke_agent local"+ }+ },+ {+ "key": "agent.run.id",+ "value": {+ "stringValue": "local-session"+ }+ },+ {+ "key": "agent.trace.schema.version",+ "value": {+ "intValue": 4+ }+ },+ {+ "key": "gen_ai.conversation.id",+ "value": {+ "stringValue": "caller-root"+ }+ },+ {+ "key": "vercel.session_id",+ "value": {+ "stringValue": "remote-session"+ }+ }+ ],+ "droppedAttributesCount": 0,+ "events": [+ {+ "attributes": [],+ "name": "turn.started",+ "timeUnixNano": "1700000000000000000",+ "droppedAttributesCount": 0+ }+ ],+ "droppedEventsCount": 0,+ "status": {+ "code": 0+ },+ "links": [],+ "droppedLinksCount": 0,+ "flags": 257+ }+ ]+ }+ ]+ }+ ]+}