Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(eve): add standalone internal agent tracing library (#4160)

Chad Hietala422b628a106f
diff --git a/.changeset/quiet-agent-traces.md b/.changeset/quiet-agent-traces.md
new file mode 100644
index 000000000..ef57d6cc1
--- /dev/null
+++ b/.changeset/quiet-agent-traces.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Extract reusable agent tracing with wrapped execution, model stream completion, remote delegation, and durable turns that save and resume themselves through a checkpointer. Preserve trace schema version 4 while sharing one span lifecycle with eve.
diff --git a/packages/eve/package.json b/packages/eve/package.json
index 487b85034..24a64cef7 100644
--- a/packages/eve/package.json
+++ b/packages/eve/package.json
@@ -595,6 +595,7 @@
"@chat-adapter/twilio": "4.41.0",
"@clack/core": "1.3.1",
"@eve/catalog": "workspace:*",
+ "@eve/tracing": "workspace:*",
"@linqapp/chat-sdk-adapter": "^0.5.1",
"@modelcontextprotocol/server": "2.0.0",
"@nuxt/kit": "^4.0.0",
diff --git a/packages/eve/scripts/vendor-compiled/@eve/tracing.mjs b/packages/eve/scripts/vendor-compiled/@eve/tracing.mjs
new file mode 100644
index 000000000..b21215088
--- /dev/null
+++ b/packages/eve/scripts/vendor-compiled/@eve/tracing.mjs
@@ -0,0 +1,58 @@
+import { execFile } from "node:child_process";
+import { createRequire } from "node:module";
+import { dirname, join, relative } from "node:path";
+import { promisify } from "node:util";
+
+import { collectFilesRecursively, createDeclarationCopier } from "../_shared.mjs";
+
+const execFileAsync = promisify(execFile);
+const require = createRequire(import.meta.url);
+const tscPath = join(dirname(require.resolve("typescript/package.json")), "bin", "tsc");
+const packageRoot = dirname(require.resolve("@eve/tracing/package.json"));
+const sources = (await collectFilesRecursively(join(packageRoot, "src"), [".ts"]))
+ .filter((file) => !file.endsWith(".test.ts"))
+ .map((file) => relative(packageRoot, file).replaceAll("\\", "/"))
+ .sort();
+
+const copyDeclarations = createDeclarationCopier({
+ declarationRoot: "dist/src",
+ files: async ({ distDir }) =>
+ (await collectFilesRecursively(distDir, [".d.ts"]))
+ .map((file) => relative(distDir, file).replaceAll("\\", "/"))
+ .sort()
+ .map((file) => ({ source: file, output: file })),
+ rewrites: {
+ "@opentelemetry/api": { kind: "vendored", compiledPath: "@opentelemetry/api" },
+ ai: { kind: "external" },
+ },
+});
+
+/**
+ * Bundle the private workspace tracing library so the published eve package
+ * stays self-contained. It shares the `workflow` chunk group with the vendored
+ * `@opentelemetry/api`, so eve and the library use one OpenTelemetry copy.
+ */
+export default {
+ packageName: "@eve/tracing",
+ compiledPath: "@eve/tracing",
+ chunkGroup: "workflow",
+ entries: ["index", "delegation", "otel", "ai-sdk"].map((name) => ({
+ entry: `src/${name}.ts`,
+ outputPath: name,
+ })),
+ fingerprintFiles: [
+ "../../tsconfig.json",
+ "package.json",
+ "tsconfig.build.json",
+ "tsconfig.json",
+ ...sources,
+ ],
+ copyDeclarations: async (context) => {
+ await execFileAsync(
+ process.execPath,
+ [tscPath, "-p", "tsconfig.build.json", "--emitDeclarationOnly", "--declarationMap", "false"],
+ { cwd: context.packageInfo.packageRoot },
+ );
+ await copyDeclarations(context);
+ },
+};
diff --git a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts
index d31678b68..8a7403dfe 100644
--- a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts
+++ b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts
@@ -31,6 +31,10 @@ export interface Link {
attributes?: Attributes;
}
+export interface TracerProvider {
+ getTracer(name: string, version?: string): Tracer;
+}
+
export interface Tracer {
startSpan(
name: string,
diff --git a/packages/eve/scripts/vendor-compiled/index.mjs b/packages/eve/scripts/vendor-compiled/index.mjs
index a3c376cbb..f2bc10dbd 100644
--- a/packages/eve/scripts/vendor-compiled/index.mjs
+++ b/packages/eve/scripts/vendor-compiled/index.mjs
@@ -18,6 +18,7 @@ import chatAdapterSlack from "./@chat-adapter/slack.mjs";
import chatAdapterStateMemory from "./@chat-adapter/state-memory.mjs";
import chatAdapterTwilio from "./@chat-adapter/twilio.mjs";
import eveCatalog from "./@eve/catalog.mjs";
+import eveTracing from "./@eve/tracing.mjs";
import photonChatAdapterIMessage from "./@photon-ai/chat-adapter-imessage.mjs";
import linqChatSdkAdapter from "./@linqapp/chat-sdk-adapter.mjs";
@@ -75,6 +76,7 @@ export const MODULES = [
eventsourceParserStream,
envRunner,
eveCatalog,
+ eveTracing,
google,
jose,
jsYaml,
diff --git a/packages/tracing/README.md b/packages/tracing/README.md
new file mode 100644
index 000000000..a2bc63931
--- /dev/null
+++ b/packages/tracing/README.md
@@ -0,0 +1,237 @@
+# Agent tracing
+
+`@eve/tracing` records agent execution with OpenTelemetry. It owns
+span topology, capture, completion, and durable checkpoints. The package is
+private and not published to npm. Its only runtime package dependency is
+`@opentelemetry/api`.
+
+eve vendors it: `pnpm --filter eve build:compiled` bundles the four
+entrypoints into `#compiled/@eve/tracing/` in the same chunk group as eve's
+vendored `@opentelemetry/api`, so both share one copy. eve imports only those
+entrypoints, never this workspace package. After changing the library, rerun
+`build:compiled` before running eve's tests.
+
+Run the library's own tests with `pnpm --filter @eve/tracing test:unit` and
+`test:integration`.
+
+## Setup
+
+Pass a telemetry object that writes spans and tracks the active trace context.
+`otelTelemetry()` adapts an OpenTelemetry tracer provider; it does not register
+global telemetry or install another model SDK tracing pipeline. The host sets
+up OpenTelemetry, for example with `registerOTel()` from `@vercel/otel`, which
+also installs the async context manager the library needs, or passes its own
+provider.
+
+```ts
+import { createAgentTracing, otelTelemetry } from "@eve/tracing";
+
+const tracing = createAgentTracing({ telemetry: otelTelemetry({ provider }) });
+```
+
+On Vercel, `otelTelemetry()` adds `vercel.session_id` from `agent.run.id` so
+Agent Runs can find a run's spans. A `mapping` you pass replaces that default.
+
+Omit `telemetry` to use the global provider. Your provider owns exporters,
+sampling, and context propagation. Install its context manager before
+concurrent work. `forceFlush()` and `shutdown()` call the telemetry's lifecycle
+methods, which default to the provider's. Other backends implement
+`AgentTelemetry` directly.
+
+## Wrapped execution
+
+Wrapped methods preserve the callback's value or error and complete their span.
+
+```ts
+await tracing.turn({ agentName: "support", identity, sequence: 0 }, (turn) =>
+ turn.attempt({ stepIndex: 0, attempt: 0 }, (attempt) =>
+ attempt.tool({ callId: "lookup", name: "lookup" }, lookup),
+ ),
+);
+```
+
+`identity` contains `conversationId`, `runId`, and `turnId`. `agentName` names the
+invocation span, so one instance can trace several agents. A turn can supply
+framework metadata, attributes, and a capture decision. Capture defaults to
+metadata only. Child operations cannot increase capture.
+
+When inputs or outputs are captured, a turn's span records its first model call's input as `gen_ai.input.messages` and its latest model output as `gen_ai.output.messages`.
+
+Attempts provide `tool`, `callSubAgent`, and `callRemoteAgent`; each tool call is
+one `execute_tool` span. Tools provide `approval` and nested `tool` calls. Turns,
+attempts, and tools provide `memory`.
+Use `describe(value)` to map a callback result to an outcome or captured output.
+
+## Handles
+
+Omit the callback when execution starts and ends in different hooks.
+
+```ts
+const turn = await tracing.turn({ identity, sequence: 0 });
+const attempt = await turn.attempt({ stepIndex: 0, attempt: 0 });
+const tool = await attempt.tool({ callId: "lookup", name: "lookup" });
+try {
+ const value = await tool.run(lookup);
+ await tool.complete({ outcome: "completed", output: value });
+} catch (error) {
+ await tool.fail(error);
+}
+await attempt.complete();
+await turn.complete();
+```
+
+Handles expose execution, attributes, completion, and identity.
+`run(execute, ceiling)` limits captured content for work inside the run without
+changing what the operation already recorded. Content any operation receives
+while the run is active, such as an existing tool's arguments, result, or error,
+is held to that ceiling. Re-entering a durable turn with a narrower `capture`
+narrows the turn and its open operations and drops retained content the new
+decision declines. `recordError(error)` records a
+failure without ending the operation, for example when a tool's work fails in a
+process that does not finish its span; a completion reported later stays
+failed. Parent
+completion closes unfinished children; in durable turns, a successful attempt
+or turn leaves open tool calls and approvals for their own completion.
+`complete()` is idempotent. A turn completed without an `outcome` records
+`completed`; pass `{ outcomeUnknown: true }` when the host knows the turn ended
+but never learned how, and the turn ends without an outcome or terminal event.
+
+A host can learn about one call twice, for example when the SDK runs a tool
+before the runtime records its dispatch. In a durable turn, calling
+`attempt.tool()` again with the same `callId` returns the same call. It fills in
+`kind` and `arguments` the first call lacked and keeps the earlier
+`startTimeMs`.
+
+## Model calls and streams
+
+`modelCall(data, execute)` accepts an envelope with `result`, `finishReason`,
+`usage`, and optional response/content metadata. It returns only `result`.
+Without a callback, `modelCall(data)` returns a model handle.
+
+```ts
+const stream = await attempt.modelStream({ provider: "test", modelId: "model" }, () => ({
+ result: response.stream,
+ completion: response.completion,
+}));
+```
+
+The completion promise supplies finish reason, usage, and optional content. It
+must settle on success, failure, or cancellation. The library returns the stream
+unchanged and never consumes it. Parent completion waits for that promise.
+
+## AI SDK
+
+`aiSdkTelemetry` is an AI SDK telemetry integration. It records each
+`generateText`, `streamText`, or agent call as a turn, with its steps, model
+calls, and tool executions, and runs model calls and tools inside their spans.
+
+```ts
+import { aiSdkTelemetry } from "@eve/tracing/ai-sdk";
+
+await generateText({
+ model,
+ prompt,
+ telemetry: {
+ integrations: [
+ aiSdkTelemetry(tracing, { turn: { agentName: "support", identity, sequence: 0 } }),
+ ],
+ },
+});
+```
+
+Pass a function as `turn` to choose the turn per call. `modelUsage` and
+`modelContent` convert AI SDK payloads for hosts that record model calls
+themselves. The entrypoint has an optional peer dependency on `ai`.
+
+## Delegation
+
+Local delegation carries lineage across asynchronous calls. A local callee's first
+turn nests under the delegating tool call, and later turns start their own trace.
+A remote callee starts its own trace with an `agent.dispatch` link to the call.
+Remote calls need an explicit sender transport and authenticated receiver trust
+check.
+
+```ts
+import { createAgentDelegationTransport } from "@eve/tracing/delegation";
+const transport = createAgentDelegationTransport();
+const fetchAgent = transport.transport(approvedAgentFetch);
+await attempt.callRemoteAgent({ callId: "remote", agentName: "research" }, () =>
+ fetchAgent(approvedAgentUrl),
+);
+
+transport.receive(request.headers, verifyAuthenticatedCaller, () =>
+ tracing.turn({ identity, sequence: 0 }, runAgent),
+);
+```
+
+Bind the sender only to approved destinations. It rejects redirects. The receiver
+validates the bounded `x-agent-tracing` header before calling your trust callback.
+Metadata is not authorization: verify authenticated provenance and the expected
+caller, parent run, and call. Rejected metadata does not change application work.
+
+## Durable turns
+
+Pass a checkpointer when a turn can outlive the process that started it, for
+example across workflow steps. Durable spans need stable IDs, so install an
+`AgentSpanIdGenerator` on the provider and give it to the telemetry.
+
+```ts
+const idGenerator = new AgentSpanIdGenerator();
+const provider = new BasicTracerProvider({ idGenerator, spanProcessors });
+const tracing = createAgentTracing({
+ telemetry: otelTelemetry({ provider, idGenerator }),
+ checkpointer: { get, set, delete: remove },
+});
+```
+
+Resume by calling the same operations with the same identifiers. The library
+continues the saved turn instead of starting a new one:
+
+```ts
+const turn = await tracing.turn({ identity, sequence: 0 });
+const attempt = await turn.attempt({ stepIndex: 0, attempt: 0 });
+const tool = await attempt.tool({ callId: "lookup", name: "lookup" });
+const approval = await tool.approval({ requestId: "approval" });
+await approval.complete({ outcome: "approved" });
+```
+
+Turns are keyed by `runId` and `turnId`; attempts by step index
+and attempt; tool calls by `callId`; approvals by `requestId`. Model calls and
+memory finish within one process and are not resumed. `tracing.resume({ identity })`
+returns a saved turn without starting one, for example to finish a tool call
+after its turn has completed. `turn.findTool(callId)`, `turn.findAttempt()`, and
+`tool.findApproval()` return open operations, or `undefined` once they have
+finished:
+
+```ts
+const tool = (await tracing.resume({ identity }))?.findTool("lookup");
+await tool?.complete({ outcome: "completed" });
+```
+
+A turn can adopt a trace and span ID reserved before it started, such as one
+already returned to a caller, with `reference`. The telemetry's sampler still
+decides whether a root turn is recorded. `parent` nests a turn under a caller's
+span instead. `lineage`, `channel`, `links`, and
+`startTimeMs` describe host-owned delegation and delivery; `turn.links()`
+replaces links learned after the turn started.
+
+The library writes JSON after each change and deletes the entry when the turn
+and its open tool calls complete. Back the checkpointer with storage that commits alongside your
+workflow step. An unreadable checkpoint is reported to `onError` and the turn
+starts fresh. Durable turns, tool calls, and approvals are exported when they
+complete, so their spans carry the IDs their children already reference.
+
+## OpenTelemetry plumbing
+
+`@eve/tracing/otel` exports helpers for hosts that run their own spans
+beside agent tracing: the active operation, capture context, content bounds,
+and MCP enrichment. Agent tracing itself needs only the package root.
+
+## Privacy and failures
+
+Declined content does not enter checkpoints. Failure classes survive redaction;
+exception content does not. Serialized content is capped at 32 KiB, each checkpointed
+operation at 64 KiB, and unfinished children at 10,000. `onError(error, context)` is the only
+tracing error channel. Trace output retains schema version 4.
+
+Local tests cover output and recovery. They do not prove live Agent Runs ingestion.
diff --git a/packages/tracing/package.json b/packages/tracing/package.json
new file mode 100644
index 000000000..403cadee9
--- /dev/null
+++ b/packages/tracing/package.json
@@ -0,0 +1,63 @@
+{
+ "name": "@eve/tracing",
+ "version": "0.0.0",
+ "private": true,
+ "description": "Framework-neutral tracing for durable AI agents. Vendored into eve; not published.",
+ "license": "Apache-2.0",
+ "type": "module",
+ "sideEffects": false,
+ "exports": {
+ ".": {
+ "eve-source": "./src/index.ts",
+ "types": "./src/index.ts",
+ "default": "./src/index.ts"
+ },
+ "./delegation": {
+ "eve-source": "./src/delegation.ts",
+ "types": "./src/delegation.ts",
+ "default": "./src/delegation.ts"
+ },
+ "./otel": {
+ "eve-source": "./src/otel.ts",
+ "types": "./src/otel.ts",
+ "default": "./src/otel.ts"
+ },
+ "./ai-sdk": {
+ "eve-source": "./src/ai-sdk.ts",
+ "types": "./src/ai-sdk.ts",
+ "default": "./src/ai-sdk.ts"
+ },
+ "./package.json": "./package.json"
+ },
+ "scripts": {
+ "clean": "rm -rf dist",
+ "build:types": "tsc -p tsconfig.build.json --emitDeclarationOnly",
+ "test:unit": "vitest run --config vitest.unit.config.ts",
+ "test:integration": "vitest run --config vitest.integration.config.ts",
+ "typecheck": "tsc -p tsconfig.json --noEmit"
+ },
+ "devDependencies": {
+ "@ai-sdk/harness": "1.0.148",
+ "@ai-sdk/provider-utils": "catalog:",
+ "@opentelemetry/api": "^1.9.0",
+ "@opentelemetry/context-async-hooks": "catalog:",
+ "@opentelemetry/sdk-trace-base": "catalog:",
+ "@types/node": "catalog:",
+ "ai": "catalog:",
+ "typescript": "catalog:",
+ "vitest": "catalog:",
+ "zod": "catalog:"
+ },
+ "peerDependencies": {
+ "@opentelemetry/api": "^1.9.0",
+ "ai": "catalog:"
+ },
+ "peerDependenciesMeta": {
+ "ai": {
+ "optional": true
+ }
+ },
+ "engines": {
+ "node": ">=24"
+ }
+}
diff --git a/packages/tracing/src/adapters/agent-transport.ts b/packages/tracing/src/adapters/agent-transport.ts
new file mode 100644
index 000000000..256e74287
--- /dev/null
+++ b/packages/tracing/src/adapters/agent-transport.ts
@@ -0,0 +1,113 @@
+import { currentAgentHandoff, withAgentHandoff, type AgentHandoff } from "../core/delegation.js";
+const HANDOFF_BYTES = 8192;
+
+const HEADER = "x-agent-tracing";
+const encoder = new TextEncoder();
+function isHexId(value: unknown, length: number): boolean {
+ return (
+ typeof value === "string" &&
+ value.length === length &&
+ /^[a-f0-9]+$/u.test(value) &&
+ !/^0+$/u.test(value)
+ );
+}
+
+/** Bind only to a client whose destinations the application has approved. */
+function agentTraceFetch(fetcher: typeof fetch, maxBytes: number): typeof fetch {
+ return (request, init) => {
+ const handoff = currentAgentHandoff();
+ if (handoff === undefined) return fetcher(request, init);
+ const encoded = JSON.stringify(handoff);
+ if (encoder.encode(encoded).length > maxBytes) return fetcher(request, init);
+ const headers = new Headers(
+ init?.headers ?? (request instanceof Request ? request.headers : undefined),
+ );
+ headers.set(HEADER, encoded);
+ return fetcher(request, { ...init, headers, redirect: "error" });
+ };
+}
+
+/** The host validates provenance; well-formed trace metadata is not authorization. */
+function receiveAgentTrace<T>(
+ headers: Headers,
+ trusted: (handoff: AgentHandoff) => boolean,
+ execute: (handoff?: AgentHandoff) => T,
+ maxBytes: number,
+): T {
+ const encoded = headers.get(HEADER);
+ if (encoded === null || encoded.length > maxBytes || encoder.encode(encoded).length > maxBytes)
+ return execute();
+ let handoff: AgentHandoff;
+ const decode = (): AgentHandoff | undefined => {
+ try {
+ handoff = JSON.parse(encoded) as AgentHandoff;
+ if (
+ handoff === null ||
+ typeof handoff !== "object" ||
+ ![
+ handoff.conversationId,
+ handoff.parentRunId,
+ handoff.parentCallId,
+ handoff.agentName,
+ ].every((value) => typeof value === "string" && value.length > 0 && value.length <= 1024)
+ )
+ return undefined;
+ if (
+ !isHexId(handoff.caller.traceId, 32) ||
+ !isHexId(handoff.caller.spanId, 16) ||
+ ![0, 1].includes(handoff.caller.traceFlags)
+ )
+ return undefined;
+ if (
+ handoff.caller.tracestate !== undefined &&
+ (typeof handoff.caller.tracestate !== "string" || handoff.caller.tracestate.length > 512)
+ )
+ return undefined;
+ if (
+ ![handoff.capture.emit, handoff.capture.recordInputs, handoff.capture.recordOutputs].every(
+ (value) => typeof value === "boolean",
+ ) ||
+ !trusted(handoff)
+ )
+ return undefined;
+ return handoff;
+ } catch {
+ return undefined;
+ }
+ };
+ const decoded = decode();
+ if (decoded === undefined) return execute();
+ const accepted = {
+ ...decoded,
+ remote: true,
+ capture: {
+ emit: decoded.capture.emit && decoded.caller.traceFlags === 1,
+ recordInputs:
+ decoded.capture.emit && decoded.caller.traceFlags === 1 && decoded.capture.recordInputs,
+ recordOutputs:
+ decoded.capture.emit && decoded.caller.traceFlags === 1 && decoded.capture.recordOutputs,
+ },
+ };
+ return withAgentHandoff(accepted, () => execute(accepted));
+}
+
+/** Host boundary for durable dispatch and authenticated remote transport. */
+export function createAgentDelegationTransport(
+ fetcher: typeof fetch = (request, init) => globalThis.fetch(request, init),
+) {
+ const maxBytes = HANDOFF_BYTES;
+ return {
+ fetch: agentTraceFetch(fetcher, maxBytes),
+ transport: (fetcher: typeof fetch) => agentTraceFetch(fetcher, maxBytes),
+ receive<T>(
+ headers: Headers,
+ trusted: (handoff: AgentHandoff) => boolean,
+ execute: (handoff?: AgentHandoff) => T,
+ ) {
+ return receiveAgentTrace(headers, trusted, execute, maxBytes);
+ },
+ resume<T>(stored: AgentHandoff, execute: (handoff: AgentHandoff) => T): T {
+ return withAgentHandoff(stored, () => execute(stored));
+ },
+ };
+}
diff --git a/packages/tracing/src/adapters/ai-sdk-payload.ts b/packages/tracing/src/adapters/ai-sdk-payload.ts
new file mode 100644
index 000000000..bb044f7c6
--- /dev/null
+++ b/packages/tracing/src/adapters/ai-sdk-payload.ts
@@ -0,0 +1,44 @@
+import type { ContentPart, Usage } from "../core/types.js";
+
+interface Result {
+ usage: {
+ inputTokens?: number;
+ outputTokens?: number;
+ inputTokenDetails?: { cacheReadTokens?: number; cacheWriteTokens?: number };
+ };
+ content: readonly unknown[];
+}
+
+export function modelUsage(usage: Result["usage"]): Usage {
+ return Object.freeze({
+ inputTokens: usage.inputTokens,
+ outputTokens: usage.outputTokens,
+ inputTokenDetails: Object.freeze({
+ cacheReadTokens: usage.inputTokenDetails?.cacheReadTokens,
+ cacheWriteTokens: usage.inputTokenDetails?.cacheWriteTokens,
+ }),
+ });
+}
+
+export function modelContent(content: Result["content"]): readonly ContentPart[] {
+ return content.flatMap((part): ContentPart[] => {
+ if (part === null || typeof part !== "object" || !("type" in part)) return [];
+ const value = part as {
+ type: string;
+ text?: string;
+ toolCallId?: string;
+ toolName?: string;
+ input?: unknown;
+ output?: unknown;
+ error?: unknown;
+ };
+ if ((value.type === "text" || value.type === "reasoning") && typeof value.text === "string")
+ return [{ type: value.type, text: value.text }];
+ if (typeof value.toolCallId !== "string" || typeof value.toolName !== "string") return [];
+ const tool = { callId: value.toolCallId, toolName: value.toolName, input: value.input };
+ if (value.type === "tool-call") return [{ type: value.type, ...tool }];
+ if (value.type === "tool-result") return [{ type: value.type, ...tool, output: value.output }];
+ if (value.type === "tool-error") return [{ type: value.type, ...tool, error: value.error }];
+ return [];
+ });
+}
diff --git a/packages/tracing/src/adapters/ai-sdk-telemetry.ts b/packages/tracing/src/adapters/ai-sdk-telemetry.ts
new file mode 100644
index 000000000..9c2f5cdc7
--- /dev/null
+++ b/packages/tracing/src/adapters/ai-sdk-telemetry.ts
@@ -0,0 +1,179 @@
+import type { Telemetry } from "ai";
+import type { AgentTracing, TurnInput } from "../agent-tracing.js";
+import type {
+ AttemptOperation,
+ ModelOperation,
+ ToolOperation,
+ TurnOperation,
+} from "../operations.js";
+import { modelContent, modelUsage } from "./ai-sdk-payload.js";
+
+type Event<K extends keyof Telemetry> = Parameters<NonNullable<Telemetry[K]>>[0];
+
+export interface AiSdkTelemetryOptions {
+ /** The turn a generation records. Called once per AI SDK call. */
+ readonly turn: TurnInput | ((call: { readonly callId: string }) => TurnInput);
+}
+
+interface Generation {
+ readonly startTimeMs: number;
+ turn?: Promise<TurnOperation>;
+ attempt?: Promise<AttemptOperation>;
+ model?: Promise<ModelOperation>;
+ readonly tools: Map<string, Promise<ToolOperation>>;
+}
+
+const ABANDONED = { outcome: "abandoned" } as const;
+
+/**
+ * An AI SDK telemetry integration that records each generation as a turn.
+ * Pass it to `telemetry.integrations` of `generateText`, `streamText`, or an
+ * agent such as `HarnessAgent`.
+ */
+export function aiSdkTelemetry(tracing: AgentTracing, options: AiSdkTelemetryOptions): Telemetry {
+ const generations = new Map<string, Generation>();
+
+ function generation(callId: string): Generation {
+ let state = generations.get(callId);
+ if (state === undefined) {
+ state = { startTimeMs: Date.now(), tools: new Map() };
+ generations.set(callId, state);
+ }
+ return state;
+ }
+
+ async function closeStep(state: Generation, failure?: unknown) {
+ const result =
+ failure === undefined ? ABANDONED : { outcome: "abandoned", failed: true, error: failure };
+ await (await state.model)?.complete(result);
+ for (const tool of state.tools.values()) await (await tool).complete(result);
+ state.model = undefined;
+ state.tools.clear();
+ const attempt = await state.attempt;
+ state.attempt = undefined;
+ if (failure === undefined) await attempt?.complete();
+ else await attempt?.fail(failure);
+ }
+
+ async function finish(
+ callId: string,
+ end: (turn: TurnOperation) => Promise<void>,
+ failure?: unknown,
+ ) {
+ const state = generations.get(callId);
+ generations.delete(callId);
+ if (state === undefined) return;
+ await closeStep(state, failure);
+ const turn = await state.turn;
+ if (turn !== undefined) await end(turn);
+ }
+
+ return {
+ onStart(event: Event<"onStart">) {
+ generation(event.callId);
+ },
+ async onStepStart(event: Event<"onStepStart">) {
+ const state = generation(event.callId);
+ await closeStep(state);
+ state.turn ??= tracing.turn({
+ startTimeMs: state.startTimeMs,
+ ...(typeof options.turn === "function"
+ ? options.turn({ callId: event.callId })
+ : options.turn),
+ });
+ const turn = await state.turn;
+ state.attempt = turn.attempt({ stepIndex: event.stepNumber, attempt: 0 });
+ },
+ async onLanguageModelCallStart(event: Event<"onLanguageModelCallStart">) {
+ const attempt = await generations.get(event.callId)?.attempt;
+ if (attempt === undefined) return;
+ generations.get(event.callId)!.model = attempt.modelCall({
+ provider: event.provider,
+ modelId: event.modelId,
+ messages: event.messages,
+ instructions: event.instructions,
+ tools: event.tools,
+ });
+ },
+ async executeLanguageModelCall({ callId, execute }) {
+ const model = await generations.get(callId)?.model;
+ if (model === undefined || model.finished) return execute();
+ try {
+ return await model.run(execute);
+ } catch (error) {
+ await model.fail(error);
+ throw error;
+ }
+ },
+ async onLanguageModelCallEnd(event: Event<"onLanguageModelCallEnd">) {
+ const state = generations.get(event.callId);
+ const model = await state?.model;
+ if (state === undefined || model === undefined) return;
+ state.model = undefined;
+ await model.complete({
+ outcome: "completed",
+ result: {
+ finishReason: event.finishReason,
+ usage: modelUsage(event.usage),
+ responseId: event.responseId,
+ responseModelId: event.modelId,
+ content: modelContent(event.content),
+ },
+ });
+ },
+ async onToolExecutionStart(event: Event<"onToolExecutionStart">) {
+ const state = generations.get(event.callId);
+ const attempt = await state?.attempt;
+ if (state === undefined || attempt === undefined) return;
+ const call = event.toolCall;
+ state.tools.set(
+ call.toolCallId,
+ attempt.tool({ callId: call.toolCallId, name: call.toolName, arguments: call.input }),
+ );
+ },
+ async executeTool({ callId, toolCallId, execute }) {
+ const tool = await generations.get(callId)?.tools.get(toolCallId);
+ return tool === undefined || tool.finished ? execute() : tool.run(execute);
+ },
+ async onToolExecutionEnd(event: Event<"onToolExecutionEnd">) {
+ const state = generations.get(event.callId);
+ const tool = await state?.tools.get(event.toolCall.toolCallId);
+ if (state === undefined || tool === undefined) return;
+ state.tools.delete(event.toolCall.toolCallId);
+ const output = event.toolOutput;
+ const completion =
+ output.type === "tool-error"
+ ? {
+ outcome: "failed",
+ errorType: output.error instanceof Error ? output.error.name : "tool_error",
+ error: output.error,
+ }
+ : { outcome: "completed", output: "output" in output ? output.output : undefined };
+ await tool.complete(completion);
+ },
+ async onStepEnd(event: Event<"onStepEnd">) {
+ const state = generations.get(event.callId);
+ if (state !== undefined) await closeStep(state);
+ },
+ async onEnd(event: Event<"onEnd">) {
+ await finish(event.callId, (turn) =>
+ turn.complete({
+ outcome: "completed",
+ usage: "totalUsage" in event ? modelUsage(event.totalUsage) : undefined,
+ }),
+ );
+ },
+ async onAbort(event: Event<"onAbort">) {
+ await finish(event.callId, (turn) => turn.complete({ outcome: "cancelled" }));
+ },
+ async onError(value: unknown) {
+ // Some agents report the bare error; it then belongs to every open call.
+ const { callId, error } =
+ value !== null && typeof value === "object" && "callId" in value && "error" in value
+ ? (value as { callId: string; error: unknown })
+ : { callId: undefined, error: value };
+ for (const id of callId === undefined ? [...generations.keys()] : [callId])
+ await finish(id, (turn) => turn.fail(error), error);
+ },
+ };
+}
diff --git a/packages/tracing/src/adapters/ai-sdk.integration.test.ts b/packages/tracing/src/adapters/ai-sdk.integration.test.ts
new file mode 100644
index 000000000..2423a584c
--- /dev/null
+++ b/packages/tracing/src/adapters/ai-sdk.integration.test.ts
@@ -0,0 +1,262 @@
+import { HarnessAgent } from "@ai-sdk/harness/agent";
+import type { HarnessV1, HarnessV1PromptControl, HarnessV1StreamPart } from "@ai-sdk/harness";
+import type { Experimental_SandboxSession as SandboxSession } from "@ai-sdk/provider-utils";
+import { context, trace } from "@opentelemetry/api";
+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
+import {
+ BasicTracerProvider,
+ InMemorySpanExporter,
+ SimpleSpanProcessor,
+} from "@opentelemetry/sdk-trace-base";
+import { generateText, isStepCount, tool } from "ai";
+import { MockLanguageModelV3 } from "ai/test";
+import { z } from "zod";
+import { describe, expect, it, vi } from "vitest";
+import { createAgentTracing, otelTelemetry } from "@eve/tracing";
+import { aiSdkTelemetry } from "@eve/tracing/ai-sdk";
+
+const usage = {
+ inputTokens: { total: 3, noCache: 3, cacheRead: 0, cacheWrite: 0 },
+ outputTokens: { total: 2, text: 2, reasoning: 0 },
+};
+const finishReason = { unified: "stop" as const, raw: undefined };
+
+function scriptedHarness(toolFails: boolean): HarnessV1 {
+ function prompt({
+ emit,
+ }: {
+ emit: (event: HarnessV1StreamPart) => void;
+ }): Promise<HarnessV1PromptControl> {
+ const done = Promise.resolve().then(() => {
+ emit({ type: "stream-start", modelId: "test-model" });
+ emit({
+ type: "tool-call",
+ toolCallId: "lookup-1",
+ toolName: "lookup",
+ input: "{}",
+ providerExecuted: true,
+ });
+ emit({
+ type: "tool-result",
+ toolCallId: "lookup-1",
+ toolName: "lookup",
+ result: toolFails ? "Private tool error" : "Private tool result",
+ isError: toolFails,
+ });
+ emit({ type: "finish-step", finishReason: { unified: "tool-calls", raw: undefined }, usage });
+ emit({ type: "text-start", id: "answer" });
+ emit({ type: "text-delta", id: "answer", delta: "Alice's answer" });
+ emit({ type: "text-end", id: "answer" });
+ emit({ type: "finish-step", finishReason, usage });
+ emit({
+ type: "finish",
+ finishReason,
+ totalUsage: {
+ ...usage,
+ inputTokens: { ...usage.inputTokens, total: 6 },
+ outputTokens: { ...usage.outputTokens, total: 4 },
+ },
+ });
+ });
+ return Promise.resolve({ done, async submitToolResult() {} });
+ }
+ const unsupported = async (): Promise<never> => {
+ throw new Error("Not used by this harness turn");
+ };
+ return {
+ specificationVersion: "harness-v1",
+ harnessId: "scripted",
+ builtinTools: { lookup: { inputSchema: z.object({}) } },
+ async doStart({ sessionId }) {
+ return {
+ sessionId,
+ isResume: false,
+ doPromptTurn: prompt,
+ doContinueTurn: prompt,
+ doCompact: unsupported,
+ doSuspendTurn: unsupported,
+ doDetach: unsupported,
+ doStop: unsupported,
+ async doDestroy() {},
+ };
+ },
+ };
+}
+
+const sandbox: SandboxSession = {
+ description: "In-memory harness test sandbox",
+ async run() {
+ return { exitCode: 0, stdout: "/workspace\n", stderr: "" };
+ },
+ async readFile() {
+ return null;
+ },
+ async readBinaryFile() {
+ return null;
+ },
+ async readTextFile() {
+ return null;
+ },
+ async writeFile() {},
+ async writeBinaryFile() {},
+ async writeTextFile() {},
+ async spawn() {
+ throw new Error("The scripted harness does not spawn processes");
+ },
+};
+
+describe("AI SDK HarnessAgent consumer", () => {
+ it.each(["generate", "stream"] as const)(
+ "records a real HarnessAgent %s lifecycle using only the package root",
+ async (method) => {
+ vi.stubEnv("VERCEL_ENV", "preview");
+ const exporter = new InMemorySpanExporter();
+ const provider = new BasicTracerProvider({
+ spanProcessors: [new SimpleSpanProcessor(exporter)],
+ });
+ context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
+ const tracing = createAgentTracing({ telemetry: otelTelemetry({ provider }) });
+ const agent = new HarnessAgent({
+ harness: scriptedHarness(method === "stream"),
+ telemetry: {
+ integrations: [
+ aiSdkTelemetry(tracing, {
+ turn: {
+ agentName: "support",
+ identity: { conversationId: "conversation", runId: method, turnId: "turn" },
+ sequence: 0,
+ framework: { name: "ai-sdk-harness", version: "1" },
+ },
+ }),
+ ],
+ },
+ });
+ const session = await agent.createSession({ sandboxSession: sandbox });
+ try {
+ let text = "";
+ if (method === "generate")
+ text = (await agent.generate({ session, prompt: "Help Alice inspect her project." }))
+ .text;
+ else {
+ const result = await agent.stream({ session, prompt: "Help Alice inspect her project." });
+ for await (const chunk of result.textStream) {
+ text += chunk;
+ expect(
+ exporter.getFinishedSpans().some((span) => span.name === "invoke_agent support"),
+ ).toBe(false);
+ }
+ }
+ expect(text).toBe("Alice's answer");
+ const spans = exporter.getFinishedSpans();
+ const turn = spans.find((span) => span.name === "invoke_agent support")!;
+ const attempts = spans.filter((span) => span.name === "agent.step");
+ const models = spans.filter((span) => span.name === "chat test-model");
+ const tool = spans.find((span) => span.name === "execute_tool lookup")!;
+ expect(spans).toHaveLength(6);
+ expect(attempts).toHaveLength(2);
+ expect(models).toHaveLength(2);
+ expect(
+ attempts.every((span) => span.parentSpanContext?.spanId === turn.spanContext().spanId),
+ ).toBe(true);
+ expect(
+ models.every((span) =>
+ attempts.some(
+ (attempt) => span.parentSpanContext?.spanId === attempt.spanContext().spanId,
+ ),
+ ),
+ ).toBe(true);
+ expect(tool.parentSpanContext?.spanId).toBe(attempts[0]!.spanContext().spanId);
+ expect(tool.status.code).toBe(method === "stream" ? 2 : 0);
+ expect(turn.attributes).toMatchObject({
+ "agent.trace.schema.version": 4,
+ "agent.framework.name": "ai-sdk-harness",
+ "gen_ai.usage.input_tokens": 6,
+ "gen_ai.usage.output_tokens": 4,
+ });
+ // Metadata-only capture is the default, so prompt and tool content stay out.
+ expect(JSON.stringify(spans.map((span) => span.attributes))).not.toMatch(/Private|Alice/);
+ expect(
+ spans.every((span) =>
+ Object.keys(span.attributes).every((key) => !key.startsWith("eve.")),
+ ),
+ ).toBe(true);
+ // Agent Runs finds a run's spans on Vercel by its session ID.
+ expect(spans.every((span) => span.attributes["vercel.session_id"] === method)).toBe(true);
+ } finally {
+ await session.destroy();
+ await tracing.shutdown();
+ context.disable();
+ vi.unstubAllEnvs();
+ }
+ },
+ );
+});
+
+describe("AI SDK generateText consumer", () => {
+ it("runs the model call and tool inside their spans", async () => {
+ const exporter = new InMemorySpanExporter();
+ const provider = new BasicTracerProvider({
+ spanProcessors: [new SimpleSpanProcessor(exporter)],
+ });
+ context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
+ const tracing = createAgentTracing({ telemetry: otelTelemetry({ provider }) });
+ const activeInModel: (string | undefined)[] = [];
+ let activeInTool: string | undefined;
+ const responses = [
+ [{ type: "tool-call" as const, toolCallId: "lookup-1", toolName: "lookup", input: "{}" }],
+ [{ type: "text" as const, text: "Bob's answer" }],
+ ];
+ const model = new MockLanguageModelV3({
+ provider: "test",
+ modelId: "test-model",
+ async doGenerate() {
+ activeInModel.push(trace.getActiveSpan()?.spanContext().spanId);
+ return {
+ content: responses.shift()!,
+ finishReason: { unified: "stop", raw: undefined },
+ usage,
+ warnings: [],
+ };
+ },
+ });
+ try {
+ const result = await generateText({
+ model,
+ prompt: "Help Bob find his order.",
+ stopWhen: isStepCount(2),
+ tools: {
+ lookup: tool({
+ inputSchema: z.object({}),
+ execute: async () => {
+ activeInTool = trace.getActiveSpan()?.spanContext().spanId;
+ return "found";
+ },
+ }),
+ },
+ telemetry: {
+ integrations: [
+ aiSdkTelemetry(tracing, {
+ turn: ({ callId }) => ({
+ agentName: "orders",
+ identity: { conversationId: "bob", runId: callId, turnId: "turn" },
+ sequence: 0,
+ }),
+ }),
+ ],
+ },
+ });
+ expect(result.text).toBe("Bob's answer");
+ const spans = exporter.getFinishedSpans();
+ const chats = spans.filter((span) => span.name === "chat test-model");
+ expect(spans.filter((span) => span.name === "invoke_agent orders")).toHaveLength(1);
+ expect(spans.filter((span) => span.name === "agent.step")).toHaveLength(2);
+ expect(activeInModel).toEqual(chats.map((span) => span.spanContext().spanId));
+ expect(activeInTool).toBe(
+ spans.find((span) => span.name === "execute_tool lookup")!.spanContext().spanId,
+ );
+ } finally {
+ await tracing.shutdown();
+ context.disable();
+ }
+ });
+});
diff --git a/packages/tracing/src/adapters/harness-agent.integration.test.ts b/packages/tracing/src/adapters/harness-agent.integration.test.ts
new file mode 100644
index 000000000..21c19b021
--- /dev/null
+++ b/packages/tracing/src/adapters/harness-agent.integration.test.ts
@@ -0,0 +1,164 @@
+import { describe, expect, it } from "vitest";
+import {
+ BasicTracerProvider,
+ InMemorySpanExporter,
+ SimpleSpanProcessor,
+} from "@opentelemetry/sdk-trace-base";
+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
+import { context } from "@opentelemetry/api";
+import { createAgentTracing, otelTelemetry } from "@eve/tracing";
+
+describe("outside-eve agent", () => {
+ it("runs wrapped operations and handle-based tools using only the facade", async () => {
+ const exporter = new InMemorySpanExporter();
+ const provider = new BasicTracerProvider({
+ spanProcessors: [new SimpleSpanProcessor(exporter)],
+ });
+ const manager = new AsyncLocalStorageContextManager().enable();
+ context.setGlobalContextManager(manager);
+ const tracing = createAgentTracing({
+ telemetry: otelTelemetry({ provider }),
+ });
+ try {
+ const value = await tracing.turn(
+ {
+ agentName: "support",
+ identity: { conversationId: "conversation", runId: "run", turnId: "turn" },
+ sequence: 0,
+ },
+ (turn) =>
+ turn.attempt({ stepIndex: 0, attempt: 0 }, async (attempt) => {
+ const answer = await attempt.modelCall({ provider: "test", modelId: "model" }, () => ({
+ result: "answer",
+ finishReason: "stop",
+ usage: { inputTokens: 3, outputTokens: 2 },
+ }));
+ expect(answer).toBe("answer");
+ return attempt.tool({ callId: "lookup", name: "lookup" }, async (tool) => {
+ await tool.approval(
+ { requestId: "approval", describe: () => ({ outcome: "approved" }) },
+ () => true,
+ );
+ return "Alice's answer";
+ });
+ }),
+ );
+ expect(value).toBe("Alice's answer");
+ await tracing.forceFlush();
+ const spans = exporter.getFinishedSpans();
+ expect(spans.map((span) => span.name)).toEqual([
+ "chat model",
+ "agent.approval",
+ "execute_tool lookup",
+ "agent.step",
+ "invoke_agent support",
+ ]);
+ const turn = spans.at(-1)!;
+ expect(spans.find((span) => span.name === "agent.step")?.parentSpanContext?.spanId).toBe(
+ turn.spanContext().spanId,
+ );
+ expect(turn.attributes["gen_ai.usage.input_tokens"]).toBe(3);
+ expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("Alice");
+ const handle = await tracing.turn({
+ agentName: "support",
+ identity: { conversationId: "conversation", runId: "next", turnId: "turn" },
+ sequence: 0,
+ });
+ expect(handle).not.toHaveProperty("snapshot");
+ await handle.complete();
+ } finally {
+ await tracing.shutdown();
+ context.disable();
+ manager.disable();
+ }
+ });
+ it("preserves application errors and closes memory and tool callbacks", async () => {
+ const exporter = new InMemorySpanExporter();
+ const provider = new BasicTracerProvider({
+ spanProcessors: [new SimpleSpanProcessor(exporter)],
+ });
+ const tracing = createAgentTracing({
+ telemetry: otelTelemetry({ provider }),
+ });
+ const error = new TypeError("private failure");
+ try {
+ await expect(
+ tracing.turn(
+ {
+ agentName: "support",
+ identity: { conversationId: "c", runId: "r", turnId: "t" },
+ sequence: 0,
+ },
+ (turn) =>
+ turn.attempt({ stepIndex: 0, attempt: 0 }, (attempt) =>
+ attempt.tool({ callId: "tool", name: "lookup" }, () => {
+ throw error;
+ }),
+ ),
+ ),
+ ).rejects.toBe(error);
+ const failures = exporter.getFinishedSpans();
+ expect(failures).toHaveLength(3);
+ expect(
+ failures.every(
+ (span) => span.status.code === 2 && span.attributes["error.type"] === "TypeError",
+ ),
+ ).toBe(true);
+ expect(JSON.stringify(failures.map((span) => span.events))).not.toContain("private failure");
+ await tracing.turn(
+ {
+ agentName: "support",
+ identity: { conversationId: "c", runId: "memory", turnId: "t" },
+ sequence: 0,
+ },
+ (turn) =>
+ turn.memory(
+ {
+ operation: "search_memory",
+ phase: "retrieve",
+ slot: "history",
+ storeId: "store",
+ describe: () => ({ recordCount: 2 }),
+ },
+ () => "records",
+ ),
+ );
+ expect(
+ exporter.getFinishedSpans().find((span) => span.name === "search_memory")?.attributes[
+ "gen_ai.memory.record.count"
+ ],
+ ).toBe(2);
+ } finally {
+ await tracing.shutdown();
+ }
+ });
+ it("honors the supplied provider's root sampling decision", async () => {
+ const exporter = new InMemorySpanExporter();
+ const provider = new BasicTracerProvider({
+ sampler: { shouldSample: () => ({ decision: 0 }), toString: () => "drop" },
+ spanProcessors: [new SimpleSpanProcessor(exporter)],
+ });
+ const tracing = createAgentTracing({
+ telemetry: otelTelemetry({ provider }),
+ });
+ try {
+ await tracing.turn(
+ {
+ agentName: "support",
+ identity: { conversationId: "c", runId: "r", turnId: "t" },
+ sequence: 0,
+ capture: { emit: true, recordInputs: true, recordOutputs: true },
+ },
+ async (turn) => {
+ expect(turn.capture).toEqual({ emit: false, recordInputs: false, recordOutputs: false });
+ await turn.attempt({ stepIndex: 0, attempt: 0 }, (attempt) =>
+ attempt.tool({ callId: "tool", name: "lookup" }, () => "private"),
+ );
+ },
+ );
+ expect(exporter.getFinishedSpans()).toHaveLength(0);
+ } finally {
+ await tracing.shutdown();
+ }
+ });
+});
diff --git a/packages/tracing/src/adapters/otel-error.ts b/packages/tracing/src/adapters/otel-error.ts
new file mode 100644
index 000000000..35363f7db
--- /dev/null
+++ b/packages/tracing/src/adapters/otel-error.ts
@@ -0,0 +1,34 @@
+import { SpanStatusCode, type Span } from "@opentelemetry/api";
+import { boundedTraceError } from "./serialization.js";
+import { truncateTelemetryText } from "./serialization.js";
+import { contentAttribute, textContentAttribute } from "./serialization.js";
+
+/** Call with policy-projected errors; undefined retains failure status without content. */
+export function recordAgentSpanError(span: Span, error: unknown, errorType?: string): void {
+ span.setAttribute(
+ "error.type",
+ truncateTelemetryText(errorType ?? (error instanceof Error ? error.name : "_OTHER"), 128),
+ );
+ // A thrown Error keeps its own name; a wrapped value takes the error type.
+ const wrapped = error !== undefined && !(error instanceof Error);
+ if (wrapped) {
+ const serialized = contentAttribute(error);
+ const message =
+ typeof error === "object" && error !== null ? Reflect.get(error, "message") : undefined;
+ const detail =
+ typeof error === "string"
+ ? textContentAttribute(error)
+ : typeof message === "string"
+ ? textContentAttribute(serialized === undefined ? message : `${message}\n${serialized}`)
+ : serialized;
+ if (detail !== undefined) error = new Error(detail);
+ }
+ if (error instanceof Error) {
+ const bounded = boundedTraceError(error);
+ if (wrapped && errorType !== undefined) bounded.name = truncateTelemetryText(errorType, 128);
+ span.recordException(bounded);
+ span.setStatus({ code: SpanStatusCode.ERROR, message: bounded.message });
+ } else {
+ span.setStatus({ code: SpanStatusCode.ERROR });
+ }
+}
diff --git a/packages/tracing/src/adapters/otel-ids.ts b/packages/tracing/src/adapters/otel-ids.ts
new file mode 100644
index 000000000..9d4d607c6
--- /dev/null
+++ b/packages/tracing/src/adapters/otel-ids.ts
@@ -0,0 +1,82 @@
+import { createHash } from "node:crypto";
+
+/**
+ * ID generator installed by the host OTel provider for reserved-ID emission. A span
+ * whose lifetime crosses durable worker boundaries (`invoke_agent`) is emitted
+ * at its terminal; priming the next span id lets that span carry the
+ * pre-allocated id its descendants already parented to.
+ */
+export class AgentSpanIdGenerator {
+ #primedSpanId: string | undefined;
+ #primedTraceId: string | undefined;
+
+ /** Reserves a span id for a span emitted later via {@link withSpanId}. */
+ allocateSpanId(): string {
+ return randomHexId(16);
+ }
+
+ /** Derives one span id for a replay-stable instrumentation event. */
+ deriveSpanId(key: string): string {
+ return deriveAgentSpanId(key);
+ }
+
+ deriveTraceId(key: string): string {
+ const traceId = createHash("sha256").update(key).digest("hex").slice(0, 32);
+ return /^0+$/u.test(traceId) ? "00000000000000000000000000000001" : traceId;
+ }
+
+ generateSpanId(): string {
+ const primed = this.#primedSpanId;
+ if (primed !== undefined) {
+ this.#primedSpanId = undefined;
+ return primed;
+ }
+ return randomHexId(16);
+ }
+
+ generateTraceId(): string {
+ const primed = this.#primedTraceId;
+ if (primed !== undefined) {
+ this.#primedTraceId = undefined;
+ return primed;
+ }
+ return randomHexId(32);
+ }
+
+ /** Runs `startSpan` so the next span started carries `spanId`. */
+ withSpanId<T>(spanId: string, startSpan: () => T): T {
+ this.#primedSpanId = spanId;
+ try {
+ return startSpan();
+ } finally {
+ this.#primedSpanId = undefined;
+ }
+ }
+
+ withTraceId<T>(traceId: string, startSpan: () => T): T {
+ this.#primedTraceId = traceId;
+ try {
+ return startSpan();
+ } finally {
+ this.#primedTraceId = undefined;
+ }
+ }
+}
+
+function deriveAgentSpanId(key: string): string {
+ const spanId = createHash("sha256").update(key).digest("hex").slice(0, 16);
+ return /^0+$/u.test(spanId) ? "0000000000000001" : spanId;
+}
+
+const HEX_DIGITS = "0123456789abcdef";
+
+function randomHexId(length: number): string {
+ let id: string;
+ do {
+ id = "";
+ for (let index = 0; index < length; index += 1) {
+ id += HEX_DIGITS[Math.trunc(Math.random() * 16)];
+ }
+ } while (!/[1-9a-f]/u.test(id));
+ return id;
+}
diff --git a/packages/tracing/src/adapters/otel.integration.test.ts b/packages/tracing/src/adapters/otel.integration.test.ts
new file mode 100644
index 000000000..fb8640861
--- /dev/null
+++ b/packages/tracing/src/adapters/otel.integration.test.ts
@@ -0,0 +1,120 @@
+import { describe, expect, it } from "vitest";
+import {
+ BasicTracerProvider,
+ InMemorySpanExporter,
+ SimpleSpanProcessor,
+} from "@opentelemetry/sdk-trace-base";
+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
+import { context, ROOT_CONTEXT, createContextKey } from "@opentelemetry/api";
+import { otelTelemetry } from "./otel.js";
+import { AgentSpanIdGenerator } from "./otel-ids.js";
+import { createSpanWriter } from "../core/writer.js";
+
+describe("OTel trace output", () => {
+ it("preserves explicit host context at span start and activation", async () => {
+ const key = createContextKey("host-audience");
+ const host = ROOT_CONTEXT.setValue(key, "private");
+ const exporter = new InMemorySpanExporter();
+ let audience: unknown;
+ const provider = new BasicTracerProvider({
+ spanProcessors: [
+ {
+ onStart(_span, parent) {
+ audience = parent.getValue(key);
+ },
+ onEnd() {},
+ async forceFlush() {},
+ async shutdown() {},
+ },
+ new SimpleSpanProcessor(exporter),
+ ],
+ });
+ const manager = new AsyncLocalStorageContextManager().enable();
+ context.setGlobalContextManager(manager);
+ try {
+ const operation = createSpanWriter({
+ telemetry: otelTelemetry({ provider }),
+ }).start(
+ {
+ type: "channelRequest",
+ operationId: "request",
+ name: "agent.channel.request",
+ kind: "SERVER",
+ attributes: {},
+ },
+ { emit: true, recordInputs: false, recordOutputs: false },
+ host,
+ );
+ expect(audience).toBe("private");
+ operation.run(() => expect(context.active().getValue(key)).toBe("private"));
+ operation.end();
+ expect(exporter.getFinishedSpans()).toHaveLength(1);
+ } finally {
+ context.disable();
+ manager.disable();
+ await provider.shutdown();
+ }
+ });
+
+ it("maps deferred spans and late updates without changing reserved identity", async () => {
+ const exporter = new InMemorySpanExporter();
+ const idGenerator = new AgentSpanIdGenerator();
+ const provider = new BasicTracerProvider({
+ idGenerator,
+ spanProcessors: [new SimpleSpanProcessor(exporter)],
+ });
+ try {
+ const telemetry = otelTelemetry({ provider, idGenerator });
+ const capture = { emit: true, recordInputs: false, recordOutputs: false };
+ const span = {
+ type: "activation" as const,
+ operationId: "activation",
+ name: "invoke_agent support",
+ root: true,
+ startTimeMs: 1000,
+ attributes: { "agent.run.id": "run", "agent.trace.schema.version": 4 },
+ links: [
+ {
+ context: { traceId: "1".repeat(32), spanId: "2".repeat(16), traceFlags: 1 },
+ relationship: "agent.dispatch" as const,
+ },
+ ],
+ };
+ const reference = {
+ traceId: telemetry.ids!.traceId("activation"),
+ spanId: telemetry.ids!.spanId("activation"),
+ traceFlags: 1,
+ };
+ const engine = createSpanWriter({ telemetry });
+ const childReference = { ...reference, spanId: telemetry.ids!.spanId("tool") };
+ const child = engine.start(
+ {
+ type: "tool",
+ operationId: "tool",
+ name: "execute_tool lookup",
+ parent: reference,
+ attributes: span.attributes,
+ },
+ capture,
+ undefined,
+ childReference,
+ );
+ child.setAttribute("agent.connection.name", "catalog");
+ child.end(2000);
+ const root = engine.start(span, capture, undefined, reference);
+ root.addEvent("turn.completed", undefined, 3000);
+ root.end(3000);
+ const recorded = exporter.getFinishedSpans();
+ expect(recorded[0]!.spanContext().spanId).toBe(childReference.spanId);
+ expect(recorded[0]!.parentSpanContext?.spanId).toBe(reference.spanId);
+ expect(recorded[0]!.attributes).toMatchObject({
+ "agent.connection.name": "catalog",
+ "agent.trace.schema.version": 4,
+ });
+ expect(recorded[1]!.spanContext()).toMatchObject(reference);
+ expect(recorded[1]!.links[0]!.attributes).toEqual({ "agent.link.type": "agent.dispatch" });
+ } finally {
+ await provider.shutdown();
+ }
+ });
+});
diff --git a/packages/tracing/src/adapters/otel.ts b/packages/tracing/src/adapters/otel.ts
new file mode 100644
index 000000000..6bc436ec3
--- /dev/null
+++ b/packages/tracing/src/adapters/otel.ts
@@ -0,0 +1,232 @@
+import {
+ ROOT_CONTEXT,
+ SpanKind,
+ SpanStatusCode,
+ context,
+ trace,
+ createTraceState,
+ createContextKey,
+ type SpanContext,
+ type Context as OtelContext,
+ type TracerProvider,
+} from "@opentelemetry/api";
+import { recordAgentSpanError } from "./otel-error.js";
+import { AgentSpanIdGenerator } from "./otel-ids.js";
+import type {
+ Attributes,
+ AgentTelemetry,
+ OutputMapping,
+ PreparedSpan,
+ SpanWriter,
+ TraceReference,
+ ExecutionContext,
+ ActiveOperation,
+} from "../core/types.js";
+import { activeOperation, intersectCapture } from "../core/activation.js";
+import { withCapture } from "../capture.js";
+
+type Context = OtelContext;
+const ACTIVE_OPERATION = createContextKey("agent.tracing.active-operation");
+const SUPPRESS_TRACING_KEY = createContextKey("OpenTelemetry SDK Context Key SUPPRESS_TRACING");
+
+export function withErrorContent<T extends { setValue(key: symbol, value: unknown): T }>(
+ context: T,
+ allowed: boolean,
+): T {
+ return withCapture(context, { emit: true, recordInputs: allowed, recordOutputs: allowed });
+}
+
+function suppressTracing(context: Context): Context {
+ return context.setValue(SUPPRESS_TRACING_KEY, true);
+}
+
+export function activeTraceOperation(
+ host: Pick<Context, "getValue"> = context.active(),
+): ActiveOperation | undefined {
+ return typeof host.getValue === "function"
+ ? (host.getValue(ACTIVE_OPERATION) as ActiveOperation | undefined)
+ : undefined;
+}
+
+function parentContext(reference: TraceReference | undefined, host?: ExecutionContext): Context {
+ const base = (host as Context | undefined) ?? ROOT_CONTEXT;
+ return reference === undefined
+ ? base
+ : trace.setSpan(base, trace.wrapSpanContext(otelReference(reference)));
+}
+
+function otelReference(reference: TraceReference): SpanContext {
+ return {
+ ...reference,
+ traceState:
+ reference.tracestate === undefined ? undefined : createTraceState(reference.tracestate),
+ };
+}
+
+function portableReference(reference: SpanContext): TraceReference {
+ return {
+ traceId: reference.traceId,
+ spanId: reference.spanId,
+ traceFlags: reference.traceFlags,
+ isRemote: reference.isRemote,
+ tracestate: reference.traceState?.serialize(),
+ };
+}
+
+function mappedAttributes(
+ mapping: OutputMapping | undefined,
+ span: PreparedSpan,
+ attributes: Attributes,
+): Attributes {
+ return mapping?.attributes(span, attributes) ?? attributes;
+}
+
+/**
+ * Applied when the host passes no mapping. On Vercel it adds the session ID
+ * Agent Runs uses to find a run's spans.
+ */
+const DEFAULT_MAPPING: OutputMapping = {
+ attributes(_span, attributes) {
+ const runId = attributes["agent.run.id"];
+ return onVercel() && typeof runId === "string" && attributes["vercel.session_id"] === undefined
+ ? { ...attributes, "vercel.session_id": runId }
+ : attributes;
+ },
+ link: (_span, link) => ({ "agent.link.type": link.relationship }),
+};
+
+function onVercel(): boolean {
+ const env = (globalThis as { process?: { env?: Record<string, string | undefined> } }).process
+ ?.env;
+ return env?.VERCEL_ENV !== undefined;
+}
+
+export interface OtelTelemetryOptions {
+ /** Defaults to the global tracer provider. */
+ readonly provider?: TracerProvider;
+ /**
+ * Generator installed on `provider`. Required for durable tracing so a span
+ * started in a later process carries the ID its children already reference.
+ */
+ readonly idGenerator?: AgentSpanIdGenerator;
+ /** Defaults to recording every trace. */
+ readonly samplesTrace?: (
+ traceId: string,
+ operation: { name: string; attributes: Attributes },
+ ) => boolean;
+ /** Replaces the default mapping, which adds `vercel.session_id` on Vercel. */
+ readonly mapping?: OutputMapping;
+ readonly tracerName?: string;
+ /** Recorded as the spans' instrumentation scope version. */
+ readonly tracerVersion?: string;
+ /** Override the provider's lifecycle, for example to flush metric readers too. */
+ readonly forceFlush?: () => Promise<void>;
+ readonly shutdown?: () => Promise<void>;
+}
+
+/** OpenTelemetry implementation of {@link AgentTelemetry}. */
+export function otelTelemetry(options: OtelTelemetryOptions = {}): AgentTelemetry {
+ const provider = options.provider ?? trace.getTracerProvider();
+ const tracer = provider.getTracer(options.tracerName ?? "agent.tracing", options.tracerVersion);
+ const mapping = options.mapping ?? DEFAULT_MAPPING;
+ const idGenerator = options.idGenerator;
+ function lifecycle(method: "forceFlush" | "shutdown"): () => Promise<void> {
+ return (
+ options[method] ??
+ (async () => {
+ const callback = (
+ provider as TracerProvider & { forceFlush?(): Promise<void>; shutdown?(): Promise<void> }
+ )[method];
+ if (callback === undefined)
+ throw new Error(`The tracer provider does not support ${method}.`);
+ await callback.call(provider);
+ })
+ );
+ }
+ function start(span: PreparedSpan, executionContext?: ExecutionContext): SpanWriter {
+ const recorded = tracer.startSpan(
+ span.name,
+ {
+ attributes: mappedAttributes(mapping, span, span.attributes),
+ kind: span.kind === undefined ? undefined : SpanKind[span.kind],
+ root: span.root,
+ startTime: span.startTimeMs,
+ links: span.links?.map((link) => ({
+ context: otelReference(link.context),
+ attributes: mapping?.link(span, link) ?? { "agent.link.type": link.relationship },
+ })),
+ },
+ parentContext(span.root ? undefined : span.parent, executionContext),
+ );
+ return {
+ reference: portableReference(recorded.spanContext()),
+ setAttribute(key, value) {
+ for (const [name, mapped] of Object.entries(
+ mappedAttributes(mapping, span, { [key]: value }),
+ )) {
+ if (mapped !== undefined) recorded.setAttribute(name, mapped);
+ }
+ },
+ addEvent: (name, attributes, timeMs) => {
+ recorded.addEvent(name, attributes, timeMs);
+ },
+ fail: (error, errorType) => recordAgentSpanError(recorded, error, errorType),
+ setStatus: (code) => {
+ recorded.setStatus({ code: SpanStatusCode[code] });
+ },
+ end: (timeMs) => {
+ recorded.end(timeMs);
+ },
+ };
+ }
+ return {
+ startSpan(span, { reference, context: host } = {}) {
+ if (reference === undefined || idGenerator === undefined) return start(span, host);
+ const writer = idGenerator.withTraceId(reference.traceId, () =>
+ idGenerator.withSpanId(reference.spanId, () => start(span, host)),
+ );
+ if (
+ writer.reference.spanId !== reference.spanId ||
+ writer.reference.traceId !== reference.traceId
+ ) {
+ throw new Error("The tracer provider must use the telemetry's ID generator.");
+ }
+ return writer;
+ },
+ active: () => activeTraceOperation(),
+ suppressed<T>(execute: () => T): T {
+ return context.with(withErrorContent(suppressTracing(ROOT_CONTEXT), false), execute);
+ },
+ run(operation, execute, executionContext) {
+ let capture = intersectCapture(operation.capture, activeTraceOperation()?.capture);
+ // Retain baggage and host context while replacing the semantic parent span.
+ let active = trace.setSpan(
+ (executionContext as Context | undefined) ?? context.active(),
+ trace.wrapSpanContext(otelReference(operation.reference)),
+ );
+ capture = intersectCapture(
+ capture,
+ (active.getValue(ACTIVE_OPERATION) as ActiveOperation | undefined)?.capture,
+ );
+ active = active.setValue(ACTIVE_OPERATION, activeOperation(operation, capture));
+ active = withCapture(active, capture);
+ if (!capture.emit || (operation.reference.traceFlags & 1) === 0)
+ active = suppressTracing(active);
+ return context.with(active, execute);
+ },
+ samples: (traceId, span) =>
+ options.samplesTrace?.(traceId, {
+ name: span.name,
+ attributes: mappedAttributes(mapping, span, span.attributes),
+ }) ?? true,
+ ids:
+ idGenerator === undefined
+ ? undefined
+ : {
+ traceId: (key) => idGenerator.deriveTraceId(key),
+ spanId: (key) => idGenerator.deriveSpanId(key),
+ },
+ forceFlush: lifecycle("forceFlush"),
+ shutdown: lifecycle("shutdown"),
+ };
+}
diff --git a/packages/tracing/src/adapters/serialization.test.ts b/packages/tracing/src/adapters/serialization.test.ts
new file mode 100644
index 000000000..448baee35
--- /dev/null
+++ b/packages/tracing/src/adapters/serialization.test.ts
@@ -0,0 +1,316 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+ CONTENT_ATTRIBUTE_LIMIT,
+ genAiInputMessagesAttribute as serializeInputMessages,
+ genAiOutputMessagesAttribute,
+ genAiSystemInstructionsAttribute,
+ genAiToolDefinitionsAttribute,
+ toolResultsContentAttribute,
+} from "./serialization.js";
+const genAiInputMessagesAttribute = (messages: unknown) =>
+ serializeInputMessages(messages, (kind) => (typeof kind === "string" ? kind : undefined));
+
+const FRAMEWORK_MESSAGE_KINDS = [
+ "context.instruction",
+ "context.state",
+ "context.compaction",
+ "memory.load",
+ "execution.continuation",
+ "execution.retry",
+] as const;
+
+describe("GenAI message attributes", () => {
+ it.each([
+ ["Buffer", () => Buffer.alloc(64, 97)],
+ ["Uint8Array", () => new Uint8Array(64).fill(97)],
+ ["base64", () => "a".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)],
+ ["data URL", () => `data:image/png;base64,${"a".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)}`],
+ ] as const)("omits %s attachment payloads", (_, createData) => {
+ const data = createData();
+ const entries = Object.entries;
+ let binaryEnumerations = 0;
+ const spy = vi.spyOn(Object, "entries").mockImplementation((value) => {
+ if (value === data) binaryEnumerations += 1;
+ return entries(value);
+ });
+
+ let attribute: string | undefined;
+ try {
+ attribute = genAiInputMessagesAttribute([
+ {
+ role: "user",
+ content: [
+ { type: "text", text: "Summarize file." },
+ { type: "file", mediaType: "application/octet-stream", filename: "test.bin", data },
+ ],
+ },
+ ]);
+ } finally {
+ spy.mockRestore();
+ }
+
+ expect(binaryEnumerations).toBe(0);
+ expect(attribute).toBe(
+ '[{"parts":[{"content":"Summarize file.","type":"text"},{"type":"file"}],"role":"user"}]',
+ );
+ });
+
+ it("preserves every user-message kind in the GenAI input attribute", () => {
+ expect(
+ genAiInputMessagesAttribute([
+ { content: "A real user message.", kind: "user", role: "user" },
+ {
+ content: "Continue the interrupted turn.",
+ kind: "execution.continuation",
+ role: "user",
+ },
+ ]),
+ ).toBe(
+ '[{"kind":"user","parts":[{"content":"A real user message.","type":"text"}],"role":"user"},{"kind":"execution.continuation","parts":[{"content":"Continue the interrupted turn.","type":"text"}],"role":"user"}]',
+ );
+ });
+
+ it.each(FRAMEWORK_MESSAGE_KINDS)("preserves %s in the GenAI input attribute", (kind) => {
+ const attribute = genAiInputMessagesAttribute([
+ { content: "Framework message.", kind, role: "user" },
+ ]);
+
+ expect(JSON.parse(attribute!)).toEqual([
+ {
+ kind,
+ parts: [{ content: "Framework message.", type: "text" }],
+ role: "user",
+ },
+ ]);
+ });
+
+ it.each([
+ ["only", [{ content: "x".repeat(CONTENT_ATTRIBUTE_LIMIT + 1), role: "user" }], undefined],
+ [
+ "newest",
+ [
+ { content: "older message", role: "assistant" },
+ {
+ content: "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2),
+ kind: "context.state",
+ role: "user",
+ },
+ ],
+ "context.state",
+ ],
+ ] as const)("keeps a truncated %s message when it alone exceeds the cap", (_, messages, kind) => {
+ const attribute = genAiInputMessagesAttribute(messages);
+
+ expect(attribute).toBeDefined();
+ expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
+ const parsed = JSON.parse(attribute!) as Array<Record<string, unknown>>;
+ expect(parsed).toHaveLength(1);
+ expect(parsed[0]).toMatchObject({
+ parts: [{ content: expect.stringMatching(/… \[truncated\]$/u), type: "text" }],
+ role: "user",
+ });
+ expect(parsed[0]?.kind).toBe(kind);
+ expect(attribute).not.toContain("older message");
+ });
+
+ it("formats model input, output, and system instructions for inspectors", () => {
+ expect(
+ genAiInputMessagesAttribute([
+ { content: "hello", kind: "user", role: "user" },
+ {
+ content: [
+ {
+ input: { message: "echo" },
+ toolCallId: "call-1",
+ toolName: "delegate",
+ type: "tool-call",
+ },
+ ],
+ role: "assistant",
+ },
+ ]),
+ ).toBe(
+ '[{"kind":"user","parts":[{"content":"hello","type":"text"}],"role":"user"},{"parts":[{"arguments":{"message":"echo"},"id":"call-1","name":"delegate","type":"tool_call"}],"role":"assistant"}]',
+ );
+ expect(genAiSystemInstructionsAttribute("Be concise.")).toBe(
+ '[{"content":"Be concise.","type":"text"}]',
+ );
+ expect(
+ genAiOutputMessagesAttribute(
+ [
+ { text: "Working.", type: "text" },
+ {
+ callId: "call-1",
+ input: { message: "echo" },
+ toolName: "delegate",
+ type: "tool-call",
+ },
+ ],
+ "tool-calls",
+ ),
+ ).toBe(
+ '[{"finish_reason":"tool_call","parts":[{"content":"Working.","type":"text"},{"arguments":{"message":"echo"},"id":"call-1","name":"delegate","type":"tool_call"}],"role":"assistant"}]',
+ );
+ });
+
+ it.each([
+ ["near the limit", "line of instructions\n".repeat(1500)],
+ ["over the limit", "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)],
+ ["escape-heavy", "\\\n".repeat(9000)],
+ ])("keeps %s system instructions as valid capped JSON", (_, instructions) => {
+ const attribute = genAiSystemInstructionsAttribute(instructions);
+
+ expect(attribute).toBeDefined();
+ expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
+ const parsed = JSON.parse(attribute!) as Array<Record<string, unknown>>;
+ expect(parsed).toHaveLength(1);
+ expect(parsed[0]).toMatchObject({ type: "text" });
+ expect(parsed[0]?.content).toEqual(expect.stringContaining("… [truncated]"));
+ });
+
+ it("marks a message truncated when oversized non-text parts are omitted", () => {
+ const attribute = genAiInputMessagesAttribute([
+ {
+ content: [
+ { text: "partial assistant answer", type: "text" },
+ {
+ input: { payload: "x".repeat(CONTENT_ATTRIBUTE_LIMIT) },
+ toolCallId: "call-1",
+ toolName: "delegate",
+ type: "tool-call",
+ },
+ ],
+ role: "assistant",
+ },
+ ]);
+
+ expect(attribute).toBeDefined();
+ expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
+ const parsed = JSON.parse(attribute!) as Array<Record<string, unknown>>;
+ expect(parsed).toEqual([
+ {
+ parts: [{ content: "partial assistant answer… [truncated]", type: "text" }],
+ role: "assistant",
+ },
+ ]);
+ });
+});
+
+describe("genAiOutputMessagesAttribute", () => {
+ it("truncates an oversized response instead of dropping it", () => {
+ const attribute = genAiOutputMessagesAttribute(
+ [{ text: "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2), type: "text" }],
+ "stop",
+ );
+ expect(attribute).toBeDefined();
+ expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
+ const [message] = JSON.parse(attribute!) as {
+ finish_reason: string;
+ role: string;
+ parts: { content: string }[];
+ }[];
+ expect(message).toMatchObject({ finish_reason: "stop", role: "assistant" });
+ expect(message!.parts[0]!.content).toContain("… [truncated]");
+ });
+});
+
+describe("genAiToolDefinitionsAttribute", () => {
+ it("maps AI SDK input schemas to GenAI parameters", () => {
+ expect(
+ genAiToolDefinitionsAttribute([
+ {
+ description: "Get the current weather.",
+ inputSchema: {
+ properties: { city: { type: "string" } },
+ required: ["city"],
+ type: "object",
+ },
+ name: "get_weather",
+ },
+ ]),
+ ).toBe(
+ '[{"name":"get_weather","description":"Get the current weather.","parameters":{"properties":{"city":{"type":"string"}},"required":["city"],"type":"object"}}]',
+ );
+ });
+
+ it("drops trailing definitions to keep oversized content valid JSON", () => {
+ const attribute = genAiToolDefinitionsAttribute([
+ { inputSchema: { type: "object" }, name: "first" },
+ {
+ description: "x".repeat(CONTENT_ATTRIBUTE_LIMIT),
+ inputSchema: { type: "object" },
+ name: "oversized",
+ },
+ { inputSchema: { type: "object" }, name: "last" },
+ ]);
+
+ expect(attribute).toBeDefined();
+ expect(attribute!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
+ expect(JSON.parse(attribute!)).toEqual([{ name: "first", parameters: { type: "object" } }]);
+ });
+
+ it("returns undefined when no tool definitions are available", () => {
+ expect(genAiToolDefinitionsAttribute([])).toBeUndefined();
+ expect(genAiToolDefinitionsAttribute(undefined)).toBeUndefined();
+ });
+});
+
+describe("toolResultsContentAttribute", () => {
+ it("returns the full payload when it fits", () => {
+ const json = toolResultsContentAttribute([
+ { input: { query: "weather" }, output: { results: ["sunny"] }, toolName: "web_search" },
+ ]);
+ expect(json).toBe(
+ '[{"input":{"query":"weather"},"output":{"results":["sunny"]},"toolName":"web_search"}]',
+ );
+ });
+
+ it("returns undefined for no results", () => {
+ expect(toolResultsContentAttribute([])).toBeUndefined();
+ });
+
+ it("keeps oversized payloads valid JSON by capping entry text", () => {
+ const json = toolResultsContentAttribute([
+ {
+ input: { query: "weather" },
+ output: { excerpts: "x".repeat(CONTENT_ATTRIBUTE_LIMIT * 2) },
+ toolName: "web_search",
+ },
+ ]);
+ expect(json).toBeDefined();
+ expect(json!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
+ const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;
+ expect(parsed[0]!.toolName).toBe("web_search");
+ expect(parsed[0]!.input).toBe('{"query":"weather"}');
+ expect(parsed[0]!.output).toContain("… [truncated]");
+ });
+
+ it("preserves the error key through truncation", () => {
+ const json = toolResultsContentAttribute([
+ {
+ error: `quota exceeded ${"y".repeat(CONTENT_ATTRIBUTE_LIMIT * 2)}`,
+ input: { query: "again" },
+ toolName: "web_search",
+ },
+ ]);
+ const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;
+ expect(parsed[0]!.error).toContain("quota exceeded");
+ expect(parsed[0]!.error).toContain("… [truncated]");
+ expect(parsed[0]!.output).toBeUndefined();
+ });
+
+ it("splits the budget across many oversized entries", () => {
+ const entries = Array.from({ length: 8 }, (_, index) => ({
+ input: { index },
+ output: "z".repeat(CONTENT_ATTRIBUTE_LIMIT),
+ toolName: `tool_${index}`,
+ }));
+ const json = toolResultsContentAttribute(entries);
+ expect(json).toBeDefined();
+ expect(json!.length).toBeLessThanOrEqual(CONTENT_ATTRIBUTE_LIMIT);
+ const parsed = JSON.parse(json!) as Array<Record<string, unknown>>;
+ expect(parsed).toHaveLength(8);
+ expect(parsed[7]!.toolName).toBe("tool_7");
+ });
+});
diff --git a/packages/tracing/src/adapters/serialization.ts b/packages/tracing/src/adapters/serialization.ts
new file mode 100644
index 000000000..b04878da1
--- /dev/null
+++ b/packages/tracing/src/adapters/serialization.ts
@@ -0,0 +1,336 @@
+/**
+ * Serializes model and tool payloads into span content attributes for the
+ * local trace viewer: prompt messages, the system prompt, responses, and
+ * tool arguments/results. Everything is capped so a giant payload cannot
+ * bloat a span segment past the reader's per-file limit, and message
+ * payloads stay valid JSON through every truncation path.
+ */
+
+/** Content attributes are capped so a giant payload cannot bloat a span segment. */
+export const CONTENT_ATTRIBUTE_LIMIT = 32 * 1024;
+import type { ContentSerializer } from "../core/types.js";
+
+export const aiSdkContentSerializer: ContentSerializer = {
+ json: contentAttribute,
+ text: textContentAttribute,
+ inputMessages: (messages) =>
+ genAiInputMessagesAttribute(messages, (value) =>
+ typeof value === "string" ? value : undefined,
+ ),
+ instructions: genAiSystemInstructionsAttribute,
+ toolDefinitions: genAiToolDefinitionsAttribute,
+ outputMessages: genAiOutputMessagesAttribute,
+ toolResults: toolResultsContentAttribute,
+};
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+const MARKER = "... [truncated]";
+
+export function telemetryByteLength(text: string): number {
+ return encoder.encode(text).length;
+}
+
+export function truncateTelemetryText(text: string, maxBytes: number): string {
+ const prefix = text.slice(0, maxBytes + 1);
+ const bytes = encoder.encode(prefix);
+ if (prefix.length === text.length && bytes.length <= maxBytes) return text;
+ const marker = MARKER.slice(0, maxBytes);
+ let end = Math.max(0, maxBytes - marker.length);
+ while (end > 0 && (bytes[end]! & 0xc0) === 0x80) end -= 1;
+ return decoder.decode(bytes.subarray(0, end)) + marker;
+}
+
+export function boundedPrincipalId(value: unknown): string | undefined {
+ return typeof value === "string" &&
+ value.length > 0 &&
+ value.length <= 1024 &&
+ telemetryByteLength(value) <= 1024
+ ? value
+ : undefined;
+}
+
+export function boundedTraceError(value: Error): Error {
+ const error = new Error(truncateTelemetryText(value.message, 4096));
+ error.name = truncateTelemetryText(value.name, 128);
+ error.stack = value.stack === undefined ? undefined : truncateTelemetryText(value.stack, 8192);
+ return error;
+}
+
+/** Serializes one payload value as capped JSON. */
+export function contentAttribute(value: unknown): string | undefined {
+ if (value === undefined) return undefined;
+ let json: string | undefined;
+ try {
+ json = JSON.stringify(value);
+ } catch {
+ return undefined;
+ }
+ if (json === undefined) return undefined;
+ return textContentAttribute(json);
+}
+
+/** Serializes model messages using the OpenTelemetry GenAI message schema. */
+export function genAiInputMessagesAttribute(
+ messages: unknown,
+ messageKind?: (value: unknown) => string | undefined,
+): string | undefined {
+ if (!Array.isArray(messages)) return undefined;
+ const formatted = messages.flatMap((message) => {
+ if (!isRecord(message) || message.role === "system" || typeof message.role !== "string") {
+ return [];
+ }
+ const parts = semanticParts(message.content);
+ if (parts.length === 0) return [];
+ const kind = message.role === "user" ? messageKind?.(message.kind) : undefined;
+ return [
+ kind === undefined ? { parts, role: message.role } : { kind, parts, role: message.role },
+ ];
+ });
+ for (let start = 0; start < formatted.length; start += 1) {
+ const json = semanticJsonAttribute(formatted.slice(start));
+ if (json !== undefined) return json;
+ }
+ return truncateSingleSemanticMessage(formatted) ?? semanticJsonAttribute([]);
+}
+
+/** Serializes the system prompt using the OpenTelemetry GenAI instruction schema. */
+export function genAiSystemInstructionsAttribute(instructions: unknown): string | undefined {
+ const text = systemPromptText(instructions);
+ if (text === undefined) return undefined;
+ return fitSemanticText(text, (content) => [{ content, type: "text" }]);
+}
+
+/** Serializes one model response using the OpenTelemetry GenAI message schema. */
+export function genAiOutputMessagesAttribute(
+ content: readonly unknown[],
+ finishReason: string,
+): string | undefined {
+ const parts = semanticParts(content);
+ const reason = finishReason === "tool-calls" ? "tool_call" : finishReason;
+ const message = (messageParts: unknown) => [
+ { finish_reason: reason, parts: messageParts, role: "assistant" },
+ ];
+ // An oversized response keeps a truncated text form, as an oversized prompt does.
+ return (
+ semanticJsonAttribute(message(parts)) ??
+ fitSemanticText(
+ semanticMessageText(parts),
+ (text) => message([{ content: text, type: "text" }]),
+ true,
+ )
+ );
+}
+
+/** Serializes memory records using the OpenTelemetry GenAI memory-records schema. */
+export function genAiMemoryRecordsAttribute(records: readonly unknown[]): string | undefined {
+ return semanticJsonAttribute(records);
+}
+
+/**
+ * Serializes provider-executed tool results, keeping the attribute valid
+ * JSON under the cap: when the full payload is too big, each entry's input
+ * and output collapse to capped text at progressively smaller budgets
+ * instead of cutting the JSON mid-string.
+ */
+export function toolResultsContentAttribute(
+ results: readonly Record<string, unknown>[],
+): string | undefined {
+ if (results.length === 0) return undefined;
+ const full = stringifyContent(results);
+ if (full !== undefined && full.length <= CONTENT_ATTRIBUTE_LIMIT) return full;
+ for (let cap = CONTENT_ATTRIBUTE_LIMIT; cap >= 0; cap = cap >= 256 ? Math.floor(cap / 2) : -1) {
+ const json = stringifyContent(results.map((entry) => cappedToolResult(entry, cap)));
+ if (json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT) return json;
+ }
+ return undefined;
+}
+
+function cappedToolResult(entry: Record<string, unknown>, cap: number): Record<string, unknown> {
+ const out: Record<string, unknown> = { toolName: entry.toolName };
+ for (const key of ["input", "output", "error"]) {
+ if (!(key in entry)) continue;
+ const value = entry[key];
+ const text = typeof value === "string" ? value : (stringifyContent(value) ?? "");
+ out[key] = text.length <= cap ? text : `${text.slice(0, cap)}… [truncated]`;
+ }
+ return out;
+}
+
+/** Serializes tool definitions, dropping trailing entries to keep the JSON whole. */
+export function genAiToolDefinitionsAttribute(tools: unknown): string | undefined {
+ if (!Array.isArray(tools)) return undefined;
+ const definitions = tools.flatMap((tool): Record<string, unknown>[] => {
+ if (!isRecord(tool) || typeof tool.name !== "string") return [];
+ const definition: Record<string, unknown> = { name: tool.name };
+ if (typeof tool.description === "string") definition.description = tool.description;
+ if (tool.inputSchema !== undefined) definition.parameters = tool.inputSchema;
+ return [definition];
+ });
+ if (definitions.length === 0) return undefined;
+
+ const serialized: string[] = [];
+ let length = 2;
+ for (const definition of definitions) {
+ const json = stringifyContent(definition);
+ if (json === undefined) break;
+ const nextLength = length + (serialized.length === 0 ? 0 : 1) + json.length;
+ if (nextLength > CONTENT_ATTRIBUTE_LIMIT) break;
+ serialized.push(json);
+ length = nextLength;
+ }
+ return `[${serialized.join(",")}]`;
+}
+
+/** Normalizes the AI SDK's `instructions` prompt to plain text. */
+function systemPromptText(instructions: unknown): string | undefined {
+ if (typeof instructions === "string") return instructions.length === 0 ? undefined : instructions;
+ if (!isRecord(instructions) && !Array.isArray(instructions)) return undefined;
+ const messages = Array.isArray(instructions) ? instructions : [instructions];
+ const texts: string[] = [];
+ for (const message of messages) {
+ if (!isRecord(message)) continue;
+ if (typeof message.content === "string") texts.push(message.content);
+ else if (Array.isArray(message.content))
+ for (const part of message.content)
+ if (isRecord(part) && part.type === "text" && typeof part.text === "string")
+ texts.push(part.text);
+ }
+ const joined = texts.join("\n\n").trim();
+ return joined.length === 0 ? undefined : joined;
+}
+
+/** Caps plain text, marking the cut. */
+export function textContentAttribute(text: string): string | undefined {
+ if (text.length === 0) return undefined;
+ return text.length <= CONTENT_ATTRIBUTE_LIMIT
+ ? text
+ : `${text.slice(0, CONTENT_ATTRIBUTE_LIMIT)}… [truncated]`;
+}
+
+function stringifyContent(value: unknown): string | undefined {
+ try {
+ return JSON.stringify(value);
+ } catch {
+ return undefined;
+ }
+}
+
+function semanticJsonAttribute(value: unknown): string | undefined {
+ const json = stringifyContent(value);
+ return json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT ? json : undefined;
+}
+
+function truncateSingleSemanticMessage(
+ messages: readonly Record<string, unknown>[],
+): string | undefined {
+ const message = messages.at(-1);
+ if (message === undefined || typeof message.role !== "string") return undefined;
+ const text = semanticMessageText(message.parts);
+ const base =
+ typeof message.kind === "string"
+ ? { kind: message.kind, role: message.role }
+ : { role: message.role };
+ return fitSemanticText(
+ text,
+ (content) => [{ ...base, parts: [{ content, type: "text" }] }],
+ true,
+ );
+}
+
+function fitSemanticText(
+ text: string,
+ wrap: (content: string) => unknown,
+ alwaysMark = false,
+): string | undefined {
+ if (!alwaysMark && text.length <= CONTENT_ATTRIBUTE_LIMIT) {
+ const full = semanticJsonAttribute(wrap(text));
+ if (full !== undefined) return full;
+ }
+ for (let length = Math.min(text.length, CONTENT_ATTRIBUTE_LIMIT); length > 0; length -= 256) {
+ const json = semanticJsonAttribute(wrap(`${text.slice(0, length)}… [truncated]`));
+ if (json !== undefined) return json;
+ }
+ return semanticJsonAttribute(wrap("… [truncated]"));
+}
+
+function semanticMessageText(parts: unknown): string {
+ if (!Array.isArray(parts)) return "";
+ let text = "";
+ for (const part of parts) {
+ if (!isRecord(part) || part.type !== "text" || typeof part.content !== "string") continue;
+ const separator = text.length === 0 ? "" : "\n";
+ const remaining = CONTENT_ATTRIBUTE_LIMIT + 1 - text.length - separator.length;
+ if (remaining <= 0) break;
+ text += separator + part.content.slice(0, remaining);
+ }
+ return text;
+}
+
+function semanticParts(content: unknown): Record<string, unknown>[] {
+ if (typeof content === "string") return [{ content, type: "text" }];
+ if (!Array.isArray(content)) return [];
+ const parts: Record<string, unknown>[] = [];
+ for (const part of content) {
+ const formatted = semanticPart(part);
+ if (formatted !== undefined) parts.push(formatted);
+ }
+ return parts;
+}
+
+function semanticPart(value: unknown): Record<string, unknown> | undefined {
+ if (!isRecord(value) || typeof value.type !== "string") return undefined;
+ switch (value.type) {
+ case "text":
+ case "reasoning": {
+ const content = typeof value.text === "string" ? value.text : value.content;
+ return typeof content === "string" ? { content, type: value.type } : undefined;
+ }
+ case "tool-call":
+ return {
+ arguments: value.input,
+ id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,
+ name: value.toolName,
+ type: "tool_call",
+ };
+ case "tool-result":
+ return {
+ id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,
+ response: toolResponse(value.output),
+ type: "tool_call_response",
+ };
+ case "tool-error":
+ return {
+ id: stringValue(value.toolCallId) ?? stringValue(value.callId) ?? null,
+ response: value.error instanceof Error ? value.error.message : value.error,
+ type: "tool_call_response",
+ };
+ default:
+ return { type: value.type };
+ }
+}
+
+function toolResponse(output: unknown): unknown {
+ if (!isRecord(output)) return output;
+ if (
+ (output.type === "text" ||
+ output.type === "error-text" ||
+ output.type === "json" ||
+ output.type === "error-json") &&
+ "value" in output
+ ) {
+ return output.value;
+ }
+ if (output.type === "execution-denied") {
+ return { denied: true, reason: output.reason };
+ }
+ return output;
+}
+
+function stringValue(value: unknown): string | undefined {
+ return typeof value === "string" ? value : undefined;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
diff --git a/packages/tracing/src/agent-tracing.ts b/packages/tracing/src/agent-tracing.ts
new file mode 100644
index 000000000..30a36d8b5
--- /dev/null
+++ b/packages/tracing/src/agent-tracing.ts
@@ -0,0 +1,339 @@
+import { randomUUID } from "node:crypto";
+import { otelTelemetry } from "./adapters/otel.js";
+import { aiSdkContentSerializer } from "./adapters/serialization.js";
+import { createScopes, type LiveOperation } from "./core/scopes.js";
+import { currentAgentHandoff } from "./core/delegation.js";
+import { intersectCapture } from "./core/activation.js";
+import { traceSnapshot } from "./core/snapshot.js";
+import { operationHandle, wrappedOperation, type TurnOperation } from "./operations.js";
+import type {
+ AgentTelemetry,
+ Attributes,
+ CaptureDecision,
+ ContentSerializer,
+ ExecutionContext,
+ FrameworkIdentity,
+ RunIdentity,
+ ScopeRecord,
+ TraceCheckpointer,
+ TraceErrorHandler,
+ TraceLink,
+ TraceReference,
+} from "./core/types.js";
+
+export interface AgentTracingOptions {
+ /** Span output and context propagation. Defaults to the global OpenTelemetry provider. */
+ readonly telemetry?: AgentTelemetry;
+ /**
+ * Makes turns durable. Each in-flight turn is saved under a library-chosen
+ * key and removed when the turn and its open tool calls complete. Calling
+ * `turn()` again with the same identity, in this or a later process,
+ * continues the saved turn.
+ */
+ readonly checkpointer?: TraceCheckpointer;
+ /** Defaults to the AI SDK content serializer. */
+ readonly serializer?: ContentSerializer;
+ readonly onError?: TraceErrorHandler;
+}
+
+export interface TurnInput {
+ readonly identity: RunIdentity;
+ /** Names the invocation span. Omit for an anonymous agent. */
+ readonly agentName?: string;
+ readonly sequence: number;
+ readonly framework?: FrameworkIdentity;
+ readonly capture?: CaptureDecision;
+ readonly attributes?: Attributes;
+ /**
+ * Adopts a trace and span ID reserved before the turn started, such as one
+ * already returned to a caller. Requires telemetry with stable `ids`.
+ */
+ readonly reference?: TraceReference;
+ /**
+ * Nests the turn under a caller's span in the caller's trace. A local
+ * handoff sets this for the first turn; remote callees start their own trace.
+ */
+ readonly parent?: TraceReference;
+ readonly links?: readonly TraceLink[];
+ readonly startTimeMs?: number;
+ /** Delegation lineage recorded by the host, when it is not carried by a handoff. */
+ readonly lineage?: {
+ readonly parentRunId?: string;
+ readonly parentCallId?: string;
+ readonly agentName?: string;
+ };
+ readonly channel?: { readonly kind?: string; readonly origin?: string };
+ /** Host context for spans this process starts, such as OpenTelemetry baggage. */
+ readonly context?: ExecutionContext;
+}
+
+export interface ResumeInput {
+ readonly identity: RunIdentity;
+ readonly context?: ExecutionContext;
+}
+
+interface MemoryInput<T> {
+ identity: RunIdentity;
+ storeId: string;
+ slot: string;
+ phase: string;
+ capture?: CaptureDecision;
+ /** Defaults to the active operation. */
+ parent?: TraceReference;
+ context?: ExecutionContext;
+ describe?: (value: T) => {
+ recordCount?: number;
+ records?: readonly { id?: string; content: string }[];
+ };
+}
+
+export interface AgentTracing {
+ turn: import("./operations.js").WrappedOperation<TurnInput, TurnOperation>;
+ /** Returns a saved durable turn, without starting one. */
+ resume(input: ResumeInput): Promise<TurnOperation | undefined>;
+ memory: AgentMemoryTracing;
+ forceFlush(): Promise<void>;
+ shutdown(): Promise<void>;
+}
+export interface AgentMemoryTracing {
+ search<T>(data: MemoryInput<T>, execute: () => T | PromiseLike<T>): Promise<T>;
+ write<T>(data: MemoryInput<T>, execute: () => T | PromiseLike<T>): Promise<T>;
+}
+
+const METADATA_ONLY: CaptureDecision = { emit: true, recordInputs: false, recordOutputs: false };
+const LIVE_TURNS = 1000;
+
+interface LiveTurn {
+ readonly root: LiveOperation;
+ /** Token of the last checkpoint this process wrote. */
+ revision: string | undefined;
+ writes: number;
+}
+
+export function createAgentTracing(input: AgentTracingOptions): AgentTracing {
+ const telemetry = input.telemetry ?? otelTelemetry();
+ const checkpointer = input.checkpointer;
+ if (checkpointer !== undefined && telemetry.ids === undefined)
+ throw new Error(
+ "Durable agent tracing needs stable span IDs. Pass the AgentSpanIdGenerator installed on your tracer provider to otelTelemetry({ idGenerator }).",
+ );
+ const report: TraceErrorHandler = (error, context) => {
+ try {
+ input.onError?.(error, context);
+ } catch {}
+ };
+ const scopes = createScopes({
+ telemetry,
+ serializer: input.serializer ?? aiSdkContentSerializer,
+ durable: checkpointer !== undefined,
+ onError: input.onError,
+ });
+ // Turns hydrated in this process, so concurrent hooks share one tree. The
+ // checkpoint stays authoritative: another process's write replaces the entry.
+ const live = new Map<string, Promise<LiveTurn | undefined>>();
+
+ function turnKey(identity: RunIdentity): string {
+ if (checkpointer === undefined) return `turn:${randomUUID()}`;
+ return ["turn", identity.runId, identity.turnId].map(encodeURIComponent).join(":");
+ }
+
+ /** Writes are serialized so a slow store never persists an older tree last. */
+ function persistence(key: string, entry: () => LiveTurn | undefined) {
+ let queue = Promise.resolve();
+ return () => {
+ const turn = entry();
+ if (checkpointer === undefined || turn === undefined) return queue;
+ turn.writes++;
+ queue = queue.then(async () => {
+ try {
+ if (turn.root.settled) {
+ live.delete(key);
+ await checkpointer.delete(key);
+ } else {
+ const revision = randomUUID();
+ await checkpointer.set(key, traceSnapshot({ ...turn.root.record(), revision }));
+ turn.revision = revision;
+ }
+ } catch (error) {
+ report(error, { phase: "checkpoint", operation: "activation" });
+ } finally {
+ turn.writes--;
+ }
+ });
+ return queue;
+ };
+ }
+
+ async function loadCheckpoint(key: string): Promise<unknown> {
+ try {
+ return await checkpointer!.get(key);
+ } catch (error) {
+ report(error, { phase: "restore", operation: "activation" });
+ return undefined;
+ }
+ }
+
+ async function hydrate(
+ key: string,
+ cached: LiveTurn | undefined,
+ open: ((onChange: () => Promise<void>) => Promise<LiveOperation>) | undefined,
+ context: ExecutionContext | undefined,
+ capture?: CaptureDecision,
+ ): Promise<LiveTurn | undefined> {
+ const saved = checkpointer === undefined ? undefined : await loadCheckpoint(key);
+ if (
+ cached !== undefined &&
+ (cached.writes > 0 || (saved as ScopeRecord | undefined)?.revision === cached.revision)
+ ) {
+ if (context !== undefined) cached.root.useContext(context);
+ // A re-entry may decline content the cached turn was allowed to record.
+ if (capture !== undefined) cached.root.narrow(capture);
+ return cached;
+ }
+ let entry: LiveTurn | undefined;
+ const onChange = persistence(key, () => entry);
+ const restored =
+ saved === undefined ? undefined : await scopes.restore(saved, { capture, context, onChange });
+ const root = restored ?? (await open?.(onChange));
+ if (root === undefined) return undefined;
+ entry = { root, revision: (saved as ScopeRecord | undefined)?.revision, writes: 0 };
+ if (restored === undefined) await onChange();
+ return entry;
+ }
+
+ function cache(
+ key: string,
+ load: (cached: LiveTurn | undefined) => Promise<LiveTurn | undefined>,
+ ): Promise<LiveTurn | undefined> {
+ const previous = live.get(key);
+ const loading = (async () =>
+ load(previous === undefined ? undefined : await previous.catch(() => undefined)))();
+ live.set(key, loading);
+ void loading.then(
+ (turn) => {
+ if (turn === undefined || turn.root.settled) live.delete(key);
+ },
+ () => live.delete(key),
+ );
+ if (live.size > LIVE_TURNS) live.delete(live.keys().next().value!);
+ return loading;
+ }
+
+ function startTurn(data: TurnInput, key: string) {
+ return (onChange: () => Promise<void>) => {
+ const handoff = currentAgentHandoff();
+ const capture = intersectCapture(data.capture ?? METADATA_ONLY, handoff?.capture);
+ const first = handoff !== undefined && data.sequence === 0;
+ const parent =
+ data.parent ?? (first && handoff?.remote !== true ? handoff?.caller : undefined);
+ const caller =
+ first && handoff?.remote === true
+ ? [{ relationship: "agent.dispatch", context: handoff.caller }]
+ : [];
+ const links = [...caller, ...(data.links ?? [])];
+ return scopes.start({
+ identity: {
+ ...data.identity,
+ conversationId: handoff?.conversationId ?? data.identity.conversationId,
+ agentName: data.agentName,
+ framework: data.framework,
+ },
+ key,
+ capture,
+ parent,
+ reference: telemetry.ids === undefined ? undefined : data.reference,
+ startTimeMs: data.startTimeMs,
+ data: {
+ type: "activation",
+ options: {
+ sequence: data.sequence,
+ attributes: data.attributes,
+ channel: data.channel,
+ subagent: handoff !== undefined || data.lineage !== undefined,
+ subagentName: handoff?.agentName ?? data.lineage?.agentName,
+ parentRunId: handoff?.parentRunId ?? data.lineage?.parentRunId,
+ parentCallId: handoff?.parentCallId ?? data.lineage?.parentCallId,
+ },
+ },
+ links: links.length === 0 ? undefined : links,
+ context: data.context,
+ onChange,
+ });
+ };
+ }
+
+ function handle(root: LiveOperation): TurnOperation {
+ return operationHandle(
+ root,
+ { type: "activation", options: { sequence: 0 } },
+ input.onError,
+ ) as TurnOperation;
+ }
+
+ async function turnOperation(data: TurnInput): Promise<TurnOperation> {
+ const key = turnKey(data.identity);
+ const turn =
+ checkpointer === undefined
+ ? await hydrate(key, undefined, startTurn(data, key), data.context)
+ : await cache(key, (cached) =>
+ hydrate(key, cached, startTurn(data, key), data.context, data.capture),
+ );
+ return handle(turn!.root);
+ }
+
+ async function resume(data: ResumeInput): Promise<TurnOperation | undefined> {
+ if (checkpointer === undefined) return undefined;
+ const key = turnKey(data.identity);
+ const turn = await cache(key, (cached) => hydrate(key, cached, undefined, data.context));
+ return turn === undefined ? undefined : handle(turn.root);
+ }
+
+ async function lifecycle(callback: () => Promise<void>) {
+ try {
+ await callback();
+ } catch (error) {
+ report(error, { phase: "complete" });
+ }
+ }
+
+ async function memory<T>(
+ operation: "search_memory" | "upsert_memory",
+ data: MemoryInput<T>,
+ execute: () => T | PromiseLike<T>,
+ ): Promise<T> {
+ const active = telemetry.active();
+ const handle = await scopes.start({
+ identity: data.identity,
+ key: `memory:${randomUUID()}`,
+ capture: data.capture ?? active?.capture ?? METADATA_ONLY,
+ parent: data.parent ?? active?.reference,
+ context: data.context,
+ data: {
+ type: "memory",
+ options: { operation, phase: data.phase, slot: data.slot, storeId: data.storeId },
+ },
+ });
+ let value: T;
+ try {
+ value = await handle.run(execute);
+ } catch (error) {
+ await handle.fail(error).catch((tracingError) => report(tracingError, { phase: "complete" }));
+ throw error;
+ }
+ await handle
+ .complete({ outcome: "completed", ...data.describe?.(value) })
+ .catch((error) => report(error, { phase: "complete" }));
+ return value;
+ }
+
+ return {
+ turn: wrappedOperation(turnOperation, undefined, input.onError),
+ resume,
+ memory: {
+ search: (data, execute) => memory("search_memory", data, execute),
+ write: (data, execute) => memory("upsert_memory", data, execute),
+ },
+ forceFlush: () => lifecycle(() => telemetry.forceFlush()),
+ shutdown: () => lifecycle(() => telemetry.shutdown()),
+ };
+}
diff --git a/packages/tracing/src/ai-sdk.ts b/packages/tracing/src/ai-sdk.ts
new file mode 100644
index 000000000..9715b43af
--- /dev/null
+++ b/packages/tracing/src/ai-sdk.ts
@@ -0,0 +1,7 @@
+/**
+ * AI SDK integration. `aiSdkTelemetry` records `generateText`, `streamText`,
+ * and AI SDK agents as turns; `modelUsage` and `modelContent` convert AI SDK
+ * payloads for hosts that record model calls themselves.
+ */
+export { aiSdkTelemetry, type AiSdkTelemetryOptions } from "./adapters/ai-sdk-telemetry.js";
+export { modelUsage, modelContent } from "./adapters/ai-sdk-payload.js";
diff --git a/packages/tracing/src/capture.ts b/packages/tracing/src/capture.ts
new file mode 100644
index 000000000..ff32a34ad
--- /dev/null
+++ b/packages/tracing/src/capture.ts
@@ -0,0 +1,13 @@
+import { context, createContextKey } from "@opentelemetry/api";
+import type { CaptureDecision } from "./core/types.js";
+
+const CAPTURE = createContextKey("agent.tracing.capture");
+export function currentCapture(): CaptureDecision | undefined {
+ return context.active().getValue(CAPTURE) as CaptureDecision | undefined;
+}
+export function withCapture<T extends { setValue(key: symbol, value: unknown): T }>(
+ host: T,
+ capture: CaptureDecision,
+): T {
+ return host.setValue(CAPTURE, capture);
+}
diff --git a/packages/tracing/src/core/activation.ts b/packages/tracing/src/core/activation.ts
new file mode 100644
index 000000000..ba8c23771
--- /dev/null
+++ b/packages/tracing/src/core/activation.ts
@@ -0,0 +1,42 @@
+import type { ActiveOperation, CaptureDecision } from "./types.js";
+
+export function intersectCapture(
+ capture: CaptureDecision,
+ ceiling?: CaptureDecision,
+): CaptureDecision {
+ const emit = capture.emit && (ceiling?.emit ?? true);
+ return {
+ emit,
+ recordInputs: emit && capture.recordInputs && (ceiling?.recordInputs ?? true),
+ recordOutputs: emit && capture.recordOutputs && (ceiling?.recordOutputs ?? true),
+ };
+}
+
+export function activeOperation(
+ operation: ActiveOperation,
+ capture: CaptureDecision,
+): ActiveOperation {
+ const mcp = operation.mcp;
+ return {
+ type: operation.type,
+ reference: operation.reference,
+ capture,
+ mcp:
+ mcp === undefined
+ ? undefined
+ : {
+ update: (input) => mcp.update(input),
+ error: (error, type) =>
+ mcp.error(
+ capture.recordOutputs ? error : undefined,
+ type ?? (error instanceof Error ? error.name : undefined),
+ ),
+ arguments: (value) => {
+ if (capture.recordInputs) mcp.arguments(value);
+ },
+ result: (value) => {
+ if (capture.recordOutputs) mcp.result(value);
+ },
+ },
+ };
+}
diff --git a/packages/tracing/src/core/attributes.test.ts b/packages/tracing/src/core/attributes.test.ts
new file mode 100644
index 000000000..5ba8f7097
--- /dev/null
+++ b/packages/tracing/src/core/attributes.test.ts
@@ -0,0 +1,14 @@
+import { describe, expect, it } from "vitest";
+import { runtimeContextAttributes } from "./attributes.js";
+
+describe("runtime context projection", () => {
+ it("retains shared objects under each path while stopping cycles", () => {
+ const user = { id: "alice" };
+ const input: Record<string, unknown> = { requester: user, assignee: user };
+ input.self = input;
+ expect(runtimeContextAttributes(input)).toEqual({
+ "ai.settings.context.requester.id": "alice",
+ "ai.settings.context.assignee.id": "alice",
+ });
+ });
+});
diff --git a/packages/tracing/src/core/attributes.ts b/packages/tracing/src/core/attributes.ts
new file mode 100644
index 000000000..d6f5df562
--- /dev/null
+++ b/packages/tracing/src/core/attributes.ts
@@ -0,0 +1,85 @@
+import type { AttributeValue, Attributes, FrameworkIdentity, RunIdentity, Usage } from "./types.js";
+const TRACE_SCHEMA_VERSION = 4;
+export const USAGE_FIELDS = {
+ costUsd: "agent.usage.cost_usd",
+ inputTokens: "agent.usage.input_tokens",
+ outputTokens: "agent.usage.output_tokens",
+ cacheReadTokens: "agent.usage.cache_read_tokens",
+ cacheWriteTokens: "agent.usage.cache_write_tokens",
+} as const;
+
+export function identityAttributes(
+ identity: Pick<RunIdentity, "runId" | "conversationId">,
+): Attributes {
+ return {
+ "agent.run.id": identity.runId,
+ "agent.trace.schema.version": TRACE_SCHEMA_VERSION,
+ "gen_ai.conversation.id": identity.conversationId,
+ };
+}
+
+export function namingAttributes(name: string, operation = name): Attributes {
+ return { "operation.name": operation, "resource.name": name };
+}
+
+export function frameworkAttributes(framework?: FrameworkIdentity): Attributes {
+ return {
+ "agent.framework.name": framework?.name || undefined,
+ "agent.framework.version": framework?.version || undefined,
+ };
+}
+
+/** Invocation and model spans use GenAI token counts; structural spans use eve's. */
+export function usageAttributes(usage: Usage, genAi = false): Attributes {
+ if (genAi)
+ return {
+ [USAGE_FIELDS.costUsd]: usage.costUsd,
+ "gen_ai.usage.input_tokens": usage.inputTokens,
+ "gen_ai.usage.output_tokens": usage.outputTokens,
+ "gen_ai.usage.cache_read.input_tokens": usage.inputTokenDetails?.cacheReadTokens,
+ "gen_ai.usage.cache_write.input_tokens": usage.inputTokenDetails?.cacheWriteTokens,
+ };
+ return {
+ [USAGE_FIELDS.costUsd]: usage.costUsd,
+ [USAGE_FIELDS.inputTokens]: usage.inputTokens,
+ [USAGE_FIELDS.outputTokens]: usage.outputTokens,
+ [USAGE_FIELDS.cacheReadTokens]: usage.inputTokenDetails?.cacheReadTokens,
+ [USAGE_FIELDS.cacheWriteTokens]: usage.inputTokenDetails?.cacheWriteTokens,
+ };
+}
+
+export function runtimeContextAttributes(
+ input: Readonly<Record<string, unknown>> | undefined,
+): Attributes {
+ const attributes: Record<string, Exclude<Attributes[string], undefined>> = {};
+ const seen = new WeakSet<object>();
+ let remaining = 256;
+ function visit(key: string, value: unknown, depth: number): void {
+ if (remaining <= 0 || depth > 8 || value == null) return;
+ if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+ attributes[key] = value;
+ remaining -= 1;
+ } else if (Array.isArray(value)) {
+ if (
+ value.length <= 256 &&
+ value.every(
+ (item) =>
+ typeof item === "string" || typeof item === "number" || typeof item === "boolean",
+ ) &&
+ new Set(value.map((item) => typeof item)).size === 1
+ ) {
+ attributes[key] = value as AttributeValue;
+ remaining -= 1;
+ }
+ } else if (typeof value === "object" && !seen.has(value)) {
+ seen.add(value);
+ for (const keyPart of Object.keys(value)) {
+ if (remaining <= 0) break;
+ visit(`${key}.${keyPart}`, Reflect.get(value, keyPart), depth + 1);
+ }
+ seen.delete(value);
+ }
+ }
+ if (input !== undefined) visit("ai.settings.context", input, 0);
+ return attributes;
+}
diff --git a/packages/tracing/src/core/content-policy.test.ts b/packages/tracing/src/core/content-policy.test.ts
new file mode 100644
index 000000000..176831feb
--- /dev/null
+++ b/packages/tracing/src/core/content-policy.test.ts
@@ -0,0 +1,125 @@
+import { describe, expect, it } from "vitest";
+
+import { withoutDeclinedContent } from "./content-policy.js";
+
+const ATTRIBUTES = {
+ "agent.channel.delivery.input": '{"message":"private"}',
+ "agent.session.title": "Private title",
+ "agent.trace.content.input": true,
+ "agent.trace.content.output": true,
+ "gen_ai.input.messages": "what the user said",
+ "gen_ai.system_instructions": '[{"content":"private instructions","type":"text"}]',
+ "ai.response.finish_reason": "stop",
+ "ai.response.text": "what the model said",
+ "gen_ai.request.model": "test-model",
+ "gen_ai.tool.call.arguments": "{}",
+ "gen_ai.tool.call.result": "42",
+ "gen_ai.tool.name": "weather",
+};
+
+describe("withoutDeclinedContent", () => {
+ it("keeps everything when the destination declined nothing", () => {
+ expect(
+ withoutDeclinedContent(ATTRIBUTES, { recordInputs: true, recordOutputs: true }),
+ ).toBeUndefined();
+ });
+
+ it("keeps everything when the span carries none of what was declined", () => {
+ expect(
+ withoutDeclinedContent(
+ { "gen_ai.request.model": "test-model" },
+ { recordInputs: false, recordOutputs: false },
+ ),
+ ).toBeUndefined();
+ });
+
+ it("drops inputs alone", () => {
+ expect(
+ withoutDeclinedContent(ATTRIBUTES, { recordInputs: false, recordOutputs: true }),
+ ).toEqual({
+ "agent.trace.content.input": false,
+ "agent.trace.content.output": true,
+ "ai.response.finish_reason": "stop",
+ "ai.response.text": "what the model said",
+ "gen_ai.request.model": "test-model",
+ "gen_ai.tool.call.result": "42",
+ "gen_ai.tool.name": "weather",
+ });
+ });
+
+ it("drops outputs alone", () => {
+ expect(
+ withoutDeclinedContent(ATTRIBUTES, { recordInputs: true, recordOutputs: false }),
+ ).toEqual({
+ "agent.channel.delivery.input": '{"message":"private"}',
+ "agent.session.title": "Private title",
+ "agent.trace.content.input": true,
+ "agent.trace.content.output": false,
+ "gen_ai.input.messages": "what the user said",
+ "gen_ai.system_instructions": '[{"content":"private instructions","type":"text"}]',
+ "ai.response.finish_reason": "stop",
+ "gen_ai.request.model": "test-model",
+ "gen_ai.tool.call.arguments": "{}",
+ "gen_ai.tool.name": "weather",
+ });
+ });
+
+ it("drops system instructions when the destination declined inputs", () => {
+ expect(
+ withoutDeclinedContent(
+ { "gen_ai.system_instructions": '[{"content":"private","type":"text"}]' },
+ { recordInputs: false, recordOutputs: true },
+ ),
+ ).toEqual({});
+ });
+
+ // The prefixes are shared: `ai.response.finish_reason` and `gen_ai.tool.name`
+ // say what happened rather than what was said, so declining content cannot
+ // cost a destination the ability to read its own traces.
+ it("keeps metadata that shares a prefix with content", () => {
+ expect(
+ withoutDeclinedContent(ATTRIBUTES, { recordInputs: false, recordOutputs: false }),
+ ).toEqual({
+ "agent.trace.content.input": false,
+ "agent.trace.content.output": false,
+ "ai.response.finish_reason": "stop",
+ "gen_ai.request.model": "test-model",
+ "gen_ai.tool.name": "weather",
+ });
+ });
+
+ it("leaves the attributes it was handed alone", () => {
+ const attributes = { ...ATTRIBUTES };
+ withoutDeclinedContent(attributes, { recordInputs: false, recordOutputs: false });
+ expect(attributes).toEqual(ATTRIBUTES);
+ });
+
+ it("narrows content policy attributes even when no content is present", () => {
+ expect(
+ withoutDeclinedContent(
+ {
+ "agent.trace.content.input": true,
+ "agent.trace.content.output": true,
+ },
+ { recordInputs: false, recordOutputs: true },
+ ),
+ ).toEqual({
+ "agent.trace.content.input": false,
+ "agent.trace.content.output": true,
+ });
+ });
+
+ it("redacts recalled memory records as input content", () => {
+ const searched = {
+ "gen_ai.memory.records": '[{"content":"Private preference"}]',
+ "gen_ai.operation.name": "search_memory",
+ };
+
+ expect(withoutDeclinedContent(searched, { recordInputs: false, recordOutputs: true })).toEqual({
+ "gen_ai.operation.name": "search_memory",
+ });
+ expect(
+ withoutDeclinedContent(searched, { recordInputs: true, recordOutputs: false }),
+ ).toBeUndefined();
+ });
+});
diff --git a/packages/tracing/src/core/content-policy.ts b/packages/tracing/src/core/content-policy.ts
new file mode 100644
index 000000000..0925d20d9
--- /dev/null
+++ b/packages/tracing/src/core/content-policy.ts
@@ -0,0 +1,116 @@
+/**
+ * Which span attributes carry conversation content, and in which direction.
+ *
+ * Two vocabularies land on the same spans: the ones the library sets on its own
+ * `agent.*` spans, and the ones the AI SDK's OpenTelemetry integration sets on
+ * the model-call spans beneath them. A destination that declined content has to
+ * be rid of both, so both are listed here rather than in the module that writes
+ * each.
+ *
+ * Listed by name rather than by prefix because the prefixes are shared with
+ * metadata that must survive: `ai.response.finish_reason` and
+ * `gen_ai.tool.name` say what happened, not what was said. The cost is that a
+ * new content attribute in a dependency is not covered until it is added here.
+ */
+
+/** Prompts, instructions, tool arguments — what went in. */
+const INPUT_CONTENT_ATTRIBUTES: ReadonlySet<string> = new Set([
+ "agent.channel.delivery.input",
+ "agent.session.title",
+ "agent.approval.request",
+ "ai.documents",
+ "ai.prompt",
+ "ai.prompt.toolChoice",
+ "ai.prompt.tools",
+ "ai.value",
+ "ai.values",
+ "gen_ai.input.messages",
+ "gen_ai.memory.records",
+ "gen_ai.system_instructions",
+ "gen_ai.tool.call.arguments",
+ "gen_ai.tool.definitions",
+]);
+
+/**
+ * Responses, reasoning, tool results — what came out.
+ *
+ * `ai.toolCall.args` is here rather than above because the AI SDK gates it on
+ * `recordOutputs`; matching that is what keeps one destination's view coherent.
+ */
+const OUTPUT_CONTENT_ATTRIBUTES: ReadonlySet<string> = new Set([
+ "agent.approval.response",
+ "ai.embedding",
+ "ai.embeddings",
+ "ai.ranking",
+ "ai.response.files",
+ "ai.response.object",
+ "ai.response.reasoning",
+ "ai.response.text",
+ "ai.response.toolCalls",
+ "ai.response.tool_calls",
+ "ai.response.tool_results",
+ "ai.toolCall.args",
+ "ai.toolCall.result",
+ "gen_ai.output.messages",
+ "gen_ai.tool.call.result",
+]);
+
+const INPUT_POLICY_ATTRIBUTE = "agent.trace.content.input";
+const OUTPUT_POLICY_ATTRIBUTE = "agent.trace.content.output";
+
+/** What one destination is willing to receive. */
+export interface ResolvedContentOptions {
+ readonly recordInputs: boolean;
+ readonly recordOutputs: boolean;
+}
+
+function isDeclined(key: string, content: ResolvedContentOptions): boolean {
+ if (!content.recordInputs && INPUT_CONTENT_ATTRIBUTES.has(key)) return true;
+ return !content.recordOutputs && OUTPUT_CONTENT_ATTRIBUTES.has(key);
+}
+
+function narrowsContentPolicy(
+ attributes: Readonly<Record<string, unknown>>,
+ content: ResolvedContentOptions,
+): boolean {
+ return (
+ (!content.recordInputs && attributes[INPUT_POLICY_ATTRIBUTE] === true) ||
+ (!content.recordOutputs && attributes[OUTPUT_POLICY_ATTRIBUTE] === true)
+ );
+}
+
+function exportedAttributeValue(
+ key: string,
+ value: unknown,
+ content: ResolvedContentOptions,
+): unknown {
+ if (key === INPUT_POLICY_ATTRIBUTE && value === true && !content.recordInputs) return false;
+ if (key === OUTPUT_POLICY_ATTRIBUTE && value === true && !content.recordOutputs) return false;
+ return value;
+}
+
+/**
+ * The attributes with the declined content removed, or `undefined` when there
+ * was none to remove.
+ *
+ * The `undefined` return is what lets the caller forward the original span
+ * untouched in the common case, so a destination that declined a direction the
+ * span never carried costs one pass over its keys and no allocation.
+ */
+export function withoutDeclinedContent(
+ attributes: Readonly<Record<string, unknown>>,
+ content: ResolvedContentOptions,
+): Record<string, unknown> | undefined {
+ const keys = Object.keys(attributes);
+ if (!keys.some((key) => isDeclined(key, content)) && !narrowsContentPolicy(attributes, content)) {
+ return undefined;
+ }
+
+ const kept: Record<string, unknown> = {};
+ for (const key of keys) {
+ if (!isDeclined(key, content)) {
+ kept[key] = exportedAttributeValue(key, attributes[key], content);
+ }
+ }
+ return kept;
+}
diff --git a/packages/tracing/src/core/context.ts b/packages/tracing/src/core/context.ts
new file mode 100644
index 000000000..b353ffdf3
--- /dev/null
+++ b/packages/tracing/src/core/context.ts
@@ -0,0 +1,34 @@
+import type { ActiveOperation, AgentTelemetry, ExecutionContext } from "./types.js";
+
+/** Runs `execute` exactly once, even when the telemetry cannot enter the context. */
+export function runTraceContext<T>(
+ telemetry: Pick<AgentTelemetry, "run" | "suppressed">,
+ operation: ActiveOperation,
+ execute: () => T,
+ host?: ExecutionContext,
+): T {
+ let entered = false;
+ try {
+ return telemetry.run(
+ operation,
+ () => {
+ entered = true;
+ return execute();
+ },
+ host,
+ );
+ } catch (error) {
+ if (entered) throw error;
+ if (telemetry.suppressed !== undefined) {
+ try {
+ return telemetry.suppressed(() => {
+ entered = true;
+ return execute();
+ });
+ } catch (error) {
+ if (entered) throw error;
+ }
+ }
+ return execute();
+ }
+}
diff --git a/packages/tracing/src/core/delegation.ts b/packages/tracing/src/core/delegation.ts
new file mode 100644
index 000000000..a2b9902b5
--- /dev/null
+++ b/packages/tracing/src/core/delegation.ts
@@ -0,0 +1,20 @@
+import { context, createContextKey } from "@opentelemetry/api";
+import type { CaptureDecision, TraceReference } from "./types.js";
+
+export interface AgentHandoff {
+ readonly caller: TraceReference;
+ readonly conversationId: string;
+ readonly parentRunId: string;
+ readonly parentCallId: string;
+ readonly agentName: string;
+ readonly capture: CaptureDecision;
+ /** Set when the handoff crossed a process boundary; the callee then starts its own trace. */
+ readonly remote?: boolean;
+}
+
+const HANDOFF = createContextKey("agent.tracing.handoff");
+export const currentAgentHandoff = () =>
+ context.active().getValue(HANDOFF) as AgentHandoff | undefined;
+export function withAgentHandoff<T>(handoff: AgentHandoff, execute: () => T): T {
+ return context.with(context.active().setValue(HANDOFF, handoff), execute);
+}
diff --git a/packages/tracing/src/core/mcp.ts b/packages/tracing/src/core/mcp.ts
new file mode 100644
index 000000000..ddc4c215e
--- /dev/null
+++ b/packages/tracing/src/core/mcp.ts
@@ -0,0 +1,60 @@
+import type { Attributes } from "./types.js";
+import type { ContentSerializer } from "./types.js";
+
+export interface McpUpdate {
+ readonly connectionName?: string;
+ readonly method?: string;
+ readonly toolName?: string;
+ readonly protocolVersion?: string;
+ readonly requestId?: string;
+ readonly sessionId?: string;
+ readonly statusCode?: string | number;
+}
+
+export interface McpLifecycle {
+ update(input: McpUpdate): void;
+ error(error?: unknown, type?: string): void;
+ arguments(value: unknown): void;
+ result(value: unknown): void;
+}
+
+export function mcpLifecycle(input: {
+ readonly write: (attributes: Attributes) => void;
+ readonly error: (error?: unknown, type?: string) => void;
+ readonly serializer: ContentSerializer;
+ readonly recordInputs: boolean;
+ readonly recordOutputs: boolean;
+}): McpLifecycle {
+ return {
+ update(update) {
+ if (update.connectionName !== undefined && update.method !== undefined)
+ input.write({
+ "agent.connection.name": update.connectionName,
+ "mcp.method.name": update.method,
+ "network.protocol.name": "http",
+ "network.transport": "tcp",
+ "mcp.protocol.version": update.protocolVersion,
+ "jsonrpc.request.id": update.requestId,
+ "gen_ai.operation.name": update.method === "tools/call" ? "execute_tool" : undefined,
+ "gen_ai.tool.name": update.method === "tools/call" ? update.toolName : undefined,
+ });
+ if (update.sessionId !== undefined) input.write({ "mcp.session.id": update.sessionId });
+ if (update.statusCode !== undefined)
+ input.write({ "rpc.response.status_code": update.statusCode });
+ },
+ error(error, type) {
+ input.error(
+ input.recordOutputs ? error : undefined,
+ type ?? (error instanceof Error ? error.name : undefined),
+ );
+ },
+ arguments(value) {
+ if (input.recordInputs)
+ input.write({ "gen_ai.tool.call.arguments": input.serializer.json(value) });
+ },
+ result(value) {
+ if (input.recordOutputs)
+ input.write({ "gen_ai.tool.call.result": input.serializer.json(value) });
+ },
+ };
+}
diff --git a/packages/tracing/src/core/scopes.ts b/packages/tracing/src/core/scopes.ts
new file mode 100644
index 000000000..fcd1ffe11
--- /dev/null
+++ b/packages/tracing/src/core/scopes.ts
@@ -0,0 +1,746 @@
+import { randomBytes } from "node:crypto";
+import { createSpanWriter, type TraceOperation } from "./writer.js";
+import { usageAttributes } from "./attributes.js";
+import {
+ completeScope,
+ capturedScopeData as capturedData,
+ prepareScope,
+ parentKinds,
+ applyAttributes,
+} from "./span-kinds.js";
+import { mcpLifecycle, type McpLifecycle } from "./mcp.js";
+import { intersectCapture } from "./activation.js";
+import { snapshotRecord, traceSnapshot, validSnapshot, boundedSerializer } from "./snapshot.js";
+import { withAgentHandoff } from "./delegation.js";
+import { withoutDeclinedContent } from "./content-policy.js";
+import { runTraceContext } from "./context.js";
+import type {
+ AgentTelemetry,
+ Attributes,
+ CaptureDecision,
+ ContentSerializer,
+ ExecutionContext,
+ Operation,
+ ScopeData,
+ ScopeIdentity,
+ ScopeRecord,
+ ScopeTerminal,
+ TraceErrorHandler,
+ TraceLink,
+ TraceReference,
+ Usage,
+} from "./types.js";
+
+const UNFINISHED_CHILDREN = 10000;
+const NO_CAPTURE: CaptureDecision = { emit: false, recordInputs: false, recordOutputs: false };
+
+interface Construction extends Partial<ScopeRecord> {
+ readonly key: string;
+ readonly deferred?: boolean;
+ readonly executionContext?: ExecutionContext;
+ /** Applies sampling to a host-supplied root reference. */
+ readonly sample?: boolean;
+}
+export interface ChildOptions {
+ readonly links?: readonly TraceLink[];
+ readonly startTimeMs?: number;
+}
+export interface LiveOperation extends Operation {
+ child(data: ScopeData, key?: string, options?: ChildOptions): Promise<LiveOperation>;
+ /** Returns an unsettled child with this identity, without starting one. */
+ find(data: ScopeData): LiveOperation | undefined;
+ /** Unsettled children, in start order. */
+ readonly children: readonly LiveOperation[];
+ usage(usage: Usage, key?: string): Promise<void>;
+ /** Carries a model call's messages up to its turn: the first input and the latest output. */
+ turnContent(content: { readonly input?: string; readonly output?: string }): void;
+ error(error?: unknown, type?: string): void;
+ /** Replaces the host context for this operation and its descendants. */
+ useContext(context: ExecutionContext): void;
+ /** Fills in a deferred operation's details from a later call with the same identity. */
+ refine(data: ScopeData, startTimeMs?: number): void;
+ /** Lowers capture for this operation and its children, and drops retained content it no longer allows. */
+ narrow(capture: CaptureDecision): void;
+ /** Completes this operation, or closes the open descendants of a finished one. */
+ release(result: ScopeTerminal): Promise<void>;
+ /** Finished, with no unfinished descendants. */
+ readonly settled: boolean;
+ readonly callId?: string;
+ readonly mcp: McpLifecycle;
+ record(): ScopeRecord;
+}
+
+/** Durable trees reserve stable IDs and defer spans whose lifetime can cross processes. */
+const DEFERRED_WHEN_DURABLE: ReadonlySet<ScopeData["type"]> = new Set([
+ "activation",
+ "tool",
+ "approval",
+]);
+/** Durable work a successful attempt or turn leaves open, such as a tool awaiting approval. */
+const OUTLIVES_ATTEMPT: ReadonlySet<ScopeData["type"]> = new Set(["tool", "approval"]);
+
+function segment(value: string | number): string {
+ return encodeURIComponent(String(value));
+}
+
+/** Children with a stable identity hydrate on re-entry; the rest are numbered. */
+function semanticKey(data: ScopeData): string | undefined {
+ switch (data.type) {
+ case "step":
+ return `step:${segment(data.options.index)}:${segment(data.options.attempt ?? 0)}`;
+ case "tool":
+ return `tool:${segment(data.options.callId)}`;
+ case "approval":
+ return `approval:${segment(data.options.requestId)}`;
+ default:
+ return undefined;
+ }
+}
+
+function terminalOf(result: ScopeTerminal & { errorType?: string; result?: unknown }) {
+ const failed = result.failed || result.outcome === "failed";
+ return {
+ ...result,
+ model: "result" in result ? result.result : result.model,
+ failed,
+ errorCode: "errorType" in result ? result.errorType : result.errorCode,
+ } as ScopeTerminal;
+}
+
+export function createScopes(input: {
+ readonly telemetry: AgentTelemetry;
+ readonly serializer: ContentSerializer;
+ readonly durable: boolean;
+ readonly onError?: TraceErrorHandler;
+}) {
+ const telemetry = input.telemetry;
+ const serializer = boundedSerializer(input.serializer, input.onError);
+ const engine = createSpanWriter({ telemetry, onError: input.onError });
+ const report: TraceErrorHandler = (error, context) => {
+ try {
+ input.onError?.(error, context);
+ } catch {}
+ };
+
+ function reserve(
+ data: ScopeData,
+ key: string,
+ parent: TraceReference | undefined,
+ prepared: ReturnType<typeof prepareScope>,
+ capture: CaptureDecision,
+ ): TraceReference | undefined {
+ const ids = telemetry.ids;
+ if (!input.durable || ids === undefined) return undefined;
+ // Re-executed model calls and memory reads are new work and get new IDs.
+ if (data.type === "model" || data.type === "memory") return undefined;
+ if (data.type === "activation" && parent !== undefined)
+ return {
+ traceId: parent.traceId,
+ spanId: ids.spanId(key),
+ traceFlags: capture.emit ? parent.traceFlags & 1 : 0,
+ };
+ if (data.type === "activation") {
+ const traceId = ids.traceId(key);
+ return {
+ traceId,
+ spanId: ids.spanId(key),
+ traceFlags: capture.emit && (telemetry.samples?.(traceId, prepared) ?? true) ? 1 : 0,
+ };
+ }
+ return parent === undefined
+ ? undefined
+ : { ...parent, spanId: ids.spanId(key), isRemote: false };
+ }
+
+ async function construct(
+ identity: ScopeIdentity,
+ capture: CaptureDecision,
+ data: ScopeData,
+ parent: LiveOperation | undefined,
+ attempt: ScopeRecord["attempt"],
+ binding: Construction,
+ onChange: () => Promise<void>,
+ ): Promise<LiveOperation> {
+ try {
+ return await constructOperation(identity, capture, data, parent, attempt, binding, onChange);
+ } catch (error) {
+ report(error, { phase: "start", operation: data.type, reference: binding.reference });
+ return constructOperation(
+ identity,
+ NO_CAPTURE,
+ capturedData(data, NO_CAPTURE),
+ parent,
+ attempt,
+ {
+ ...binding,
+ sample: false,
+ reference: {
+ traceId:
+ binding.reference?.traceId ??
+ parent?.reference.traceId ??
+ binding.parent?.traceId ??
+ randomBytes(16).toString("hex"),
+ spanId: binding.reference?.spanId ?? randomBytes(8).toString("hex"),
+ traceFlags: 0,
+ },
+ },
+ onChange,
+ );
+ }
+ }
+
+ async function constructOperation(
+ identity: ScopeIdentity,
+ capture: CaptureDecision,
+ data: ScopeData,
+ parent: LiveOperation | undefined,
+ attempt: ScopeRecord["attempt"],
+ binding: Construction,
+ onChange: () => Promise<void>,
+ ): Promise<LiveOperation> {
+ if (
+ binding.terminal?.error !== null &&
+ typeof binding.terminal?.error === "object" &&
+ !(binding.terminal.error instanceof Error) &&
+ "message" in binding.terminal.error &&
+ typeof binding.terminal.error.message === "string"
+ ) {
+ const error = new Error(binding.terminal.error.message);
+ error.name = binding.terminal.errorCode ?? "Error";
+ binding = { ...binding, terminal: { ...binding.terminal, error } };
+ }
+ const key = binding.key;
+ let startTimeMs = binding.startTimeMs ?? Date.now();
+ // A restored record keeps its saved parent; a nested activation names its own.
+ const parentReference = binding.parent ?? parent?.reference;
+ if (capture.emit) capture = intersectCapture(capture, telemetry.active()?.capture);
+ let actualCapture = capture;
+ let actualData = capturedData(data, actualCapture);
+ let prepared = prepareScope(
+ {
+ identity,
+ data: actualData,
+ attempt,
+ capture: actualCapture,
+ key,
+ parent: parentReference,
+ startTimeMs,
+ links: binding.links,
+ },
+ serializer,
+ );
+ prepared = { ...prepared, attributes: { ...prepared.attributes, ...binding.attributes } };
+ actualData = snapshotRecord(
+ {
+ key,
+ identity,
+ data: actualData,
+ capture: actualCapture,
+ reference: binding.reference ?? {
+ traceId: "0".repeat(32),
+ spanId: "0".repeat(16),
+ traceFlags: 0,
+ },
+ startTimeMs,
+ },
+ serializer,
+ ).data;
+ const deferred =
+ binding.deferred ?? (input.durable && DEFERRED_WHEN_DURABLE.has(actualData.type));
+ let host = binding.executionContext;
+ let reference =
+ binding.reference ?? reserve(data, key, parentReference, prepared, actualCapture);
+ // A nested turn joins its parent's sampling decision; a root turn asks the sampler.
+ if (binding.sample === true && reference !== undefined)
+ reference = {
+ ...reference,
+ traceFlags:
+ (reference.traceFlags & 1) !== 0 &&
+ actualCapture.emit &&
+ (parentReference !== undefined ||
+ (telemetry.samples?.(reference.traceId, prepared) ?? true))
+ ? 1
+ : 0,
+ };
+ let operation: TraceOperation | undefined = deferred
+ ? undefined
+ : engine.start(prepared, actualCapture, host, reference);
+ const retainedReference = reference ?? operation?.reference;
+ if (retainedReference === undefined)
+ throw new Error("A deferred trace scope requires a reserved reference.");
+ const sampled = (retainedReference.traceFlags & 1) !== 0;
+ actualCapture = {
+ emit: actualCapture.emit && sampled,
+ recordInputs: actualCapture.emit && sampled && actualCapture.recordInputs,
+ recordOutputs: actualCapture.emit && sampled && actualCapture.recordOutputs,
+ };
+ actualData = capturedData(actualData, actualCapture);
+ let finished = binding.finished ?? false;
+ let terminalResult = binding.terminal;
+ const children = new Map<string, LiveOperation>();
+ const pending = new Set<Promise<void>>();
+ let childSequence = binding.childSequence ?? 0;
+ let totalUsage = binding.usage;
+ const usageKeys = new Set(binding.usageKeys ?? []);
+ const enrichment: Record<string, Attributes[string]> = {};
+ const callId = actualData.type === "tool" ? actualData.options.callId : undefined;
+
+ function changed(): void {
+ void onChange();
+ }
+
+ /** Content arriving now is held to any narrower run ceiling in force. */
+ function current(): CaptureDecision {
+ return intersectCapture(actualCapture, telemetry.active()?.capture);
+ }
+
+ async function child(
+ childData: ScopeData,
+ childKey: string,
+ options: ChildOptions = {},
+ ): Promise<LiveOperation> {
+ const existing = children.get(childKey);
+ if (input.durable && existing !== undefined && !existing.settled) {
+ existing.refine(childData, options.startTimeMs);
+ return existing;
+ }
+ if (finished) throw new Error("The operation is not permitted in this trace scope.");
+ for (const [previousKey, previous] of children)
+ if (previous.settled) children.delete(previousKey);
+ let childCapture = actualCapture;
+ if (children.size >= UNFINISHED_CHILDREN) {
+ report(new Error("Trace unfinished-child limit reached."), {
+ phase: "start",
+ operation: childData.type,
+ reference: retainedReference,
+ });
+ childCapture = NO_CAPTURE;
+ }
+ if (childData.type === "model" && operation !== undefined)
+ applyAttributes(operation, { "agent.model.id": childData.options.modelId });
+ const next = await construct(
+ identity,
+ childCapture,
+ childData,
+ runtime,
+ childData.type === "step"
+ ? { index: childData.options.index, attempt: childData.options.attempt ?? 0 }
+ : attempt,
+ {
+ key: childKey,
+ executionContext: host,
+ links: options.links,
+ startTimeMs: options.startTimeMs,
+ },
+ onChange,
+ );
+ if (childCapture.emit || children.size < UNFINISHED_CHILDREN) children.set(childKey, next);
+ await onChange();
+ return next;
+ }
+
+ async function closeChildren(result: ScopeTerminal, all: boolean): Promise<void> {
+ const container = actualData.type === "step" || actualData.type === "activation";
+ for (const next of children.values())
+ if (all || !container || !OUTLIVES_ATTEMPT.has(next.type))
+ await next.release({ failed: result.failed, error: result.error, outcome: "abandoned" });
+ }
+
+ async function finish(result: ScopeTerminal): Promise<void> {
+ if (pending.size > 0) await Promise.allSettled(pending);
+ if (finished) return;
+ const capture = current();
+ if (!capture.recordOutputs) result = { ...result, error: undefined };
+ // A call whose work already failed cannot finish as completed.
+ if (terminalResult?.failed)
+ result = {
+ ...result,
+ outcome: result.outcome === "completed" ? "failed" : result.outcome,
+ failed: true,
+ error: terminalResult.error,
+ errorCode: result.errorCode ?? terminalResult.errorCode,
+ };
+ finished = true;
+ terminalResult = result;
+ await closeChildren(result, result.failed === true || !input.durable);
+ operation ??= engine.start(prepared, actualCapture, host, retainedReference);
+ const terminal =
+ actualData.type === "activation" && result.usage === undefined
+ ? { ...result, usage: totalUsage }
+ : result;
+ try {
+ completeScope(
+ operation,
+ actualData,
+ terminal,
+ { ...capture, emit: actualCapture.emit },
+ startTimeMs,
+ serializer,
+ );
+ } catch (error) {
+ report(error, {
+ phase: "complete",
+ operation: actualData.type,
+ reference: retainedReference,
+ });
+ }
+ applyAttributes(operation, enrichment);
+ if (actualData.type === "model" && result.model !== undefined) {
+ await parent?.usage(result.model.usage, key);
+ const content = result.model.content;
+ if (capture.recordOutputs && actualCapture.recordOutputs && content !== undefined) {
+ const output = serializer.outputMessages(content, result.model.finishReason);
+ if (output !== undefined) parent?.turnContent({ output });
+ }
+ }
+ operation.end(result.endTimeMs);
+ await onChange();
+ }
+
+ const runtime: LiveOperation = {
+ waitUntil(completion) {
+ pending.add(completion);
+ parent?.waitUntil(completion);
+ void completion.then(
+ () => pending.delete(completion),
+ () => pending.delete(completion),
+ );
+ },
+ snapshot: () => traceSnapshot(runtime.record()),
+ attributes: (attributes) => runtime.update({ attributes }),
+ modelCall: (options, key) => runtime.child({ type: "model", options }, key),
+ fail(error) {
+ runtime.error(error);
+ return runtime.complete({ outcome: "failed", failed: true });
+ },
+ update(update) {
+ if (finished) return;
+ const attributes = (withoutDeclinedContent(update.attributes ?? {}, current()) ??
+ update.attributes) as Attributes;
+ if (actualData.type === "activation")
+ actualData = {
+ ...actualData,
+ options: {
+ ...actualData.options,
+ attributes: { ...actualData.options.attributes, ...attributes },
+ },
+ };
+ prepared = {
+ ...prepared,
+ attributes: { ...prepared.attributes, ...attributes },
+ links: update.links ?? prepared.links,
+ };
+ if (operation !== undefined) applyAttributes(operation, attributes);
+ changed();
+ },
+ useContext(context) {
+ host = context;
+ for (const next of children.values()) next.useContext(context);
+ },
+ refine(next, earlierStart) {
+ if (finished || operation !== undefined) return;
+ if (earlierStart !== undefined && earlierStart < startTimeMs) startTimeMs = earlierStart;
+ if (next.type === "tool" && actualData.type === "tool") {
+ const known = actualData.options;
+ actualData = capturedData(
+ {
+ type: "tool",
+ options: {
+ ...known,
+ kind: known.kind ?? next.options.kind,
+ // Earlier arguments were recorded with consent; new ones need it now.
+ arguments:
+ known.arguments ?? (current().recordInputs ? next.options.arguments : undefined),
+ },
+ },
+ actualCapture,
+ );
+ }
+ const again = prepareScope(
+ {
+ identity,
+ data: actualData,
+ attempt,
+ capture: actualCapture,
+ key,
+ parent: parentReference,
+ startTimeMs,
+ links: prepared.links,
+ },
+ serializer,
+ );
+ const defined = Object.fromEntries(
+ Object.entries(again.attributes).filter(([, value]) => value !== undefined),
+ );
+ prepared = {
+ ...prepared,
+ name: again.name,
+ kind: again.kind,
+ startTimeMs,
+ attributes: { ...prepared.attributes, ...defined },
+ };
+ changed();
+ },
+ narrow(next) {
+ const narrowed = intersectCapture(actualCapture, next);
+ actualCapture = { ...narrowed, emit: actualCapture.emit };
+ actualData = capturedData(actualData, actualCapture);
+ const kept = (withoutDeclinedContent(prepared.attributes, actualCapture) ??
+ prepared.attributes) as Attributes;
+ prepared = {
+ ...prepared,
+ attributes:
+ actualData.type === "activation"
+ ? {
+ ...kept,
+ "agent.trace.content.input": actualCapture.recordInputs,
+ "agent.trace.content.output": actualCapture.recordOutputs,
+ }
+ : kept,
+ };
+ const permitted = withoutDeclinedContent(enrichment, actualCapture) ?? enrichment;
+ for (const key of Object.keys(enrichment)) if (!(key in permitted)) delete enrichment[key];
+ if (!actualCapture.recordOutputs && terminalResult?.error !== undefined)
+ terminalResult = { ...terminalResult, error: undefined };
+ operation?.narrow(actualCapture);
+ for (const child of children.values()) child.narrow(actualCapture);
+ changed();
+ },
+ async release(result) {
+ if (finished) await closeChildren(result, result.failed === true || !input.durable);
+ else await finish(terminalOf(result));
+ },
+ record() {
+ return snapshotRecord(
+ {
+ version: 1,
+ finished,
+ terminal: terminalResult,
+ usage: totalUsage,
+ usageKeys: [...usageKeys],
+ attributes: { ...prepared.attributes, ...enrichment },
+ childSequence,
+ children: [...children.values()]
+ .filter((child) => !child.settled)
+ .map((child) => child.record()),
+ key,
+ identity,
+ data: actualData,
+ capture: actualCapture,
+ reference: retainedReference,
+ parent: parentReference,
+ startTimeMs,
+ attempt,
+ links: prepared.links,
+ },
+ serializer,
+ );
+ },
+ get parent() {
+ return parentReference;
+ },
+ get startTimeMs() {
+ return startTimeMs;
+ },
+ child(data, childKey, options) {
+ if (!parentKinds(data).includes(actualData.type))
+ throw new Error("The operation is not permitted in this trace scope.");
+ const name =
+ childKey === undefined
+ ? ((input.durable ? semanticKey(data) : undefined) ?? `${data.type}:${childSequence++}`)
+ : `${data.type}:${segment(childKey)}`;
+ return child(data, `${key}/${name}`, options);
+ },
+ find(data) {
+ const name = semanticKey(data);
+ const found = name === undefined ? undefined : children.get(`${key}/${name}`);
+ return found === undefined || found.settled ? undefined : found;
+ },
+ get children() {
+ return [...children.values()].filter((next) => !next.settled);
+ },
+ type: actualData.type,
+ reference: retainedReference,
+ callId,
+ get capture() {
+ return actualCapture;
+ },
+ mcp: mcpLifecycle({
+ serializer,
+ ...actualCapture,
+ write(attributes) {
+ const permitted = (withoutDeclinedContent(attributes, current()) ??
+ attributes) as Attributes;
+ Object.assign(enrichment, permitted);
+ if (operation !== undefined) applyAttributes(operation, permitted);
+ },
+ error(error, type) {
+ runtime.error(error, type);
+ },
+ }),
+ get finished() {
+ return finished;
+ },
+ get settled() {
+ return finished && [...children.values()].every((next) => next.settled);
+ },
+ run(execute, ceiling) {
+ if (finished) throw new Error("The trace scope has already finished.");
+ // The ceiling governs work inside this run, not what the operation already recorded.
+ const scoped = intersectCapture(
+ intersectCapture(actualCapture, ceiling),
+ telemetry.active()?.capture,
+ );
+ const delegation =
+ actualData.type === "tool" &&
+ (actualData.options.kind === "subagent-call" ||
+ actualData.options.kind === "remote-agent-call")
+ ? actualData.options
+ : undefined;
+ const callback =
+ delegation === undefined
+ ? execute
+ : () =>
+ withAgentHandoff(
+ {
+ caller: retainedReference,
+ conversationId: identity.conversationId,
+ parentRunId: identity.runId,
+ parentCallId: delegation.callId,
+ agentName: delegation.name,
+ capture: scoped,
+ },
+ execute,
+ );
+ return runTraceContext(
+ telemetry,
+ { type: runtime.type, reference: retainedReference, capture: scoped, mcp: runtime.mcp },
+ callback,
+ host,
+ );
+ },
+ async complete(result = terminalResult ?? {}) {
+ if (finished) return;
+ await finish(terminalOf(result));
+ },
+ async usage(usage, callKey) {
+ if (callKey !== undefined && usageKeys.has(callKey)) return;
+ if (callKey !== undefined) usageKeys.add(callKey);
+ if (operation !== undefined) applyAttributes(operation, usageAttributes(usage));
+ totalUsage = {
+ inputTokens:
+ usage.inputTokens === undefined
+ ? totalUsage?.inputTokens
+ : (totalUsage?.inputTokens ?? 0) + usage.inputTokens,
+ outputTokens:
+ usage.outputTokens === undefined
+ ? totalUsage?.outputTokens
+ : (totalUsage?.outputTokens ?? 0) + usage.outputTokens,
+ costUsd:
+ usage.costUsd === undefined
+ ? totalUsage?.costUsd
+ : (totalUsage?.costUsd ?? 0) + usage.costUsd,
+ };
+ await parent?.usage(usage, callKey);
+ },
+ turnContent(content) {
+ if (actualData.type !== "activation") return parent?.turnContent(content);
+ if (finished) return;
+ const attributes: Record<string, Attributes[string]> = {};
+ if (
+ content.input !== undefined &&
+ prepared.attributes["gen_ai.input.messages"] === undefined
+ )
+ attributes["gen_ai.input.messages"] = content.input;
+ if (content.output !== undefined) attributes["gen_ai.output.messages"] = content.output;
+ if (Object.keys(attributes).length > 0) runtime.update({ attributes });
+ },
+ error(error, errorType) {
+ terminalResult = {
+ ...terminalResult,
+ failed: true,
+ error: current().recordOutputs ? error : undefined,
+ errorCode: errorType ?? (error instanceof Error ? error.name : undefined),
+ };
+ if (operation !== undefined) operation.fail(terminalResult.error, terminalResult.errorCode);
+ changed();
+ },
+ };
+ for (const saved of binding.children ?? [])
+ children.set(
+ saved.key,
+ await construct(
+ saved.identity,
+ intersectCapture(saved.capture, actualCapture),
+ saved.data,
+ runtime,
+ saved.attempt,
+ { ...saved, deferred: true, executionContext: host },
+ onChange,
+ ),
+ );
+ if (!finished && operation !== undefined && actualData.type === "step")
+ operation.addEvent("step.started", undefined, startTimeMs);
+ const inputMessages = prepared.attributes["gen_ai.input.messages"];
+ if (!finished && actualData.type === "model" && typeof inputMessages === "string")
+ parent?.turnContent({ input: inputMessages });
+ return runtime;
+ }
+
+ return {
+ /** Starts a turn tree. Durable turns reserve their IDs from `key`. */
+ start(facts: {
+ identity: ScopeIdentity;
+ key: string;
+ capture: CaptureDecision;
+ data: ScopeData;
+ parent?: TraceReference;
+ reference?: TraceReference;
+ startTimeMs?: number;
+ links?: readonly TraceLink[];
+ context?: ExecutionContext;
+ onChange?: () => Promise<void>;
+ }): Promise<LiveOperation> {
+ return construct(
+ facts.identity,
+ facts.capture,
+ facts.data,
+ undefined,
+ undefined,
+ {
+ key: facts.key,
+ parent: facts.parent,
+ reference: facts.reference,
+ sample: facts.reference !== undefined && facts.data.type === "activation",
+ startTimeMs: facts.startTimeMs,
+ links: facts.links,
+ executionContext: facts.context,
+ },
+ facts.onChange ?? (() => Promise.resolve()),
+ );
+ },
+ /** Rebuilds a tree from a checkpoint, or returns `undefined` for an unusable one. */
+ async restore(
+ snapshot: unknown,
+ options: {
+ capture?: CaptureDecision;
+ context?: ExecutionContext;
+ onChange?: () => Promise<void>;
+ } = {},
+ ): Promise<LiveOperation | undefined> {
+ if (!validSnapshot(snapshot)) {
+ report(new Error("Invalid tracing checkpoint."), { phase: "restore" });
+ return undefined;
+ }
+ return construct(
+ snapshot.identity,
+ intersectCapture(snapshot.capture, options.capture),
+ snapshot.data,
+ undefined,
+ snapshot.attempt,
+ { ...snapshot, deferred: true, executionContext: options.context },
+ options.onChange ?? (() => Promise.resolve()),
+ );
+ },
+ };
+}
diff --git a/packages/tracing/src/core/snapshot.ts b/packages/tracing/src/core/snapshot.ts
new file mode 100644
index 000000000..c17e514ea
--- /dev/null
+++ b/packages/tracing/src/core/snapshot.ts
@@ -0,0 +1,234 @@
+import type { ScopeRecord } from "./types.js";
+import type { ContentSerializer } from "./types.js";
+import { capturedScopeData } from "./span-kinds.js";
+const SERIALIZED_BYTES = 32768;
+const SNAPSHOT_BYTES = 65536;
+const UNFINISHED_CHILDREN = 10000;
+import { withoutDeclinedContent } from "./content-policy.js";
+
+import type { TraceSnapshot } from "./types.js";
+export function boundedSerializer(
+ serializer: ContentSerializer,
+ onError?: import("./types.js").TraceErrorHandler,
+): ContentSerializer {
+ function invoke(method: keyof ContentSerializer, args: unknown[]): string | undefined {
+ try {
+ const value = Reflect.apply(serializer[method], serializer, args) as string | undefined;
+ return value === undefined || new TextEncoder().encode(value).length > SERIALIZED_BYTES
+ ? undefined
+ : value;
+ } catch (error) {
+ try {
+ onError?.(error, { phase: "serialize" });
+ } catch {}
+ return undefined;
+ }
+ }
+ return {
+ json: (value) => invoke("json", [value]),
+ text: (value) => invoke("text", [value]),
+ inputMessages: (value) => invoke("inputMessages", [value]),
+ instructions: (value) => invoke("instructions", [value]),
+ toolDefinitions: (value) => invoke("toolDefinitions", [value]),
+ outputMessages: (value, reason) => invoke("outputMessages", [value, reason]),
+ toolResults: (value) => invoke("toolResults", [value]),
+ };
+}
+
+/** Each node is capped on its own; settled children leave the tree. */
+function nodeBytes(record: ScopeRecord): number {
+ return new TextEncoder().encode(JSON.stringify({ ...record, children: undefined })).length;
+}
+
+export function traceSnapshot(value: unknown): TraceSnapshot {
+ return JSON.parse(JSON.stringify(value)) as TraceSnapshot;
+}
+
+export function validSnapshot(
+ value: unknown,
+ maxBytes: number = SNAPSHOT_BYTES,
+ depth = 0,
+): value is ScopeRecord {
+ try {
+ if (depth > 16) return false;
+ if (typeof value !== "object" || value === null) return false;
+ const record = value as ScopeRecord;
+ if (
+ record.children !== undefined &&
+ (!Array.isArray(record.children) ||
+ record.children.length > UNFINISHED_CHILDREN ||
+ !record.children.every((child) => validSnapshot(child, maxBytes, depth + 1)))
+ )
+ return false;
+ if (record.version !== 1) return false;
+ if (record.finished !== undefined && typeof record.finished !== "boolean") return false;
+ if (
+ record.childSequence !== undefined &&
+ (!Number.isSafeInteger(record.childSequence) || record.childSequence < 0)
+ )
+ return false;
+ if (
+ typeof record.key !== "string" ||
+ typeof record.startTimeMs !== "number" ||
+ !Number.isFinite(record.startTimeMs)
+ )
+ return false;
+ if (record.reference.traceFlags !== 0 && record.reference.traceFlags !== 1) return false;
+ if (
+ record.reference.tracestate !== undefined &&
+ (typeof record.reference.tracestate !== "string" || record.reference.tracestate.length > 512)
+ )
+ return false;
+ if (
+ record.parent !== undefined &&
+ (!/^[a-f0-9]{32}$/u.test(record.parent.traceId) ||
+ !/^[a-f0-9]{16}$/u.test(record.parent.spanId))
+ )
+ return false;
+ if (
+ ![record.identity.conversationId, record.identity.runId, record.identity.turnId].every(
+ (v) => typeof v === "string",
+ )
+ )
+ return false;
+ const options = record.data.options as Record<string, unknown>;
+ const fields = {
+ activation: [],
+ step: [],
+ model: ["provider", "modelId"],
+ tool: ["callId", "name"],
+ approval: ["requestId", "callId", "toolName"],
+ memory: ["operation", "phase", "slot", "storeId"],
+ };
+ if (!fields[record.data.type].every((key) => typeof options[key] === "string")) return false;
+ if (record.data.type === "activation" && !Number.isSafeInteger(options.sequence)) return false;
+ if (record.data.type === "step" && !Number.isSafeInteger(options.index)) return false;
+ if (
+ record.data.type === "memory" &&
+ options.operation !== "search_memory" &&
+ options.operation !== "upsert_memory"
+ )
+ return false;
+ if (
+ !/^[a-f0-9]{32}$/u.test(record.reference.traceId) ||
+ !/^[a-f0-9]{16}$/u.test(record.reference.spanId) ||
+ typeof record.reference.traceFlags !== "number"
+ )
+ return false;
+ if (
+ ![record.capture.emit, record.capture.recordInputs, record.capture.recordOutputs].every(
+ (v) => typeof v === "boolean",
+ )
+ )
+ return false;
+ if (
+ !["activation", "step", "model", "tool", "approval", "memory"].includes(record.data.type) ||
+ typeof record.data.options !== "object" ||
+ record.data.options === null
+ )
+ return false;
+ if (nodeBytes(record) > maxBytes) return false;
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+export function snapshotRecord(record: ScopeRecord, serializer: ContentSerializer): ScopeRecord {
+ const terminal = record.terminal;
+ record = {
+ ...record,
+ children: record.children?.map((child) =>
+ snapshotRecord(
+ {
+ ...child,
+ capture: {
+ emit: child.capture.emit && record.capture.emit,
+ recordInputs: child.capture.recordInputs && record.capture.recordInputs,
+ recordOutputs: child.capture.recordOutputs && record.capture.recordOutputs,
+ },
+ },
+ serializer,
+ ),
+ ),
+ };
+ record = {
+ ...record,
+ terminal:
+ terminal === undefined || record.finished
+ ? undefined
+ : {
+ ...terminal,
+ errorCode:
+ terminal.errorCode ??
+ (terminal.error instanceof Error ? terminal.error.name : undefined),
+ error: record.capture.recordOutputs
+ ? terminal.error instanceof Error
+ ? { name: terminal.error.name, message: serializer.text(terminal.error.message) }
+ : terminal.error
+ : undefined,
+ output: record.capture.recordOutputs ? terminal.output : undefined,
+ response: record.capture.recordOutputs ? terminal.response : undefined,
+ records: record.capture.recordInputs ? terminal.records : undefined,
+ model:
+ terminal.model === undefined
+ ? undefined
+ : {
+ ...terminal.model,
+ content: record.capture.recordOutputs ? terminal.model.content : undefined,
+ },
+ },
+ };
+ record = {
+ ...record,
+ attributes: (withoutDeclinedContent(record.attributes ?? {}, record.capture) ??
+ record.attributes) as ScopeRecord["attributes"],
+ };
+ const data = capturedScopeData(record.data, record.capture);
+ let json: string | undefined;
+ try {
+ json = serializer.json(data);
+ } catch {}
+ const fallback = () => {
+ let structural = capturedScopeData(data, {
+ emit: record.capture.emit,
+ recordInputs: false,
+ recordOutputs: false,
+ });
+ if (structural.type === "step")
+ structural = { ...structural, options: { ...structural.options, runtimeContext: undefined } };
+ if (structural.type === "model")
+ structural = { ...structural, options: { ...structural.options, runtimeContext: undefined } };
+ return JSON.parse(
+ JSON.stringify({
+ ...record,
+ version: 1,
+ data: structural,
+ terminal:
+ record.terminal === undefined
+ ? undefined
+ : {
+ outcome: record.terminal.outcome,
+ failed: record.terminal.failed,
+ errorCode: record.terminal.errorCode,
+ usage: record.terminal.usage,
+ },
+ attributes:
+ withoutDeclinedContent(record.attributes ?? {}, {
+ recordInputs: false,
+ recordOutputs: false,
+ }) ?? record.attributes,
+ }),
+ ) as ScopeRecord;
+ };
+ if (json === undefined || new TextEncoder().encode(json).length > SERIALIZED_BYTES)
+ return fallback();
+ try {
+ const snapshot = JSON.parse(
+ JSON.stringify({ ...record, version: 1, data: JSON.parse(json) }),
+ ) as ScopeRecord;
+ return nodeBytes(snapshot) <= SNAPSHOT_BYTES ? snapshot : fallback();
+ } catch {
+ return fallback();
+ }
+}
diff --git a/packages/tracing/src/core/span-kinds.ts b/packages/tracing/src/core/span-kinds.ts
new file mode 100644
index 000000000..b649be592
--- /dev/null
+++ b/packages/tracing/src/core/span-kinds.ts
@@ -0,0 +1,400 @@
+import {
+ identityAttributes,
+ usageAttributes,
+ frameworkAttributes,
+ namingAttributes,
+ runtimeContextAttributes,
+} from "./attributes.js";
+export const invocationName = (name?: string) =>
+ name === undefined ? "invoke_agent" : `invoke_agent ${name}`;
+const modelName = (name: string) => `chat ${name}`;
+const toolName = (name: string) => `execute_tool ${name}`;
+const SPAN_NAMES = {
+ approval: "agent.approval",
+ step: "agent.step",
+} as const;
+const CONTENT_FIELDS = {
+ toolArguments: "gen_ai.tool.call.arguments",
+ toolResult: "gen_ai.tool.call.result",
+ approvalRequest: "agent.approval.request",
+ approvalResponse: "agent.approval.response",
+ memoryRecords: "gen_ai.memory.records",
+} as const;
+export function applyAttributes(
+ span: { setAttribute(key: string, value: Exclude<Attributes[string], undefined>): unknown },
+ attributes: Attributes,
+): void {
+ for (const [key, value] of Object.entries(attributes))
+ if (value !== undefined) span.setAttribute(key, value);
+}
+import type { ContentSerializer, ModelResult } from "./types.js";
+import { withoutDeclinedContent } from "./content-policy.js";
+import type { ScopeData, ScopeIdentity, ScopeRecord, ScopeTerminal } from "./types.js";
+import type { Attributes, CaptureDecision, PreparedSpan } from "./types.js";
+import type { TraceOperation } from "./writer.js";
+
+interface KindContext {
+ identity: ScopeIdentity;
+ attributes: Attributes;
+ attempt: { turnId: string; index: number; attempt: number };
+ capture: CaptureDecision;
+ serializer: ContentSerializer;
+}
+type Kind<K extends ScopeData["type"]> = {
+ parents: readonly ScopeData["type"][];
+ prepare(
+ data: Extract<ScopeData, { type: K }>,
+ context: KindContext,
+ ): Pick<PreparedSpan, "name" | "kind" | "attributes">;
+ complete?(
+ span: TraceOperation,
+ data: Extract<ScopeData, { type: K }>,
+ result: ScopeTerminal,
+ context: CompletionContext,
+ ): void;
+ redact?(data: Extract<ScopeData, { type: K }>, capture: CaptureDecision): ScopeData;
+};
+interface CompletionContext {
+ capture: CaptureDecision;
+ serializer: ContentSerializer;
+ outcome: string;
+ startTimeMs: number;
+}
+function attemptAttributes(context: KindContext): Attributes {
+ return {
+ "agent.turn.id": context.attempt.turnId,
+ "agent.step.index": context.attempt.index,
+ "agent.step.attempt": context.attempt.attempt,
+ };
+}
+function permitted(
+ attributes: Attributes | undefined,
+ capture: CaptureDecision,
+): Attributes | undefined {
+ return (withoutDeclinedContent(attributes ?? {}, capture) ?? attributes) as
+ | Attributes
+ | undefined;
+}
+function toolInput(arguments_: unknown, context: KindContext): Attributes {
+ return {
+ [CONTENT_FIELDS.toolArguments]: context.capture.recordInputs
+ ? context.serializer.json(arguments_)
+ : undefined,
+ };
+}
+function toolOutput(span: TraceOperation, result: ScopeTerminal, context: CompletionContext): void {
+ if (context.capture.recordOutputs)
+ applyAttributes(span, { [CONTENT_FIELDS.toolResult]: context.serializer.json(result.output) });
+}
+const kinds: { [K in ScopeData["type"]]: Kind<K> } = {
+ activation: {
+ parents: [],
+ prepare({ options }, context) {
+ const { identity, capture } = context;
+ return {
+ name: invocationName(identity.agentName),
+ attributes: {
+ ...context.attributes,
+ ...frameworkAttributes(identity.framework),
+ ...namingAttributes(invocationName(identity.agentName), "invoke_agent"),
+ "agent.channel.kind": options.channel?.kind,
+ "agent.session.origin": options.channel?.origin,
+ "agent.name": identity.agentName,
+ "gen_ai.agent.name": identity.agentName,
+ "gen_ai.operation.name": "invoke_agent",
+ "agent.turn.id": identity.turnId,
+ "agent.turn.sequence": options.sequence,
+ "agent.run.type": options.subagent ? "subagent" : "session",
+ "agent.parent_call.id": options.parentCallId,
+ "agent.parent_run.id": options.parentRunId,
+ "agent.subagent.name": options.subagentName,
+ "agent.trace.content.input": capture.recordInputs,
+ "agent.trace.content.output": capture.recordOutputs,
+ ...permitted(options.attributes, capture),
+ },
+ };
+ },
+ complete(span, _data, result, context) {
+ span.addEvent("turn.started", undefined, context.startTimeMs);
+ if (result.outcomeUnknown !== true || result.failed === true)
+ span.addEvent(`turn.${context.outcome}`, undefined, result.endTimeMs);
+ },
+ redact(data, capture) {
+ return {
+ ...data,
+ options: { ...data.options, attributes: permitted(data.options.attributes, capture) },
+ };
+ },
+ },
+ step: {
+ parents: ["activation"],
+ prepare({ options }, context) {
+ return {
+ name: SPAN_NAMES.step,
+ attributes: {
+ ...context.attributes,
+ ...frameworkAttributes(context.identity.framework),
+ ...attemptAttributes(context),
+ ...namingAttributes(SPAN_NAMES.step, "workflow"),
+ "gen_ai.operation.name": "workflow",
+ "agent.name": context.identity.agentName,
+ "agent.channel.kind":
+ options.channel?.kind === "unknown" ? undefined : options.channel?.kind,
+ "agent.session.origin":
+ options.channel?.kind === "unknown" ? undefined : options.channel?.origin,
+ ...runtimeContextAttributes(options.runtimeContext),
+ },
+ };
+ },
+ complete(span, _data, result) {
+ span.addEvent(result.failed ? "step.failed" : "step.completed", undefined, result.endTimeMs);
+ },
+ },
+ model: {
+ parents: ["step"],
+ prepare({ options }, context) {
+ return {
+ name: modelName(options.modelId),
+ kind: "CLIENT",
+ attributes: {
+ ...context.attributes,
+ ...namingAttributes(modelName(options.modelId), "chat"),
+ "gen_ai.agent.name": context.identity.agentName,
+ "gen_ai.operation.name": "chat",
+ "gen_ai.provider.name": options.provider,
+ "gen_ai.request.model": options.modelId,
+ ...runtimeContextAttributes(options.runtimeContext),
+ ...(context.capture.recordInputs && options.messages !== undefined
+ ? {
+ "gen_ai.input.messages": context.serializer.inputMessages(options.messages),
+ "gen_ai.system_instructions": context.serializer.instructions(options.instructions),
+ }
+ : undefined),
+ "gen_ai.tool.definitions": context.capture.recordInputs
+ ? context.serializer.toolDefinitions(options.tools)
+ : undefined,
+ },
+ };
+ },
+ complete(span, _data, result, context) {
+ if (result.model !== undefined)
+ applyAttributes(
+ span,
+ modelResultAttributes(result.model, context.serializer, context.capture.recordOutputs),
+ );
+ },
+ redact(data, capture) {
+ return capture.recordInputs
+ ? data
+ : {
+ ...data,
+ options: {
+ ...data.options,
+ messages: undefined,
+ instructions: undefined,
+ tools: undefined,
+ },
+ };
+ },
+ },
+ tool: {
+ parents: ["step", "tool"],
+ prepare({ options }, context) {
+ const delegates = options.kind === "subagent-call" || options.kind === "remote-agent-call";
+ return {
+ name: toolName(options.name),
+ kind: options.kind === "remote-agent-call" ? "CLIENT" : "INTERNAL",
+ attributes: {
+ ...context.attributes,
+ ...frameworkAttributes(context.identity.framework),
+ ...attemptAttributes(context),
+ ...namingAttributes(toolName(options.name), "execute_tool"),
+ "gen_ai.agent.name": delegates ? options.name : context.identity.agentName,
+ "gen_ai.operation.name": "execute_tool",
+ "gen_ai.tool.call.id": options.callId,
+ "gen_ai.tool.name": options.name,
+ "gen_ai.tool.type": "function",
+ "agent.tool.kind": options.kind,
+ "agent.invocation.role": delegates ? "caller" : undefined,
+ ...toolInput(options.arguments, context),
+ },
+ };
+ },
+ complete(span, _data, result, context) {
+ if (result.outcome !== undefined)
+ applyAttributes(span, { "agent.tool.outcome": result.outcome });
+ toolOutput(span, result, context);
+ },
+ redact(data, capture) {
+ return capture.recordInputs
+ ? data
+ : { ...data, options: { ...data.options, arguments: undefined } };
+ },
+ },
+ approval: {
+ parents: ["tool"],
+ prepare({ options }, context) {
+ return {
+ name: SPAN_NAMES.approval,
+ attributes: {
+ ...context.attributes,
+ ...frameworkAttributes(context.identity.framework),
+ ...attemptAttributes(context),
+ ...namingAttributes(SPAN_NAMES.approval, "workflow"),
+ "gen_ai.operation.name": "workflow",
+ "gen_ai.tool.call.id": options.callId,
+ "gen_ai.tool.name": options.toolName,
+ "agent.approval.kind": "tool-approval",
+ "agent.approval.request_id": options.requestId,
+ [CONTENT_FIELDS.approvalRequest]: context.capture.recordInputs
+ ? context.serializer.json(options.request)
+ : undefined,
+ },
+ };
+ },
+ complete(span, _data, result, context) {
+ if (context.capture.recordOutputs)
+ applyAttributes(span, {
+ [CONTENT_FIELDS.approvalResponse]: context.serializer.json(result.response),
+ });
+ },
+ redact(data, capture) {
+ return capture.recordInputs
+ ? data
+ : { ...data, options: { ...data.options, request: undefined } };
+ },
+ },
+ memory: {
+ parents: ["activation", "step", "tool"],
+ prepare({ options }, context) {
+ return {
+ name: options.operation,
+ kind: "CLIENT",
+ attributes: {
+ ...context.attributes,
+ ...namingAttributes(options.operation),
+ "gen_ai.operation.name": options.operation,
+ "gen_ai.memory.store.id": options.storeId,
+ "agent.memory.phase": options.phase,
+ "agent.memory.slot": options.slot,
+ "agent.turn.id": context.identity.turnId,
+ },
+ };
+ },
+ complete(span, _data, result, context) {
+ if (result.recordCount !== undefined)
+ applyAttributes(span, { "gen_ai.memory.record.count": result.recordCount });
+ if (context.capture.recordInputs)
+ applyAttributes(span, {
+ [CONTENT_FIELDS.memoryRecords]: context.serializer.json(result.records),
+ });
+ },
+ },
+};
+function kind<K extends ScopeData["type"]>(data: Extract<ScopeData, { type: K }>): Kind<K> {
+ return kinds[data.type] as Kind<K>;
+}
+export function parentKinds(data: ScopeData): readonly ScopeData["type"][] {
+ return kind(data).parents;
+}
+export function prepareScope(
+ record: Omit<ScopeRecord, "reference">,
+ serializer: ContentSerializer,
+): PreparedSpan {
+ const context: KindContext = {
+ identity: record.identity,
+ attributes: identityAttributes(record.identity),
+ attempt: {
+ turnId: record.identity.turnId,
+ index: record.attempt?.index ?? 0,
+ attempt: record.attempt?.attempt ?? 0,
+ },
+ capture: record.capture,
+ serializer,
+ };
+ return {
+ type: record.data.type,
+ operationId: record.key,
+ kind: "INTERNAL",
+ ...kind(record.data).prepare(record.data, context),
+ parent: record.parent,
+ root: record.data.type === "activation" && record.parent === undefined,
+ startTimeMs: record.startTimeMs,
+ links: record.links,
+ };
+}
+export function completeScope(
+ span: TraceOperation,
+ data: ScopeData,
+ result: ScopeTerminal,
+ capture: CaptureDecision,
+ startTimeMs: number,
+ serializer: ContentSerializer,
+): void {
+ const outcome = result.outcome ?? (result.failed ? "failed" : "completed");
+ // A failure is always recorded; otherwise an unknown outcome leaves the span without one.
+ const known = result.outcomeUnknown !== true || result.failed === true;
+ if ((data.type === "activation" || data.type === "approval") && known)
+ applyAttributes(span, {
+ [data.type === "activation" ? "agent.turn.outcome" : "agent.approval.outcome"]: outcome,
+ });
+ if (result.usage !== undefined)
+ applyAttributes(
+ span,
+ usageAttributes(result.usage, data.type === "activation" || data.type === "model"),
+ );
+ kind(data).complete?.(span, data, result, { capture, startTimeMs, serializer, outcome });
+ if (result.failed) span.fail(result.error, result.errorCode);
+}
+export function capturedScopeData(data: ScopeData, capture: CaptureDecision): ScopeData {
+ if (!capture.emit) capture = { emit: false, recordInputs: false, recordOutputs: false };
+ return kind(data).redact?.(data, capture) ?? data;
+}
+function modelResultAttributes(
+ input: ModelResult,
+ serializer: ContentSerializer,
+ recordOutputs: boolean,
+): Attributes {
+ const attributes: Record<string, Attributes[string]> = {
+ ...usageAttributes(input.usage, true),
+ "gen_ai.response.id": input.responseId,
+ "gen_ai.response.model": input.responseModelId,
+ "gen_ai.response.finish_reasons": [input.finishReason],
+ };
+ if (!recordOutputs) return attributes;
+ attributes["ai.response.finish_reason"] = input.finishReason;
+ const content = input.content;
+ if (content === undefined) return attributes;
+ attributes["gen_ai.output.messages"] = serializer.outputMessages(content, input.finishReason);
+ attributes["ai.response.reasoning"] = serializer.text(
+ content
+ .filter((part) => part.type === "reasoning")
+ .map((part) => part.text)
+ .filter((text) => text.trim().length > 0)
+ .join("\n"),
+ );
+ attributes["ai.response.text"] = serializer.text(
+ content
+ .filter((part) => part.type === "text")
+ .map((part) => part.text)
+ .join(""),
+ );
+ const calls = content
+ .filter((part) => part.type === "tool-call")
+ .map((part) => ({ callId: part.callId, input: part.input, toolName: part.toolName }));
+ if (calls.length > 0) attributes["ai.response.tool_calls"] = serializer.json(calls);
+ const results = content
+ .filter((part) => part.type === "tool-result" || part.type === "tool-error")
+ .map((part) =>
+ part.type === "tool-result"
+ ? { callId: part.callId, input: part.input, output: part.output, toolName: part.toolName }
+ : {
+ callId: part.callId,
+ input: part.input,
+ error: part.error instanceof Error ? part.error.message : part.error,
+ toolName: part.toolName,
+ },
+ );
+ if (results.length > 0) attributes["ai.response.tool_results"] = serializer.toolResults(results);
+ return attributes;
+}
diff --git a/packages/tracing/src/core/types.ts b/packages/tracing/src/core/types.ts
new file mode 100644
index 000000000..6fa1f7fd1
--- /dev/null
+++ b/packages/tracing/src/core/types.ts
@@ -0,0 +1,299 @@
+export type AttributeValue =
+ | string
+ | number
+ | boolean
+ | (string | null | undefined)[]
+ | (number | null | undefined)[]
+ | (boolean | null | undefined)[];
+export type Attributes = Readonly<Record<string, AttributeValue | undefined>>;
+export type TraceJson =
+ | null
+ | boolean
+ | number
+ | string
+ | readonly TraceJson[]
+ | { readonly [key: string]: TraceJson };
+declare const traceSnapshotBrand: unique symbol;
+export type TraceSnapshot = TraceJson & { readonly [traceSnapshotBrand]: true };
+export type SpanType =
+ | "activation"
+ | "step"
+ | "model"
+ | "tool"
+ | "approval"
+ | "memory"
+ | "channelRequest"
+ | "mcp";
+export type SpanKind = "INTERNAL" | "CLIENT" | "SERVER" | "PRODUCER" | "CONSUMER";
+
+/** Host-owned process-local context. Never stored in a checkpoint. */
+export type ExecutionContext = object;
+
+export interface TraceReference {
+ readonly traceId: string;
+ readonly spanId: string;
+ readonly traceFlags: number;
+ readonly isRemote?: boolean;
+ readonly tracestate?: string;
+}
+
+export interface CaptureDecision {
+ readonly emit: boolean;
+ readonly recordInputs: boolean;
+ readonly recordOutputs: boolean;
+}
+
+export interface TraceErrorContext {
+ readonly phase: "start" | "serialize" | "complete" | "context" | "restore" | "checkpoint";
+ readonly operation?: SpanType;
+ readonly reference?: TraceReference;
+}
+export type TraceErrorHandler = (error: unknown, context: TraceErrorContext) => void;
+
+export interface TraceLink {
+ readonly context: TraceReference;
+ readonly relationship: string;
+}
+
+export interface PreparedSpan {
+ readonly type: SpanType;
+ readonly operationId: string;
+ readonly name: string;
+ readonly kind?: SpanKind;
+ readonly attributes: Attributes;
+ readonly parent?: TraceReference;
+ readonly root?: boolean;
+ readonly startTimeMs?: number;
+ readonly links?: readonly TraceLink[];
+}
+
+export interface SpanWriter {
+ readonly reference: TraceReference;
+ setAttribute(key: string, value: AttributeValue): void;
+ addEvent(name: string, attributes?: Attributes, timeMs?: number): void;
+ fail(error?: unknown, errorType?: string): void;
+ setStatus(code: "UNSET" | "OK" | "ERROR"): void;
+ end(timeMs?: number): void;
+}
+
+/** Span output and context propagation used by agent tracing. */
+export interface AgentTelemetry {
+ /** Starts a span. A `reference` requests that exact trace and span ID. */
+ startSpan(
+ span: PreparedSpan,
+ options?: { readonly reference?: TraceReference; readonly context?: ExecutionContext },
+ ): SpanWriter;
+ /** Returns the operation active in the current execution context. */
+ active(): ActiveOperation | undefined;
+ /** Runs `execute` with `operation` as the active trace context. */
+ run<T>(operation: ActiveOperation, execute: () => T, context?: ExecutionContext): T;
+ /** Runs `execute` without creating spans. */
+ suppressed?<T>(execute: () => T): T;
+ /** Decides whether a durable turn's trace is recorded. Defaults to recording. */
+ samples?(traceId: string, span: PreparedSpan): boolean;
+ /** Derives stable IDs from keys. Present only when `startSpan` honors `reference`. */
+ readonly ids?: {
+ traceId(key: string): string;
+ spanId(key: string): string;
+ };
+ forceFlush(): Promise<void>;
+ shutdown(): Promise<void>;
+}
+
+/** Durable storage for in-flight turns. Values are JSON; keys are chosen by the library. */
+export interface TraceCheckpointer {
+ get(key: string): unknown | PromiseLike<unknown>;
+ set(key: string, value: TraceSnapshot): void | PromiseLike<void>;
+ delete(key: string): void | PromiseLike<void>;
+}
+
+export interface ActiveOperation {
+ readonly type: SpanType;
+ readonly reference: TraceReference;
+ readonly capture: CaptureDecision;
+ readonly mcp?: import("./mcp.js").McpLifecycle;
+}
+
+export interface MappingContext {
+ readonly type: SpanType;
+ readonly operationId: string;
+}
+
+export interface OutputMapping {
+ attributes(span: MappingContext, attributes: Attributes): Attributes;
+ link(span: MappingContext, link: TraceLink): Attributes;
+}
+
+export interface RunIdentity {
+ readonly conversationId: string;
+ readonly runId: string;
+ readonly turnId: string;
+}
+
+export interface FrameworkIdentity {
+ readonly name: string;
+ readonly version?: string;
+}
+
+export interface Usage {
+ readonly costUsd?: number;
+ readonly inputTokens?: number;
+ readonly outputTokens?: number;
+ readonly inputTokenDetails?: {
+ readonly cacheReadTokens?: number;
+ readonly cacheWriteTokens?: number;
+ };
+}
+
+/** `subagent-call` and `remote-agent-call` delegate to another agent. */
+export type ToolKind = string;
+export interface ContentSerializer {
+ json(value: unknown): string | undefined;
+ text(value: string): string | undefined;
+ inputMessages(value: unknown): string | undefined;
+ instructions(value: unknown): string | undefined;
+ toolDefinitions(value: unknown): string | undefined;
+ outputMessages(value: readonly ContentPart[], finishReason: string): string | undefined;
+ toolResults(value: readonly Record<string, unknown>[]): string | undefined;
+}
+export interface ModelResult {
+ readonly usage: Usage;
+ readonly responseId?: string;
+ readonly responseModelId?: string;
+ readonly finishReason: string;
+ readonly content?: readonly ContentPart[];
+}
+export interface ScopeIdentity extends RunIdentity {
+ readonly agentName?: string;
+ readonly framework?: FrameworkIdentity;
+}
+export interface TurnMetadata {
+ readonly attributes?: Attributes;
+ readonly sequence: number;
+ readonly subagent?: boolean;
+ readonly subagentName?: string;
+ readonly parentCallId?: string;
+ readonly parentRunId?: string;
+ readonly channel?: { kind?: string; origin?: string };
+}
+export interface StepOptions {
+ readonly index: number;
+ readonly attempt?: number;
+ readonly channel?: { kind?: string; origin?: string };
+ readonly runtimeContext?: Readonly<Record<string, unknown>>;
+}
+export interface ModelOptions {
+ readonly provider: string;
+ readonly modelId: string;
+ readonly messages?: readonly unknown[];
+ readonly instructions?: unknown;
+ /** Tools offered to the model, recorded as `gen_ai.tool.definitions`. */
+ readonly tools?: unknown;
+ readonly runtimeContext?: Readonly<Record<string, unknown>>;
+}
+export interface ToolOptions {
+ readonly callId: string;
+ readonly name: string;
+ readonly kind?: ToolKind;
+ readonly arguments?: unknown;
+}
+export type ScopeData =
+ | { readonly type: "activation"; readonly options: TurnMetadata }
+ | { readonly type: "step"; readonly options: StepOptions }
+ | { readonly type: "model"; readonly options: ModelOptions }
+ | { readonly type: "tool"; readonly options: ToolOptions }
+ | {
+ readonly type: "approval";
+ readonly options: {
+ requestId: string;
+ request?: unknown;
+ callId: string;
+ toolName: string;
+ };
+ }
+ | {
+ readonly type: "memory";
+ readonly options: {
+ operation: "search_memory" | "upsert_memory";
+ phase: string;
+ slot: string;
+ storeId: string;
+ };
+ };
+export interface ScopeTerminal {
+ readonly outcome?: string;
+ /** Ends the operation without recording an outcome, when the host never learned one. */
+ readonly outcomeUnknown?: boolean;
+ readonly failed?: boolean;
+ readonly error?: unknown;
+ readonly errorCode?: string;
+ readonly output?: unknown;
+ readonly response?: unknown;
+ readonly usage?: Usage;
+ readonly model?: ModelResult;
+ readonly recordCount?: number;
+ readonly records?: readonly { id?: string; content: string }[];
+ readonly endTimeMs?: number;
+}
+export interface ScopeRecord {
+ readonly version?: 1;
+ readonly finished?: boolean;
+ /** Write token used to detect a checkpoint changed by another process. */
+ readonly revision?: string;
+ readonly key: string;
+ readonly identity: ScopeIdentity;
+ readonly data: ScopeData;
+ readonly reference: TraceReference;
+ readonly parent?: TraceReference;
+ readonly attempt?: { readonly index: number; readonly attempt: number };
+ readonly capture: CaptureDecision;
+ readonly startTimeMs: number;
+ readonly links?: readonly TraceLink[];
+ readonly usage?: Usage;
+ readonly usageKeys?: readonly string[];
+ readonly attributes?: Attributes;
+ readonly childSequence?: number;
+ readonly children?: readonly ScopeRecord[];
+ readonly terminal?: ScopeTerminal;
+}
+export interface Operation {
+ child(data: ScopeData, key?: string): Promise<Operation>;
+ waitUntil(completion: Promise<void>): void;
+ readonly type: ScopeData["type"];
+ readonly reference: TraceReference;
+ readonly parent?: TraceReference;
+ readonly startTimeMs: number;
+ readonly capture: CaptureDecision;
+ readonly finished: boolean;
+ snapshot(): TraceSnapshot;
+ run<T>(execute: () => T, ceiling?: CaptureDecision): T;
+ attributes(attributes: Attributes): void;
+ update(input: { attributes?: Attributes; links?: readonly TraceLink[] }): void;
+ complete(result?: ScopeTerminal & { errorType?: string; result?: ModelResult }): Promise<void>;
+ fail(error: unknown): Promise<void>;
+ modelCall(input: ModelOptions, key?: string): Promise<Operation>;
+}
+
+export type ContentPart =
+ | { readonly type: "text"; readonly text: string }
+ | { readonly type: "reasoning"; readonly text: string }
+ | {
+ readonly type: "tool-call";
+ readonly callId: string;
+ readonly toolName: string;
+ readonly input: unknown;
+ }
+ | {
+ readonly type: "tool-result";
+ readonly callId: string;
+ readonly toolName: string;
+ readonly input: unknown;
+ readonly output: unknown;
+ }
+ | {
+ readonly type: "tool-error";
+ readonly callId: string;
+ readonly toolName: string;
+ readonly input: unknown;
+ readonly error: unknown;
+ };
diff --git a/packages/tracing/src/core/writer.test.ts b/packages/tracing/src/core/writer.test.ts
new file mode 100644
index 000000000..f577f8dc0
--- /dev/null
+++ b/packages/tracing/src/core/writer.test.ts
@@ -0,0 +1,75 @@
+import { describe, expect, it, vi } from "vitest";
+import { createSpanWriter } from "./writer.js";
+import type { AgentTelemetry, SpanWriter, PreparedSpan } from "./types.js";
+
+describe("trace engine failure and capture boundary", () => {
+ it("runs application code exactly once when context fails and preserves application errors", () => {
+ const writer: SpanWriter = {
+ reference: { traceId: "1".repeat(32), spanId: "2".repeat(16), traceFlags: 1 },
+ setAttribute: vi.fn(),
+ addEvent: vi.fn(),
+ fail: vi.fn(),
+ setStatus: vi.fn(),
+ end: vi.fn(),
+ };
+ const telemetry: Pick<AgentTelemetry, "startSpan" | "run" | "suppressed"> = {
+ startSpan: () => writer,
+ run() {
+ throw new Error("context unavailable");
+ },
+ suppressed() {
+ throw new Error("suppression unavailable");
+ },
+ };
+ const engine = createSpanWriter({ telemetry });
+ const operation = engine.start(
+ { type: "tool", operationId: "tool", name: "execute_tool lookup", attributes: {} },
+ { emit: true, recordInputs: false, recordOutputs: false },
+ );
+ const execute = vi.fn(() => "result");
+ expect(operation.run(execute)).toBe("result");
+ expect(execute).toHaveBeenCalledTimes(1);
+ const error = new Error("application");
+ telemetry.run = (_operation, callback) => callback();
+ const fail = vi.fn(() => {
+ throw error;
+ });
+ expect(() => operation.run(fail)).toThrow(error);
+ expect(fail).toHaveBeenCalledTimes(1);
+ });
+
+ it("filters denied initial and late content and closes each operation once", () => {
+ const writer: SpanWriter = {
+ reference: { traceId: "1".repeat(32), spanId: "2".repeat(16), traceFlags: 1 },
+ setAttribute: vi.fn(),
+ addEvent: vi.fn(),
+ fail: vi.fn(),
+ setStatus: vi.fn(),
+ end: vi.fn(),
+ };
+ const start = vi.fn((_span: PreparedSpan) => writer);
+ const telemetry: Pick<AgentTelemetry, "startSpan" | "run"> = {
+ startSpan: start,
+ run: (_operation, callback) => callback(),
+ };
+ const operation = createSpanWriter({ telemetry }).start(
+ {
+ type: "tool",
+ operationId: "tool",
+ name: "execute_tool lookup",
+ attributes: { "gen_ai.tool.name": "lookup", "gen_ai.tool.call.arguments": "secret" },
+ },
+ { emit: true, recordInputs: false, recordOutputs: false },
+ );
+ expect(start.mock.calls[0]![0].attributes).toEqual({ "gen_ai.tool.name": "lookup" });
+ operation.setAttribute("gen_ai.tool.call.result", "secret");
+ operation.addEvent("protocol", { "gen_ai.input.messages": "secret", "rpc.method": "call" });
+ operation.fail(new TypeError("secret"));
+ operation.end();
+ operation.end();
+ expect(writer.setAttribute).not.toHaveBeenCalled();
+ expect(writer.addEvent).toHaveBeenCalledWith("protocol", { "rpc.method": "call" }, undefined);
+ expect(writer.fail).toHaveBeenCalledWith(undefined, "TypeError");
+ expect(writer.end).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/packages/tracing/src/core/writer.ts b/packages/tracing/src/core/writer.ts
new file mode 100644
index 000000000..aeb1efbf2
--- /dev/null
+++ b/packages/tracing/src/core/writer.ts
@@ -0,0 +1,135 @@
+import type {
+ PreparedSpan,
+ SpanWriter,
+ ExecutionContext,
+ AgentTelemetry,
+ TraceReference,
+ CaptureDecision,
+ Attributes,
+ TraceErrorHandler,
+} from "./types.js";
+import { withoutDeclinedContent } from "./content-policy.js";
+import { runTraceContext } from "./context.js";
+import { intersectCapture } from "./activation.js";
+export interface TraceOperation extends SpanWriter {
+ readonly capture: CaptureDecision;
+ readonly finished: boolean;
+ run<T>(execute: () => T): T;
+ /** Lowers what the span accepts from now on. */
+ narrow(capture: CaptureDecision): void;
+}
+export function createSpanWriter(input: {
+ telemetry: Pick<AgentTelemetry, "startSpan" | "run" | "suppressed">;
+ onError?: TraceErrorHandler;
+}) {
+ function safely<T>(
+ execute: () => T,
+ phase: "start" | "complete" = "complete",
+ reference?: TraceReference,
+ ): T | undefined {
+ try {
+ return execute();
+ } catch (error) {
+ try {
+ input.onError?.(error, { phase, reference });
+ } catch {}
+ return undefined;
+ }
+ }
+ function start(
+ span: PreparedSpan,
+ capture: CaptureDecision,
+ host?: ExecutionContext,
+ reserved?: TraceReference,
+ ): TraceOperation {
+ const attributes = (withoutDeclinedContent(span.attributes, capture) ??
+ span.attributes) as Attributes;
+ const writer = capture.emit
+ ? safely(
+ () =>
+ input.telemetry.startSpan(
+ { ...span, attributes },
+ { reference: reserved, context: host },
+ ),
+ "start",
+ reserved,
+ )
+ : undefined;
+ const reference = writer?.reference ?? {
+ traceId: "0".repeat(32),
+ spanId: "0".repeat(16),
+ traceFlags: 0,
+ };
+ let finished = false;
+ return {
+ reference,
+ get capture() {
+ return capture;
+ },
+ narrow(next) {
+ capture = intersectCapture(capture, next);
+ },
+ get finished() {
+ return finished;
+ },
+ setAttribute(key, value) {
+ const permitted = withoutDeclinedContent({ [key]: value }, capture);
+ if (!finished && (permitted === undefined || key in permitted))
+ safely(() =>
+ writer?.setAttribute(
+ key,
+ permitted === undefined ? value : (permitted[key] as typeof value),
+ ),
+ );
+ },
+ addEvent(name, attributes, timeMs) {
+ if (!finished)
+ safely(() =>
+ writer?.addEvent(
+ name,
+ attributes === undefined
+ ? undefined
+ : ((withoutDeclinedContent(attributes, capture) ?? attributes) as Attributes),
+ timeMs,
+ ),
+ );
+ },
+ fail(error, type) {
+ if (!finished)
+ safely(() =>
+ writer?.fail(
+ capture.recordOutputs ? error : undefined,
+ type ?? (error instanceof Error ? error.name : undefined),
+ ),
+ );
+ },
+ setStatus(code) {
+ if (!finished) safely(() => writer?.setStatus(code));
+ },
+ end(timeMs) {
+ if (finished) return;
+ finished = true;
+ safely(() => writer?.end(timeMs));
+ },
+ run(execute) {
+ return finished
+ ? execute()
+ : runTraceContext(
+ input.telemetry,
+ { type: span.type, reference, capture },
+ execute,
+ host,
+ );
+ },
+ };
+ }
+ return {
+ start,
+ startReserved: (
+ span: PreparedSpan,
+ reference: TraceReference,
+ capture: CaptureDecision,
+ host?: ExecutionContext,
+ ) => start(span, capture, host, reference),
+ };
+}
diff --git a/packages/tracing/src/delegation.integration.test.ts b/packages/tracing/src/delegation.integration.test.ts
new file mode 100644
index 000000000..4d3c3d607
--- /dev/null
+++ b/packages/tracing/src/delegation.integration.test.ts
@@ -0,0 +1,191 @@
+import { describe, expect, it } from "vitest";
+import {
+ BasicTracerProvider,
+ InMemorySpanExporter,
+ SimpleSpanProcessor,
+} from "@opentelemetry/sdk-trace-base";
+import { context, propagation, trace } from "@opentelemetry/api";
+import { createAgentTracing, otelTelemetry } from "@eve/tracing";
+import { createAgentDelegationTransport } from "@eve/tracing/delegation";
+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
+
+describe("agent delegation", () => {
+ it("inherits local lineage across awaits and isolates parallel calls", async () => {
+ const exporter = new InMemorySpanExporter();
+ const otel = { spanProcessors: [new SimpleSpanProcessor(exporter)] };
+ const provider = new BasicTracerProvider(otel);
+ context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
+ const parent = createAgentTracing({
+ telemetry: otelTelemetry({ provider }),
+ });
+ const child = createAgentTracing({
+ telemetry: otelTelemetry({ provider }),
+ });
+ try {
+ await parent.turn(
+ {
+ agentName: "parent",
+ identity: { conversationId: "conversation", runId: "parent-run", turnId: "parent-turn" },
+ sequence: 0,
+ },
+ (turn) =>
+ turn.attempt({ stepIndex: 0, attempt: 0 }, (attempt) =>
+ Promise.all(
+ ["alice", "bob"].map((callId) =>
+ attempt.callSubAgent({ callId, agentName: "child" }, async () => {
+ await Promise.resolve();
+ await child.turn(
+ {
+ agentName: "child",
+ identity: { conversationId: "ignored", runId: callId, turnId: "first" },
+ sequence: 0,
+ },
+ async () => "done",
+ );
+ await child.turn(
+ {
+ agentName: "child",
+ identity: { conversationId: "ignored", runId: callId, turnId: "second" },
+ sequence: 1,
+ },
+ async () => "done",
+ );
+ }),
+ ),
+ ),
+ ),
+ );
+ const spans = exporter.getFinishedSpans();
+ const calls = spans.filter((span) => span.name === "execute_tool child");
+ expect(calls).toHaveLength(2);
+ for (const callId of ["alice", "bob"]) {
+ const turns = spans.filter(
+ (span) =>
+ span.name === "invoke_agent child" && span.attributes["agent.run.id"] === callId,
+ );
+ expect(turns).toHaveLength(2);
+ const first = turns.find((span) => span.attributes["agent.turn.sequence"] === 0)!;
+ const second = turns.find((span) => span.attributes["agent.turn.sequence"] === 1)!;
+ expect(first.attributes).toMatchObject({
+ "gen_ai.conversation.id": "conversation",
+ "agent.parent_run.id": "parent-run",
+ });
+ // A local child's first turn nests under the call that delegated to it.
+ const call = calls.find((span) => span.attributes["gen_ai.tool.call.id"] === callId)!;
+ expect(first.parentSpanContext?.spanId).toBe(call.spanContext().spanId);
+ expect(first.spanContext().traceId).toBe(call.spanContext().traceId);
+ expect(first.links).toHaveLength(0);
+ expect(second.parentSpanContext).toBeUndefined();
+ expect(second.links).toHaveLength(0);
+ }
+ } finally {
+ await parent.shutdown();
+ trace.disable();
+ propagation.disable();
+ context.disable();
+ }
+ });
+
+ it("injects only configured transport calls and accepts only trusted remote handoffs", async () => {
+ const exporter = new InMemorySpanExporter();
+ const otel = { spanProcessors: [new SimpleSpanProcessor(exporter)] };
+ const provider = new BasicTracerProvider(otel);
+ context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
+ const caller = createAgentTracing({
+ telemetry: otelTelemetry({ provider }),
+ });
+ const callee = createAgentTracing({
+ telemetry: otelTelemetry({ provider }),
+ });
+ const transport = createAgentDelegationTransport();
+ let sent = new Headers();
+ const fetcher = transport.transport(async (_request, init) => {
+ sent = new Headers(init?.headers);
+ return transport.receive(
+ sent,
+ () => true,
+ async () => {
+ await callee.turn(
+ {
+ agentName: "callee",
+ identity: { conversationId: "local", runId: "remote-run", turnId: "remote-turn" },
+ sequence: 0,
+ capture: { emit: true, recordInputs: true, recordOutputs: true },
+ },
+ async (turn) => {
+ expect(turn.capture.recordOutputs).toBe(false);
+ },
+ );
+ return new Response("done");
+ },
+ );
+ });
+ try {
+ await caller.turn(
+ {
+ agentName: "caller",
+ identity: { conversationId: "conversation", runId: "caller-run", turnId: "turn" },
+ sequence: 0,
+ },
+ (turn) =>
+ turn.attempt({ stepIndex: 0, attempt: 0 }, (attempt) =>
+ attempt.callRemoteAgent({ callId: "remote", agentName: "callee" }, () =>
+ fetcher("https://approved.example/agent", { headers: { "x-app": "kept" } }),
+ ),
+ ),
+ );
+ expect(sent.get("x-app")).toBe("kept");
+ expect(sent.has("x-agent-tracing")).toBe(true);
+ expect(
+ await transport
+ .transport(async (_request, init) => {
+ expect(new Headers(init?.headers).has("x-agent-tracing")).toBe(false);
+ return new Response("independent");
+ })("https://approved.example/agent")
+ .then((response) => response.text()),
+ ).toBe("independent");
+ const spans = exporter.getFinishedSpans();
+ const call = spans.find((span) => span.name === "execute_tool callee")!;
+ expect(call.kind).toBe(2);
+ const remoteTurn = spans.find((span) => span.name === "invoke_agent callee")!;
+ expect(remoteTurn.parentSpanContext).toBeUndefined();
+ expect(remoteTurn.links[0]?.context.spanId).toBe(call.spanContext().spanId);
+ const rejection = new Error("original");
+ let executions = 0;
+ expect(() =>
+ transport.receive(
+ sent,
+ () => false,
+ () => {
+ executions++;
+ throw rejection;
+ },
+ ),
+ ).toThrow(rejection);
+ expect(executions).toBe(1);
+ await transport.receive(
+ new Headers({ "x-agent-tracing": "invalid" }),
+ () => true,
+ () =>
+ callee.turn(
+ {
+ agentName: "callee",
+ identity: { conversationId: "independent", runId: "independent", turnId: "turn" },
+ sequence: 0,
+ },
+ async () => undefined,
+ ),
+ );
+ expect(
+ exporter
+ .getFinishedSpans()
+ .find((span) => span.attributes["agent.run.id"] === "independent")?.links,
+ ).toHaveLength(0);
+ } finally {
+ await caller.shutdown();
+ trace.disable();
+ propagation.disable();
+ context.disable();
+ }
+ });
+});
diff --git a/packages/tracing/src/delegation.ts b/packages/tracing/src/delegation.ts
new file mode 100644
index 000000000..20d4574a1
--- /dev/null
+++ b/packages/tracing/src/delegation.ts
@@ -0,0 +1,2 @@
+export { createAgentDelegationTransport } from "./adapters/agent-transport.js";
+export { currentAgentHandoff, withAgentHandoff, type AgentHandoff } from "./core/delegation.js";
diff --git a/packages/tracing/src/durable.integration.test.ts b/packages/tracing/src/durable.integration.test.ts
new file mode 100644
index 000000000..38703a8a9
--- /dev/null
+++ b/packages/tracing/src/durable.integration.test.ts
@@ -0,0 +1,417 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { context } from "@opentelemetry/api";
+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
+import {
+ BasicTracerProvider,
+ InMemorySpanExporter,
+ SimpleSpanProcessor,
+} from "@opentelemetry/sdk-trace-base";
+import {
+ AgentSpanIdGenerator,
+ createAgentTracing,
+ otelTelemetry,
+ type TraceCheckpointer,
+} from "@eve/tracing";
+
+const identity = { conversationId: "conversation", runId: "run", turnId: "turn" };
+
+function memoryCheckpointer() {
+ const entries = new Map<string, string>();
+ const checkpointer: TraceCheckpointer = {
+ get: (key) => {
+ const value = entries.get(key);
+ return value === undefined ? undefined : JSON.parse(value);
+ },
+ set: (key, value) => void entries.set(key, JSON.stringify(value)),
+ delete: (key) => void entries.delete(key),
+ };
+ return { checkpointer, entries };
+}
+
+function worker(checkpointer: TraceCheckpointer, exporter: InMemorySpanExporter) {
+ const idGenerator = new AgentSpanIdGenerator();
+ const provider = new BasicTracerProvider({
+ idGenerator,
+ spanProcessors: [new SimpleSpanProcessor(exporter)],
+ });
+ return createAgentTracing({
+ telemetry: otelTelemetry({ provider, idGenerator }),
+ checkpointer,
+ });
+}
+
+describe("durable agent tracing", () => {
+ it("continues a turn in a new worker by calling the same operations again", async () => {
+ const { checkpointer, entries } = memoryCheckpointer();
+ const exporter = new InMemorySpanExporter();
+
+ const first = worker(checkpointer, exporter);
+ const turn = await first.turn({ agentName: "support", identity, sequence: 0 });
+ const attempt = await turn.attempt({ stepIndex: 0, attempt: 0 });
+ await attempt.modelCall({ provider: "test", modelId: "model" }, () => ({
+ result: "call lookup",
+ finishReason: "tool-calls",
+ usage: { inputTokens: 3, outputTokens: 2 },
+ }));
+ const tool = await attempt.tool({ callId: "lookup", name: "lookup" });
+ await tool.approval({ requestId: "approval" });
+ expect(entries.size).toBe(1);
+
+ // The first worker is lost while Alice reviews the approval.
+ const second = worker(checkpointer, exporter);
+ const resumedTurn = await second.turn({ agentName: "support", identity, sequence: 0 });
+ const resumedAttempt = await resumedTurn.attempt({ stepIndex: 0, attempt: 0 });
+ const resumedTool = await resumedAttempt.tool({ callId: "lookup", name: "lookup" });
+ const approval = await resumedTool.approval({ requestId: "approval" });
+ expect(resumedTurn.reference).toEqual(turn.reference);
+ expect(resumedTool.reference).toEqual(tool.reference);
+
+ await approval.complete({ outcome: "approved" });
+ await resumedTool.complete({ outcome: "completed", output: "Alice's answer" });
+ await resumedAttempt.complete();
+ await resumedTurn.complete();
+ await second.forceFlush();
+
+ const spans = exporter.getFinishedSpans();
+ expect(spans.map((span) => span.name)).toEqual([
+ "chat model",
+ "agent.approval",
+ "execute_tool lookup",
+ "agent.step",
+ "invoke_agent support",
+ ]);
+ const byName = new Map(spans.map((span) => [span.name, span]));
+ const root = byName.get("invoke_agent support")!;
+ expect(root.spanContext().spanId).toBe(turn.reference.spanId);
+ expect(byName.get("agent.step")!.parentSpanContext?.spanId).toBe(turn.reference.spanId);
+ expect(byName.get("agent.approval")!.attributes["agent.approval.outcome"]).toBe("approved");
+ expect(root.attributes["gen_ai.usage.input_tokens"]).toBe(3);
+ expect(new Set(spans.map((span) => span.spanContext().traceId)).size).toBe(1);
+ expect(entries.size).toBe(0);
+ });
+
+ it("keeps a tool call open after its attempt and the turn complete", async () => {
+ const { checkpointer, entries } = memoryCheckpointer();
+ const exporter = new InMemorySpanExporter();
+ const first = worker(checkpointer, exporter);
+ const turn = await first.turn({ agentName: "support", identity, sequence: 0 });
+ const attempt = await turn.attempt({ stepIndex: 0, attempt: 0 });
+ const tool = await attempt.tool({ callId: "lookup", name: "lookup" });
+ await tool.approval({ requestId: "approval" });
+ await attempt.complete();
+ await turn.complete();
+ expect(entries.size).toBe(1);
+
+ // Bob approves the request after the turn has parked.
+ const second = worker(checkpointer, exporter);
+ expect(await second.resume({ identity: { ...identity, turnId: "other" } })).toBeUndefined();
+ const resumed = await second.resume({ identity });
+ expect(resumed!.findAttempt({ stepIndex: 0, attempt: 0 })?.reference).toEqual(
+ attempt.reference,
+ );
+ expect(resumed!.findTool("missing")).toBeUndefined();
+ const resumedTool = resumed!.findTool("lookup")!;
+ await resumedTool.findApproval("approval")!.complete({ outcome: "approved" });
+ expect(resumedTool.findApproval("approval")).toBeUndefined();
+ await resumedTool.complete();
+ expect(resumed!.findTool("lookup")).toBeUndefined();
+ await second.forceFlush();
+
+ const spans = exporter.getFinishedSpans();
+ expect(spans.map((span) => span.name)).toEqual([
+ "agent.step",
+ "invoke_agent support",
+ "agent.approval",
+ "execute_tool lookup",
+ ]);
+ expect(spans[3]!.parentSpanContext?.spanId).toBe(attempt.reference.spanId);
+ expect(entries.size).toBe(0);
+ });
+
+ it("merges a tool call reported twice into one span", async () => {
+ const { checkpointer } = memoryCheckpointer();
+ const exporter = new InMemorySpanExporter();
+ const tracing = worker(checkpointer, exporter);
+ const turn = await tracing.turn({ agentName: "support", identity, sequence: 0 });
+ const attempt = await turn.attempt({ stepIndex: 0, attempt: 0 });
+ // The SDK runs the tool before the runtime records its dispatch.
+ const requestedAt = Date.now() - 60_000;
+ const executed = await attempt.tool({
+ callId: "lookup",
+ name: "lookup",
+ startTimeMs: requestedAt + 1000,
+ });
+ const dispatched = await attempt.tool({
+ callId: "lookup",
+ name: "lookup",
+ kind: "subagent-call",
+ startTimeMs: requestedAt,
+ });
+ expect(dispatched.reference).toEqual(executed.reference);
+ const nested = await dispatched.tool({ callId: "inner", name: "inner" });
+ await nested.complete();
+ await dispatched.complete({ outcome: "completed" });
+ await attempt.complete();
+ await turn.complete();
+ await tracing.forceFlush();
+
+ const spans = new Map(exporter.getFinishedSpans().map((span) => [span.name, span]));
+ const tool = spans.get("execute_tool lookup")!;
+ expect(tool.startTime[0] * 1000 + tool.startTime[1] / 1e6).toBeCloseTo(requestedAt, -1);
+ expect(tool.attributes).toMatchObject({
+ "agent.tool.kind": "subagent-call",
+ "agent.tool.outcome": "completed",
+ "agent.invocation.role": "caller",
+ });
+ expect(spans.get("execute_tool inner")!.parentSpanContext?.spanId).toBe(
+ tool.spanContext().spanId,
+ );
+ });
+
+ it("adopts a reserved turn reference and applies sampling", async () => {
+ const { checkpointer } = memoryCheckpointer();
+ const exporter = new InMemorySpanExporter();
+ const idGenerator = new AgentSpanIdGenerator();
+ const tracing = createAgentTracing({
+ telemetry: otelTelemetry({
+ provider: new BasicTracerProvider({
+ idGenerator,
+ spanProcessors: [new SimpleSpanProcessor(exporter)],
+ }),
+ idGenerator,
+ samplesTrace: (traceId) => traceId !== "b".repeat(32),
+ }),
+ checkpointer,
+ });
+ const reference = { traceId: "a".repeat(32), spanId: "c".repeat(16), traceFlags: 1 };
+ const turn = await tracing.turn({ agentName: "support", identity, sequence: 0, reference });
+ expect(turn.reference).toMatchObject(reference);
+ const dropped = await tracing.turn({
+ agentName: "support",
+ identity: { ...identity, turnId: "dropped" },
+ sequence: 1,
+ reference: { ...reference, traceId: "b".repeat(32) },
+ });
+ expect(dropped.reference.traceFlags).toBe(0);
+ });
+
+ it("starts a fresh turn when the checkpoint is unusable", async () => {
+ const { checkpointer, entries } = memoryCheckpointer();
+ const exporter = new InMemorySpanExporter();
+ const errors: string[] = [];
+ const idGenerator = new AgentSpanIdGenerator();
+ const tracing = createAgentTracing({
+ telemetry: otelTelemetry({
+ provider: new BasicTracerProvider({
+ idGenerator,
+ spanProcessors: [new SimpleSpanProcessor(exporter)],
+ }),
+ idGenerator,
+ }),
+ checkpointer: {
+ ...checkpointer,
+ get: () => ({ version: 1, key: "corrupt" }),
+ },
+ onError: (_error, context) => errors.push(context.phase),
+ });
+ const turn = await tracing.turn({ agentName: "support", identity, sequence: 0 });
+ await turn.complete();
+ expect(errors).toEqual(["restore"]);
+ expect(exporter.getFinishedSpans().map((span) => span.name)).toEqual(["invoke_agent support"]);
+ expect(entries.size).toBe(0);
+ });
+
+ it("requires stable span IDs", () => {
+ expect(() =>
+ createAgentTracing({
+ telemetry: otelTelemetry({ provider: new BasicTracerProvider() }),
+ checkpointer: memoryCheckpointer().checkpointer,
+ }),
+ ).toThrow(/AgentSpanIdGenerator/u);
+ });
+});
+
+describe("content capture after a narrower decision", () => {
+ const ALL = { emit: true, recordInputs: true, recordOutputs: true } as const;
+ const NO_OUTPUTS = { emit: true, recordInputs: true, recordOutputs: false } as const;
+ const NO_INPUTS = { emit: true, recordInputs: false, recordOutputs: true } as const;
+ const METADATA = { emit: true, recordInputs: false, recordOutputs: false } as const;
+ // Run ceilings reach nested work through the active context.
+ beforeEach(() => {
+ context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
+ });
+ afterEach(() => {
+ context.disable();
+ });
+
+ async function setup() {
+ const { checkpointer, entries } = memoryCheckpointer();
+ const exporter = new InMemorySpanExporter();
+ const tracing = worker(checkpointer, exporter);
+ const turn = await tracing.turn({ agentName: "support", identity, sequence: 0, capture: ALL });
+ const attempt = await turn.attempt({ stepIndex: 0, attempt: 0 });
+ const exported = async () => {
+ await tracing.forceFlush();
+ const spans = exporter
+ .getFinishedSpans()
+ .map((span) => ({ attributes: span.attributes, events: span.events }));
+ return JSON.stringify(spans) + JSON.stringify([...entries.values()]);
+ };
+ return { tracing, turn, attempt, exported };
+ }
+
+ it("keeps an existing tool's failure out when its parent's run declines outputs", async () => {
+ const { turn, attempt, exported } = await setup();
+ const tool = await attempt.tool({ callId: "lookup", name: "lookup" });
+ await attempt.run(() => tool.fail(new Error("Alice's private note")), NO_OUTPUTS);
+ await attempt.complete();
+ await turn.complete();
+ expect(await exported()).not.toContain("private note");
+ });
+
+ it("keeps an error recorded inside an output-declined run out", async () => {
+ const { turn, attempt, exported } = await setup();
+ const tool = await attempt.tool({ callId: "lookup", name: "lookup" });
+ tool.run(() => tool.recordError(new Error("Alice's private note")), NO_OUTPUTS);
+ await tool.complete();
+ await attempt.complete();
+ await turn.complete();
+ expect(await exported()).not.toContain("private note");
+ });
+
+ it("drops arguments a second report adds inside an input-declined run", async () => {
+ const { turn, attempt, exported } = await setup();
+ const tool = await attempt.tool({ callId: "lookup", name: "lookup" });
+ await attempt.run(
+ () =>
+ attempt.tool({ callId: "lookup", name: "lookup", arguments: { note: "Bob's address" } }),
+ NO_INPUTS,
+ );
+ expect(await exported()).not.toContain("Bob's address");
+ await tool.complete();
+ await attempt.complete();
+ await turn.complete();
+ expect(await exported()).not.toContain("Bob's address");
+ });
+
+ it("holds an existing tool's result to its parent's narrower run", async () => {
+ const { turn, attempt, exported } = await setup();
+ const tool = await attempt.tool({ callId: "lookup", name: "lookup" });
+ await attempt.run(() => tool.complete({ output: "Alice's private note" }), NO_OUTPUTS);
+ await attempt.complete();
+ await turn.complete();
+ expect(await exported()).not.toContain("private note");
+ });
+
+ it("narrows a cached turn re-entered with less capture", async () => {
+ const { tracing, exported } = await setup();
+ const reentered = await tracing.turn({
+ agentName: "support",
+ identity,
+ sequence: 0,
+ capture: METADATA,
+ });
+ const attempt = reentered.findAttempt({ stepIndex: 0, attempt: 0 })!;
+ await attempt.tool(
+ {
+ callId: "lookup",
+ name: "lookup",
+ arguments: { note: "Bob's address" },
+ describe: (output: string) => ({ outcome: "completed", output }),
+ },
+ () => "Alice's private note",
+ );
+ await attempt.complete();
+ await reentered.complete();
+ const output = await exported();
+ expect(output).not.toContain("Bob's address");
+ expect(output).not.toContain("private note");
+ });
+});
+
+describe("turn outcome", () => {
+ it("ends a turn without an outcome when the host never learned one", async () => {
+ const exporter = new InMemorySpanExporter();
+ const tracing = worker(memoryCheckpointer().checkpointer, exporter);
+ const waiting = await tracing.turn({ agentName: "support", identity, sequence: 0 });
+ await waiting.complete({ outcomeUnknown: true });
+ const failed = await tracing.turn({
+ agentName: "support",
+ identity: { ...identity, turnId: "failed" },
+ sequence: 1,
+ });
+ await failed.complete({ outcomeUnknown: true, failed: true });
+ await tracing.forceFlush();
+
+ const [first, second] = exporter.getFinishedSpans();
+ expect(first!.attributes["agent.turn.outcome"]).toBeUndefined();
+ expect(first!.events.map((event) => event.name)).toEqual(["turn.started"]);
+ expect(second!.attributes["agent.turn.outcome"]).toBe("failed");
+ expect(second!.events.map((event) => event.name)).toEqual(["turn.started", "turn.failed"]);
+ });
+});
+
+describe("turn content", () => {
+ it("records the first model input and the latest output on the turn", async () => {
+ const exporter = new InMemorySpanExporter();
+ const tracing = worker(memoryCheckpointer().checkpointer, exporter);
+ const all = { emit: true, recordInputs: true, recordOutputs: true };
+ const turn = await tracing.turn({ agentName: "support", identity, sequence: 0, capture: all });
+ for (const [stepIndex, prompt, answer] of [
+ [0, "Alice asks for the forecast", "calling weather"],
+ [1, "weather returned sunny", "It is sunny"],
+ ] as const) {
+ const attempt = await turn.attempt({ stepIndex, attempt: 0 });
+ await attempt.modelCall(
+ {
+ provider: "test",
+ modelId: "model",
+ messages: [{ role: "user", content: [{ type: "text", text: prompt }] }],
+ },
+ () => ({
+ result: answer,
+ finishReason: "stop",
+ usage: { inputTokens: 1, outputTokens: 1 },
+ content: [{ type: "text" as const, text: answer }],
+ }),
+ );
+ await attempt.complete();
+ }
+ await turn.complete();
+ await tracing.forceFlush();
+
+ const spans = exporter.getFinishedSpans();
+ const invocation = spans.find((span) => span.name === "invoke_agent support")!;
+ expect(invocation.attributes["gen_ai.input.messages"]).toContain("Alice asks for the forecast");
+ expect(invocation.attributes["gen_ai.input.messages"]).not.toContain("weather returned");
+ expect(invocation.attributes["gen_ai.output.messages"]).toContain("It is sunny");
+ expect(invocation.attributes["gen_ai.output.messages"]).not.toContain("calling weather");
+ const step = spans.find((span) => span.name === "agent.step")!;
+ expect(step.attributes).toMatchObject({
+ "gen_ai.operation.name": "workflow",
+ "operation.name": "workflow",
+ });
+ });
+
+ it("leaves turn content out when capture declines it", async () => {
+ const exporter = new InMemorySpanExporter();
+ const tracing = worker(memoryCheckpointer().checkpointer, exporter);
+ const turn = await tracing.turn({ agentName: "support", identity, sequence: 0 });
+ const attempt = await turn.attempt({ stepIndex: 0, attempt: 0 });
+ await attempt.modelCall(
+ { provider: "test", modelId: "model", messages: [{ role: "user", content: "Alice" }] },
+ () => ({
+ result: "Bob",
+ finishReason: "stop",
+ usage: { inputTokens: 1, outputTokens: 1 },
+ content: [{ type: "text" as const, text: "Bob" }],
+ }),
+ );
+ await attempt.complete();
+ await turn.complete();
+ await tracing.forceFlush();
+ expect(JSON.stringify(exporter.getFinishedSpans().map((span) => span.attributes))).not.toMatch(
+ /Alice|Bob/,
+ );
+ });
+});
diff --git a/packages/tracing/src/index.ts b/packages/tracing/src/index.ts
new file mode 100644
index 000000000..13447bfb3
--- /dev/null
+++ b/packages/tracing/src/index.ts
@@ -0,0 +1,44 @@
+export {
+ createAgentTracing,
+ type AgentTracing,
+ type AgentTracingOptions,
+ type AgentMemoryTracing,
+ type TurnInput,
+ type ResumeInput,
+} from "./agent-tracing.js";
+export { otelTelemetry, type OtelTelemetryOptions } from "./adapters/otel.js";
+export { AgentSpanIdGenerator } from "./adapters/otel-ids.js";
+export type {
+ Operation,
+ TurnOperation,
+ AttemptInput,
+ AttemptOperation,
+ ToolInput,
+ ModelOperation,
+ ToolOperation,
+ ApprovalOperation,
+ MemoryOperation,
+ ModelCallReturn,
+ ModelStreamReturn,
+} from "./operations.js";
+export type {
+ ActiveOperation,
+ AgentTelemetry,
+ Attributes,
+ CaptureDecision,
+ ContentPart,
+ ContentSerializer,
+ ExecutionContext,
+ MappingContext,
+ OutputMapping,
+ PreparedSpan,
+ SpanWriter,
+ TraceCheckpointer,
+ TraceErrorContext,
+ TraceErrorHandler,
+ TraceLink,
+ TraceReference,
+ TraceSnapshot,
+ Usage,
+} from "./core/types.js";
+export { currentCapture } from "./capture.js";
diff --git a/packages/tracing/src/model-execution.integration.test.ts b/packages/tracing/src/model-execution.integration.test.ts
new file mode 100644
index 000000000..174cb4325
--- /dev/null
+++ b/packages/tracing/src/model-execution.integration.test.ts
@@ -0,0 +1,80 @@
+import { describe, expect, it } from "vitest";
+import {
+ BasicTracerProvider,
+ InMemorySpanExporter,
+ SimpleSpanProcessor,
+} from "@opentelemetry/sdk-trace-base";
+import { createAgentTracing, otelTelemetry } from "./index.js";
+
+describe("wrapped model streams", () => {
+ it.each(["completed", "failed", "cancelled"])(
+ "preserves the stream and waits for %s completion",
+ async (outcome) => {
+ const exporter = new InMemorySpanExporter();
+ const provider = new BasicTracerProvider({
+ spanProcessors: [new SimpleSpanProcessor(exporter)],
+ });
+ const tracing = createAgentTracing({
+ telemetry: otelTelemetry({ provider }),
+ });
+ try {
+ const turn = await tracing.turn({
+ agentName: "stream",
+ identity: { conversationId: "c", runId: "r", turnId: "t" },
+ sequence: 0,
+ });
+ const attempt = await turn.attempt({ stepIndex: 0, attempt: 0 });
+ const completion = Promise.withResolvers<{
+ finishReason: string;
+ usage: { inputTokens: number };
+ }>();
+ let pulls = 0;
+ const failure = new TypeError("private");
+ const source = new ReadableStream<string>(
+ {
+ pull(controller) {
+ pulls++;
+ if (outcome === "failed") {
+ controller.error(failure);
+ completion.reject(failure);
+ } else {
+ controller.enqueue("answer");
+ controller.close();
+ completion.resolve({ finishReason: "stop", usage: { inputTokens: 3 } });
+ }
+ },
+ cancel() {
+ completion.reject(new DOMException("Cancelled", "AbortError"));
+ },
+ },
+ { highWaterMark: 0 },
+ );
+ const stream = await attempt.modelStream({ provider: "test", modelId: "model" }, () => ({
+ result: source,
+ completion: completion.promise,
+ }));
+ expect(stream).toBe(source);
+ expect(pulls).toBe(0);
+ let ended = false;
+ const ending = turn.complete().then(() => {
+ ended = true;
+ });
+ await Promise.resolve();
+ expect(ended).toBe(false);
+ if (outcome === "cancelled") await stream.cancel();
+ else if (outcome === "failed")
+ await expect(stream.getReader().read()).rejects.toBe(failure);
+ else expect(await stream.getReader().read()).toMatchObject({ value: "answer" });
+ await ending;
+ expect(exporter.getFinishedSpans().map((span) => span.name)).toEqual([
+ "chat model",
+ "agent.step",
+ "invoke_agent stream",
+ ]);
+ expect(exporter.getFinishedSpans()[0]!.status.code).toBe(outcome === "completed" ? 0 : 2);
+ } finally {
+ await tracing.shutdown();
+ }
+ },
+ );
+});
diff --git a/packages/tracing/src/operations.ts b/packages/tracing/src/operations.ts
new file mode 100644
index 000000000..5f4ecc122
--- /dev/null
+++ b/packages/tracing/src/operations.ts
@@ -0,0 +1,342 @@
+import type { LiveOperation } from "./core/scopes.js";
+import type {
+ StepOptions,
+ TraceLink,
+ ScopeData,
+ ScopeTerminal,
+ ModelOptions,
+ ToolOptions,
+ ModelResult,
+ CaptureDecision,
+ Attributes,
+ TraceReference,
+ TraceErrorHandler,
+} from "./core/types.js";
+
+type Completion = ScopeTerminal & { errorType?: string; result?: ModelResult };
+export interface Operation {
+ readonly type: ScopeData["type"];
+ readonly reference: TraceReference;
+ readonly parent?: TraceReference;
+ readonly startTimeMs: number;
+ readonly finished: boolean;
+ readonly capture: CaptureDecision;
+ run<T>(execute: () => T, ceiling?: CaptureDecision): T;
+ attributes(attributes: Attributes): void;
+ complete(result?: Completion): Promise<void>;
+ fail(error: unknown): Promise<void>;
+ /** Records a failure without ending the operation; completion keeps it. */
+ recordError(error: unknown, errorType?: string): void;
+}
+export interface WrappedOperation<I, H extends Operation> {
+ (input: I): Promise<H>;
+ <T>(
+ input: I & { describe?: (value: T) => Completion },
+ execute: (operation: H) => T | PromiseLike<T>,
+ ): Promise<T>;
+}
+type MemoryInput = Extract<ScopeData, { type: "memory" }>["options"];
+export interface AttemptInput {
+ readonly stepIndex: number;
+ readonly attempt: number;
+ readonly runtimeContext?: StepOptions["runtimeContext"];
+ readonly channel?: StepOptions["channel"];
+ readonly links?: readonly TraceLink[];
+}
+export interface TurnOperation extends Operation {
+ attempt: WrappedOperation<AttemptInput, AttemptOperation>;
+ /** Returns an open attempt without starting one. Durable turns only. */
+ findAttempt(input: { stepIndex: number; attempt: number }): AttemptOperation | undefined;
+ /** Returns an open tool call from any attempt of this turn. Durable turns only. */
+ findTool(callId: string): ToolOperation | undefined;
+ /** Replaces the turn's links before its deferred span is exported. */
+ links(links: readonly TraceLink[]): void;
+ memory: WrappedOperation<MemoryInput, MemoryOperation>;
+}
+/**
+ * One tool call. In a durable turn, calling `tool` again with the same `callId`
+ * returns the same call and fills in details it did not have, such as `kind`,
+ * `arguments`, or an earlier `startTimeMs`.
+ */
+export interface ToolInput extends ToolOptions {
+ readonly startTimeMs?: number;
+}
+export interface AttemptOperation extends Operation {
+ tool: WrappedOperation<ToolInput, ToolOperation>;
+ callSubAgent: WrappedOperation<{ callId: string; agentName: string }, ToolOperation>;
+ callRemoteAgent: WrappedOperation<{ callId: string; agentName: string }, ToolOperation>;
+ modelCall(input: ModelOptions, key?: string): Promise<ModelOperation>;
+ modelCall<T>(
+ input: ModelOptions,
+ execute: () => ModelCallReturn<T> | PromiseLike<ModelCallReturn<T>>,
+ ): Promise<T>;
+ modelStream<T>(
+ input: ModelOptions,
+ execute: () => ModelStreamReturn<T> | PromiseLike<ModelStreamReturn<T>>,
+ ): Promise<T>;
+ memory: WrappedOperation<MemoryInput, MemoryOperation>;
+}
+export interface ToolOperation extends Operation {
+ approval: WrappedOperation<{ requestId: string; request?: unknown }, ApprovalOperation>;
+ /** Returns an open approval without starting one. Durable turns only. */
+ findApproval(requestId: string): ApprovalOperation | undefined;
+ /** A tool call made while this tool runs, such as a connector's nested call. */
+ tool: WrappedOperation<ToolInput, ToolOperation>;
+ memory: WrappedOperation<MemoryInput, MemoryOperation>;
+}
+export type ModelOperation = Operation;
+export type ApprovalOperation = Operation;
+export type MemoryOperation = Operation;
+export interface ModelCallReturn<T> extends ModelResult {
+ readonly result: T;
+}
+export interface ModelStreamReturn<T> {
+ readonly result: T;
+ readonly completion: PromiseLike<ModelResult>;
+}
+
+function report(
+ onError: TraceErrorHandler | undefined,
+ error: unknown,
+ phase: "start" | "complete",
+) {
+ try {
+ onError?.(error, { phase });
+ } catch {}
+}
+export function wrappedOperation<I, H extends Operation>(
+ start: (input: I) => Promise<H>,
+ fallback: Completion = { outcome: "completed" },
+ onError?: TraceErrorHandler,
+): WrappedOperation<I, H> {
+ function invoke(input: I): Promise<H>;
+ function invoke<T>(
+ input: I & { describe?: (value: T) => Completion },
+ execute: (operation: H) => T | PromiseLike<T>,
+ ): Promise<T>;
+ async function invoke<T>(
+ input: I,
+ execute?: (operation: H) => T | PromiseLike<T>,
+ ): Promise<H | T> {
+ if (execute === undefined) return start(input);
+ const handle = await start(input);
+ let completion = fallback;
+ try {
+ const value = await handle.run(() => execute(handle));
+ try {
+ completion =
+ (input as I & { describe?: (value: T) => Completion }).describe?.(value) ?? fallback;
+ } catch (error) {
+ report(onError, error, "complete");
+ }
+ return value;
+ } catch (error) {
+ completion = {
+ outcome: "failed",
+ errorType: error instanceof Error ? error.name : "_OTHER",
+ error,
+ };
+ throw error;
+ } finally {
+ try {
+ await handle.complete(completion);
+ } catch (error) {
+ report(onError, error, "complete");
+ }
+ }
+ }
+ return invoke;
+}
+
+export function operationHandle(
+ runtime: LiveOperation,
+ data: ScopeData,
+ onError?: TraceErrorHandler,
+): TurnOperation | AttemptOperation | ToolOperation | Operation {
+ const live = <T extends Operation>(handle: T): T =>
+ Object.defineProperties(handle, {
+ finished: { get: () => runtime.finished },
+ capture: { get: () => runtime.capture },
+ parent: { get: () => runtime.parent },
+ });
+ const common: Operation = {
+ type: runtime.type,
+ reference: runtime.reference,
+ get startTimeMs() {
+ return runtime.startTimeMs;
+ },
+ get parent() {
+ return runtime.parent;
+ },
+ get finished() {
+ return runtime.finished;
+ },
+ get capture() {
+ return runtime.capture;
+ },
+ run: (execute, ceiling) => runtime.run(execute, ceiling),
+ attributes: (attributes) => runtime.attributes(attributes),
+ complete: (result) => runtime.complete(result),
+ fail: (error) => runtime.fail(error),
+ recordError: (error, errorType) => runtime.error(error, errorType),
+ };
+ const child = async (
+ data: ScopeData,
+ key?: string,
+ options?: { links?: readonly TraceLink[]; startTimeMs?: number },
+ ) => operationHandle(await runtime.child(data, key, options), data, onError);
+ const tool = wrappedOperation(
+ async ({ startTimeMs, ...options }: ToolInput) =>
+ child({ type: "tool", options }, undefined, { startTimeMs }) as Promise<ToolOperation>,
+ undefined,
+ onError,
+ );
+ const found = <T>(operation: LiveOperation | undefined): T | undefined =>
+ operation === undefined
+ ? undefined
+ : (operationHandle(operation, operation.record().data, onError) as T);
+ const memory = wrappedOperation(
+ async (options: MemoryInput) => child({ type: "memory", options }),
+ undefined,
+ onError,
+ );
+ if (data.type === "activation")
+ return live({
+ ...common,
+ memory,
+ links: (links: readonly TraceLink[]) => runtime.update({ links }),
+ findAttempt: (input: { stepIndex: number; attempt: number }) =>
+ found<AttemptOperation>(
+ runtime.find({
+ type: "step",
+ options: { index: input.stepIndex, attempt: input.attempt },
+ }),
+ ),
+ findTool: (callId: string) => found<ToolOperation>(findTool(runtime, callId)),
+ attempt: wrappedOperation(
+ async (input: AttemptInput) =>
+ child(
+ {
+ type: "step",
+ options: {
+ index: input.stepIndex,
+ attempt: input.attempt,
+ runtimeContext: input.runtimeContext,
+ channel: input.channel,
+ },
+ },
+ undefined,
+ { links: input.links },
+ ) as Promise<AttemptOperation>,
+ undefined,
+ onError,
+ ),
+ });
+ if (data.type === "step") {
+ const delegation = (kind: string) =>
+ wrappedOperation(
+ async (input: { callId: string; agentName: string }) =>
+ child({
+ type: "tool",
+ options: { callId: input.callId, name: input.agentName, kind },
+ }) as Promise<ToolOperation>,
+ undefined,
+ onError,
+ );
+ const startModel = (options: ModelOptions, key?: string) =>
+ child({ type: "model", options }, key);
+ function modelCall(input: ModelOptions, key?: string): Promise<ModelOperation>;
+ function modelCall<T>(
+ input: ModelOptions,
+ execute: () => ModelCallReturn<T> | PromiseLike<ModelCallReturn<T>>,
+ ): Promise<T>;
+ async function modelCall<T>(
+ input: ModelOptions,
+ execute?: string | (() => ModelCallReturn<T> | PromiseLike<ModelCallReturn<T>>),
+ ): Promise<ModelOperation | T> {
+ if (typeof execute !== "function") return startModel(input, execute);
+ const handle = await startModel(input);
+ let result: ModelCallReturn<T>;
+ try {
+ result = await handle.run(execute);
+ } catch (error) {
+ try {
+ await handle.fail(error);
+ } catch (tracingError) {
+ report(onError, tracingError, "complete");
+ }
+ throw error;
+ }
+ try {
+ await handle.complete({ outcome: "completed", result });
+ } catch (error) {
+ report(onError, error, "complete");
+ }
+ return result.result;
+ }
+ return live({
+ ...common,
+ memory,
+ tool,
+ callSubAgent: delegation("subagent-call"),
+ callRemoteAgent: delegation("remote-agent-call"),
+ modelCall,
+ async modelStream(input, execute) {
+ const handle = await startModel(input);
+ try {
+ const value = await handle.run(execute);
+ const completion = Promise.resolve(value.completion)
+ .then(
+ (result) => handle.complete({ outcome: "completed", result }),
+ (error) => handle.fail(error),
+ )
+ .catch((error) => report(onError, error, "complete"));
+ runtime.waitUntil(completion);
+ return value.result;
+ } catch (error) {
+ try {
+ await handle.fail(error);
+ } catch (tracingError) {
+ report(onError, tracingError, "complete");
+ }
+ throw error;
+ }
+ },
+ } as AttemptOperation);
+ }
+ if (data.type === "tool")
+ return live({
+ ...common,
+ memory,
+ tool,
+ findApproval: (requestId: string) =>
+ found<ApprovalOperation>(
+ runtime.find({
+ type: "approval",
+ options: { requestId, callId: data.options.callId, toolName: data.options.name },
+ }),
+ ),
+ approval: wrappedOperation(
+ async (input: { requestId: string; request?: unknown }) =>
+ child({
+ type: "approval",
+ options: { ...input, callId: data.options.callId, toolName: data.options.name },
+ }),
+ { outcome: "ignored" },
+ onError,
+ ),
+ } as ToolOperation);
+ return common;
+}
+
+/** Tool calls can nest, so a call ID is searched through every open call of the turn. */
+function findTool(parent: LiveOperation, callId: string): LiveOperation | undefined {
+ for (const next of parent.children) {
+ if (next.type === "tool" && next.callId === callId) return next;
+ if (next.type === "step" || next.type === "tool") {
+ const found = findTool(next, callId);
+ if (found !== undefined) return found;
+ }
+ }
+ return undefined;
+}
diff --git a/packages/tracing/src/otel.ts b/packages/tracing/src/otel.ts
new file mode 100644
index 000000000..ffd82de1c
--- /dev/null
+++ b/packages/tracing/src/otel.ts
@@ -0,0 +1,18 @@
+/**
+ * OpenTelemetry plumbing for hosts that run their own spans beside agent
+ * tracing: active-operation lookup, capture context, content bounds, and MCP
+ * enrichment. Agent tracing itself needs only the package root.
+ */
+export { activeTraceOperation, withErrorContent } from "./adapters/otel.js";
+export { withCapture } from "./capture.js";
+export {
+ boundedPrincipalId,
+ contentAttribute,
+ telemetryByteLength,
+ truncateTelemetryText,
+ CONTENT_ATTRIBUTE_LIMIT,
+} from "./adapters/serialization.js";
+export { withoutDeclinedContent, type ResolvedContentOptions } from "./core/content-policy.js";
+export { mcpLifecycle, type McpLifecycle, type McpUpdate } from "./core/mcp.js";
+export { invocationName } from "./core/span-kinds.js";
+export { USAGE_FIELDS } from "./core/attributes.js";
diff --git a/packages/tracing/tsconfig.build.json b/packages/tracing/tsconfig.build.json
new file mode 100644
index 000000000..8886e590a
--- /dev/null
+++ b/packages/tracing/tsconfig.build.json
@@ -0,0 +1,6 @@
+{
+ "$schema": "https://json.schemastore.org/tsconfig",
+ "extends": "./tsconfig.json",
+ "include": ["src/**/*.ts"],
+ "exclude": ["src/**/*.test.ts"]
+}
diff --git a/packages/tracing/tsconfig.json b/packages/tracing/tsconfig.json
new file mode 100644
index 000000000..87c988523
--- /dev/null
+++ b/packages/tracing/tsconfig.json
@@ -0,0 +1,17 @@
+{
+ "$schema": "https://json.schemastore.org/tsconfig",
+ "extends": "../../tsconfig.json",
+ "compilerOptions": {
+ "lib": ["ES2024"],
+ "types": ["node"],
+ "rootDir": ".",
+ "outDir": "./dist",
+ "declaration": true,
+ "declarationMap": true,
+ "sourceMap": true,
+ "noEmit": false,
+ "noEmitOnError": true
+ },
+ "include": ["src/**/*.ts", "vitest.*.config.ts"],
+ "exclude": []
+}
diff --git a/packages/tracing/vitest.integration.config.ts b/packages/tracing/vitest.integration.config.ts
new file mode 100644
index 000000000..c9333f208
--- /dev/null
+++ b/packages/tracing/vitest.integration.config.ts
@@ -0,0 +1,8 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+ test: {
+ environment: "node",
+ include: ["src/**/*.integration.test.ts"],
+ },
+});
diff --git a/packages/tracing/vitest.unit.config.ts b/packages/tracing/vitest.unit.config.ts
new file mode 100644
index 000000000..95ba6cbd2
--- /dev/null
+++ b/packages/tracing/vitest.unit.config.ts
@@ -0,0 +1,9 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+ test: {
+ environment: "node",
+ include: ["src/**/*.test.ts"],
+ exclude: ["src/**/*.integration.test.ts"],
+ },
+});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 641d5443d..170ea9fcc 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1829,6 +1829,9 @@ importers:
'@eve/catalog':
specifier: workspace:*
version: link:../eve-catalog
+ '@eve/tracing':
+ specifier: workspace:*
+ version: link:../tracing
'@linqapp/chat-sdk-adapter':
specifier: ^0.5.1
@@ -2089,6 +2092,39 @@ importers:
specifier: 'catalog:'
version: 7.0.2
+ packages/tracing:
+ devDependencies:
+ '@ai-sdk/harness':
+ specifier: 1.0.148
+ '@ai-sdk/provider-utils':
+ specifier: 'catalog:'
+ version: 5.0.58([email protected])
+ '@opentelemetry/api':
+ specifier: ^1.9.0
+ version: 1.9.1
+ '@opentelemetry/context-async-hooks':
+ specifier: 'catalog:'
+ version: 2.6.1(@opentelemetry/[email protected])
+ '@opentelemetry/sdk-trace-base':
+ specifier: 'catalog:'
+ version: 2.6.1(@opentelemetry/[email protected])
+ '@types/node':
+ specifier: 'catalog:'
+ version: 24.13.3
+ ai:
+ specifier: 'catalog:'
+ version: 7.0.137([email protected])
+ typescript:
+ specifier: 'catalog:'
+ version: 7.0.2
+ vitest:
+ specifier: 'catalog:'
+ version: 5.0.2(@edge-runtime/[email protected])(@opentelemetry/[email protected])(@types/[email protected])([email protected](@types/[email protected]))
+ zod:
+ specifier: 'catalog:'
+ version: 4.5.4
+
packages:
'@agent-browser/[email protected]':
@@ -2218,6 +2254,16 @@ packages:
peerDependencies:
zod: ^3.25.76 || ^4.1.8
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-m+FTa81r5VWC7FPPtS6T1JfcxbZYYnu/ZXt0y87ffqGDHhBWPPKnxobauK3ASH/HmZyNJUZRBbU4SaGQ2Q3UsQ==}
+ engines: {node: '>=22'}
+ peerDependencies:
+ ws: ^8.21.0
+ zod: ^3.25.76 || ^4.1.8
+ peerDependenciesMeta:
+ ws:
+ optional: true
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-cMEif3Lbvh+FozPpBM5UPy6QQhH8v8L/u1kcR2UXp+86EfIDXIaIeOaH+wYCk2CfloFccUn1K8LJuPXqZMZsfw==}
engines: {node: '>=22'}
@@ -15538,6 +15584,15 @@ snapshots:
zod: 4.4.3
optional: true
+ dependencies:
+ '@ai-sdk/provider': 4.0.26
+ '@ai-sdk/provider-utils': 5.0.58([email protected])
+ ai: 7.0.137([email protected])
+ zod: 4.5.4
+ optionalDependencies:
+ ws: 8.21.3([email protected])
+
dependencies:
'@ai-sdk/provider': 4.0.22
diff --git a/scripts/guard-invariants.mjs b/scripts/guard-invariants.mjs
index 7fbd8f326..1757ae2ae 100644
--- a/scripts/guard-invariants.mjs
+++ b/scripts/guard-invariants.mjs
@@ -135,6 +135,10 @@
* events, so nothing changes without readers hearing it. The model
* step's streamed content (its calls and their inline results) is
* built where it streams.
+ * rule 52 — The tracing library in `packages/tracing/src/` imports only
+ * itself and external packages. eve reaches it only through the
+ * vendored root, delegation, otel, and ai-sdk entrypoints under
+ * `#compiled/@eve/tracing/`, never the workspace package itself.
*
* Baselines for rules with pre-existing violations live in
* `guard-invariants-baseline.json`. Counts and allowlists in that file
@@ -256,6 +260,7 @@ function isTsLike(relPath) {
* rule48: Violation[];
* rule50: Violation[];
* rule51: Violation[];
+ * rule52: Violation[];
* symlinks: string[];
* }} state
*/
@@ -291,6 +296,50 @@ async function scanRepo(state) {
checkRule48(posix, lines, state.rule48);
checkRule50(posix, lines, state.rule50);
checkRule51(posix, lines, state.rule51);
+ if (posix.startsWith("packages/tracing/src/")) {
+ for (const [index, line] of lines.entries()) {
+ const imports = [...line.matchAll(/(?:from\s+|import\s*\()(["'])([^"']+)\1/g)];
+ for (const match of imports) {
+ const specifier = match[2];
+ const escapes =
+ specifier.startsWith("#") ||
+ (specifier.startsWith(".") &&
+ !toPosix(relative(REPO_ROOT, resolve(dirname(absPath), specifier))).startsWith(
+ "packages/tracing/src/",
+ ));
+ if (escapes)
+ state.rule52.push({
+ rule: 52,
+ file: posix,
+ line: index + 1,
+ message:
+ "Tracing library imports must stay inside the library or use an external package. Host integration belongs in the host.",
+ });
+ }
+ }
+ }
+ if (posix.startsWith("packages/eve/")) {
+ lines.forEach((line, index) => {
+ const privateImport =
+ /['"]#compiled\/@eve\/tracing\/(?!(?:index|delegation|otel|ai-sdk)\.js['"])/.test(line) ||
+ /(?:from\s+|import\s*\()['"]@eve\/tracing(?:\/[^'"]*)?['"]/.test(line) ||
+ [...line.matchAll(/(?:from\s+|import\s*\()(["'])([^"']+)\1/g)].some(
+ (match) =>
+ match[2].startsWith(".") &&
+ toPosix(relative(REPO_ROOT, resolve(dirname(absPath), match[2]))).startsWith(
+ "packages/tracing/",
+ ),
+ );
+ if (privateImport)
+ state.rule52.push({
+ rule: 52,
+ file: posix,
+ line: index + 1,
+ message:
+ "Import the tracing library from #compiled/@eve/tracing/{index,delegation,otel,ai-sdk}.js, not the workspace package or its private modules.",
+ });
+ });
+ }
}
}
@@ -1697,6 +1746,7 @@ async function main() {
rule48: /** @type {Violation[]} */ ([]),
rule50: /** @type {Violation[]} */ ([]),
rule51: /** @type {Violation[]} */ ([]),
+ rule52: /** @type {Violation[]} */ ([]),
symlinks: /** @type {string[]} */ ([]),
};
@@ -1814,6 +1864,7 @@ async function main() {
violations.push(...(await checkFrameworkActionIdentity()));
violations.push(...state.rule50);
violations.push(...state.rule51);
+ violations.push(...state.rule52);
if (violations.length === 0) {
process.stdout.write("[eve:guard:invariants] ok — all mechanical lints passed.\n");