The Open Framework for Building Agents
feat(eve): assemble OpenTelemetry from declarations (#1857)
Chad Hietala084f8f156353
diff --git a/.changeset/otel-pipeline-primitive.md b/.changeset/otel-pipeline-primitive.mdnew file mode 100644index 000000000..b8da54762--- /dev/null+++ b/.changeset/otel-pipeline-primitive.md@@ -0,0 +1,5 @@+---+"eve": patch+---++eve now assembles its local OpenTelemetry runtime from declarative singleton settings and ordered destinations. The local trace spool is an ordinary span processor, and tracer-provider ownership, flushing, and shutdown are managed centrally without changing recorded spans.diff --git a/packages/eve/package.json b/packages/eve/package.jsonindex 3ab1cfbe8..5346416a2 100644--- a/packages/eve/package.json+++ b/packages/eve/package.json@@ -328,6 +328,7 @@"@eve/catalog": "workspace:*","@nuxt/kit": "^4.0.0","@opentelemetry/context-async-hooks": "catalog:",+ "@opentelemetry/core": "catalog:","@opentelemetry/otlp-transformer": "0.214.0","@opentelemetry/sdk-trace-base": "catalog:","@photon-ai/chat-adapter-imessage": "3.2.0",diff --git a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.tsindex 7954271d7..d7168a656 100644--- a/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts+++ b/packages/eve/scripts/vendor-compiled/declarations/@opentelemetry/api.d.ts@@ -30,7 +30,11 @@ export interface Tracer {): Span;}-export interface Context {}+export interface Context {+ setValue(key: symbol, value: unknown): Context;+}++export declare function createContextKey(description: string): symbol;export declare const ROOT_CONTEXT: Context;@@ -40,6 +44,11 @@ export declare enum SpanStatusCode {ERROR = 2,}+export declare enum TraceFlags {+ NONE = 0,+ SAMPLED = 1,+}+export declare const context: {active(): Context;with<T>(context: Context, fn: () => T): T;@@ -55,13 +64,19 @@ export interface TextMapGetter<Carrier = unknown> {keys(carrier: Carrier): string[];}+export interface TextMapSetter<Carrier = unknown> {+ set(carrier: Carrier, key: string, value: string): void;+}+export declare const propagation: {extract<Carrier>(context: Context, carrier: Carrier, getter: TextMapGetter<Carrier>): Context;+ inject<Carrier>(context: Context, carrier: Carrier, setter: TextMapSetter<Carrier>): void;};export declare const trace: {getActiveSpan(): Span | undefined;getTracer(name: string, version?: string): Tracer;+ getTracerProvider(): unknown;setSpan(context: Context, span: Span): Context;wrapSpanContext(spanContext: SpanContext): Span;};diff --git a/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts b/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.tsindex e6cfcfc8e..d98c03d4c 100644--- a/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts+++ b/packages/eve/scripts/vendor-compiled/declarations/@vercel/otel.d.ts@@ -5,18 +5,61 @@ export interface SpanProcessor {shutdown(): Promise<void>;}+export type SpanProcessorOrName = SpanProcessor | "auto";+export interface IdGenerator {generateSpanId(): string;generateTraceId(): string;}+/**+ * A `SpanExporter`. Structural for the same reason the propagator is: the+ * instance comes from whichever `@opentelemetry/*` build the app installed, and+ * eve only ever hands it spans and waits for the callback.+ *+ * `code` is `ExportResultCode`: `0` succeeded, `1` failed.+ */+export interface SpanExporter {+ export(+ spans: readonly unknown[],+ resultCallback: (result: { code: number; error?: Error }) => void,+ ): void;+ forceFlush?(): Promise<void>;+ shutdown(): Promise<void>;+}++/**+ * A `TextMapPropagator`, or one of the names `@vercel/otel` resolves for you.+ * Structural rather than imported: the instance comes from whichever+ * `@opentelemetry/*` build the app installed, not from eve's.+ */+export type PropagatorOrName =+ | { inject(...args: never[]): void; extract(...args: never[]): unknown; fields(): string[] }+ | "auto"+ | "none"+ | "tracecontext"+ | "baggage";++/** A `Sampler`, or one of the names `@vercel/otel` resolves for you. */+export type SamplerOrName =+ | { shouldSample(...args: never[]): unknown; toString(): string }+ | "auto"+ | "always_off"+ | "always_on"+ | "parentbased_always_off"+ | "parentbased_always_on"+ | "parentbased_traceidratio"+ | "traceidratio";+export interface Configuration {+ readonly attributes?: Readonly<Record<string, unknown>>;readonly autoDetectResources?: boolean;readonly idGenerator?: IdGenerator;readonly instrumentations?: readonly unknown[];- readonly propagators?: readonly ["none"];+ readonly propagators?: readonly PropagatorOrName[];readonly serviceName?: string;- readonly spanProcessors?: readonly SpanProcessor[];+ readonly spanProcessors?: readonly SpanProcessorOrName[];+ readonly traceSampler?: SamplerOrName;}export declare function registerOTel(configuration?: Configuration | string): void;diff --git a/packages/eve/src/harness/instrumentation-config.test.ts b/packages/eve/src/harness/instrumentation-config.test.tsindex af254587d..ef907d37a 100644--- a/packages/eve/src/harness/instrumentation-config.test.ts+++ b/packages/eve/src/harness/instrumentation-config.test.ts@@ -1,4 +1,10 @@-import { describe, expect, it, vi } from "vitest";+import { beforeEach, describe, expect, it, vi } from "vitest";++const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime");++beforeEach(() => {+ delete (globalThis as Record<symbol, unknown>)[RUNTIME_GLOBAL_KEY];+});/*** Regression coverage for the instrumentation-config chunk-isolation failure.@@ -61,6 +67,28 @@ describe("instrumentation-config chunk-isolation regression", () => {expect(secondRef).toBe(firstRef);});+ it("installs harness telemetry settings on the instrumentation runtime", async () => {+ const { registerInstrumentationConfig } = await import("#harness/instrumentation-config.js");+ const { getInstrumentationRuntime } = await import("#harness/instrumentation-runtime.js");++ await registerInstrumentationConfig(+ {+ functionId: "weather",+ recordInputs: true,+ recordOutputs: false,+ traceChannelRequests: true,+ },+ { agentName: "test-agent" },+ );++ expect(getInstrumentationRuntime()?.otelSettings).toEqual({+ functionId: "weather",+ recordInputs: true,+ recordOutputs: false,+ traceChannelRequests: true,+ });+ });+it("invokes the setup callback with the supplied context", async () => {vi.resetModules();const { registerInstrumentationConfig } = await import("#harness/instrumentation-config.js");diff --git a/packages/eve/src/harness/instrumentation-config.ts b/packages/eve/src/harness/instrumentation-config.tsindex 509340fc0..fbe66be55 100644--- a/packages/eve/src/harness/instrumentation-config.ts+++ b/packages/eve/src/harness/instrumentation-config.ts@@ -2,6 +2,8 @@ import type {InstrumentationDefinition,InstrumentationSetupContext,} from "#public/instrumentation/index.js";+import { createInstrumentationHooks } from "#harness/instrumentation-lifecycle.js";+import { registerInstrumentationRuntime } from "#harness/instrumentation-runtime.js";/*** Process-global store for the authored instrumentation config.@@ -36,14 +38,24 @@ const globalContainer = globalThis as typeof globalThis & InstrumentationConfigG** @internal — not part of the public API.*/-export function registerInstrumentationConfig(+export async function registerInstrumentationConfig(config: InstrumentationDefinition,context: InstrumentationSetupContext,-): void {- if (config.setup !== undefined) {- config.setup(context);- }+): Promise<void> {globalContainer[INSTRUMENTATION_CONFIG_GLOBAL_KEY] = config;+ registerInstrumentationRuntime({+ forceFlush: async () => undefined,+ hooks: createInstrumentationHooks([]),+ otelSettings: {+ functionId: config.functionId,+ recordInputs: config.recordInputs,+ recordOutputs: config.recordOutputs,+ traceChannelRequests: config.traceChannelRequests === true,+ },+ runInContext: (_operation, execute) => execute(),+ shutdown: async () => undefined,+ });+ await config.setup?.(context);}/**diff --git a/packages/eve/src/harness/instrumentation-lifecycle.ts b/packages/eve/src/harness/instrumentation-lifecycle.tsindex d3e87826c..8ec601e80 100644--- a/packages/eve/src/harness/instrumentation-lifecycle.ts+++ b/packages/eve/src/harness/instrumentation-lifecycle.ts@@ -292,6 +292,7 @@ export type InstrumentationEventHandler<TEvent> = (event: TEvent) => void | Prom/** Internal provider shape mirrored by the future public hook contract. */export interface InstrumentationProviderDefinition {+ readonly name?: string;readonly events?: {readonly "step.attempt.started"?: InstrumentationEventHandler<InstrumentationStepAttemptStartedEvent>;readonly "step.attempt.completed"?: InstrumentationEventHandler<InstrumentationStepAttemptTerminalEvent>;@@ -314,6 +315,8 @@ export interface InstrumentationProviderDefinition {readonly "turn.failed"?: InstrumentationEventHandler<InstrumentationTurnTerminalEvent>;readonly "turn.started"?: InstrumentationEventHandler<InstrumentationTurnStartedEvent>;};+ readonly flush?: () => void | PromiseLike<void>;+ readonly shutdown?: () => void | PromiseLike<void>;}/** Events that carry an operation `id`, pairing a start with its terminal. */diff --git a/packages/eve/src/harness/instrumentation-runtime.ts b/packages/eve/src/harness/instrumentation-runtime.tsindex 33c99efed..584471749 100644--- a/packages/eve/src/harness/instrumentation-runtime.ts+++ b/packages/eve/src/harness/instrumentation-runtime.ts@@ -2,6 +2,7 @@ import type {InstrumentationContextRunner,InstrumentationHooks,} from "#harness/instrumentation-lifecycle.js";+import type { OtelHarnessSettings } from "#tracing/otel-declaration.js";const INSTRUMENTATION_RUNTIME_KEY = Symbol.for("eve.instrumentation-runtime");@@ -9,7 +10,9 @@ const INSTRUMENTATION_RUNTIME_KEY = Symbol.for("eve.instrumentation-runtime");export interface InstrumentationRuntime {readonly forceFlush: () => Promise<void>;readonly hooks: InstrumentationHooks;+ otelSettings: OtelHarnessSettings | undefined;readonly runInContext: InstrumentationContextRunner;+ readonly shutdown: () => Promise<void>;}/** Instrumentation capabilities consumed inside one harness execution. */@@ -26,7 +29,11 @@ export function registerInstrumentationRuntime(runtime: InstrumentationRuntime,): InstrumentationRuntime {const existing = globalRuntime[INSTRUMENTATION_RUNTIME_KEY];- if (existing !== undefined) return existing;+ if (existing !== undefined) {+ // A legacy config may reload without taking ownership from the installed runtime.+ existing.otelSettings = runtime.otelSettings;+ return existing;+ }globalRuntime[INSTRUMENTATION_RUNTIME_KEY] = runtime;return runtime;}diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex 660c43e44..90f3ebb6e 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -96,6 +96,25 @@ vi.mock("./instrumentation-config.js", () => ({getInstrumentationConfig: (...args: unknown[]) => mockGetInstrumentationConfig(...args),}));+const mockGetInstrumentationRuntime = vi.fn().mockReturnValue(undefined);+vi.mock("./instrumentation-runtime.js", () => ({+ getInstrumentationRuntime: (...args: unknown[]) => mockGetInstrumentationRuntime(...args),+}));++function declareTelemetry(config: Readonly<Record<string, unknown>> | undefined): void {+ mockGetInstrumentationConfig.mockReturnValue(config);+ mockGetInstrumentationRuntime.mockReturnValue(+ config === undefined+ ? undefined+ : {+ otelSettings: {+ ...config,+ traceChannelRequests: config["traceChannelRequests"] === true,+ },+ },+ );+}+vi.mock("./compaction.js", () => ({compactMessages: vi.fn(),estimateTokens: vi.fn().mockReturnValue(5000),@@ -115,7 +134,7 @@ vi.mock("./compaction.js", () => ({afterEach(() => {vi.clearAllMocks();vi.unstubAllEnvs();- mockGetInstrumentationConfig.mockReturnValue(undefined);+ declareTelemetry(undefined);});function createTestSession(overrides?: Partial<HarnessSession>): HarnessSession {@@ -4448,7 +4467,7 @@ describe("createToolLoopHarness", () => {"test.attempt": typeof input.modelInput.instructions === "string" ? "original" : "retry",},}));- mockGetInstrumentationConfig.mockReturnValue({+ declareTelemetry({events: {"step.started": resolveRuntimeContext,},@@ -10029,11 +10048,11 @@ describe("createToolLoopHarness", () => {toolResults: [{ toolCallId: "call-1", toolName: "add", output: "42" }],});- mockGetInstrumentationConfig.mockReturnValue({});+ declareTelemetry({});const config = createTestConfig("conversation");const runStep = createToolLoopHarness(config);const result = await runStep(createTestSession(), { message: "add stuff" });- mockGetInstrumentationConfig.mockReturnValue(undefined);+ declareTelemetry(undefined);expect(result.next).toBe(runStep);expect(result.session.state?.["eve.harness.turnTrace"]).toEqual({@@ -10096,7 +10115,7 @@ describe("createToolLoopHarness", () => {toolResults: [{ toolCallId: "call-1", toolName: "add", output: "42" }],});- mockGetInstrumentationConfig.mockReturnValue({});+ declareTelemetry({});const step1Config = createTestConfig("conversation");const step1 = createToolLoopHarness(step1Config);const result1 = await step1(createTestSession(), { message: "add stuff" });@@ -10125,7 +10144,7 @@ describe("createToolLoopHarness", () => {const step2 = createToolLoopHarness(step2Config);// No input — continuation stepconst result2 = await step2(result1.session);- mockGetInstrumentationConfig.mockReturnValue(undefined);+ declareTelemetry(undefined);expect(result2.next).toBeNull();@@ -10155,11 +10174,11 @@ describe("createToolLoopHarness", () => {toolResults: [],});- mockGetInstrumentationConfig.mockReturnValue({});+ declareTelemetry({});const config = createTestConfig("conversation");const runStep = createToolLoopHarness(config);await runStep(createTestSession(), { message: "hi" });- mockGetInstrumentationConfig.mockReturnValue(undefined);+ declareTelemetry(undefined);const agentCall = vi.mocked(ToolLoopAgent).mock.calls[0]?.[0] as {runtimeContext?: Record<string, unknown>;@@ -10234,7 +10253,7 @@ describe("createToolLoopHarness", () => {toolCalls: [],toolResults: [],});- mockGetInstrumentationConfig.mockReturnValue({ recordInputs: true, recordOutputs: false });+ declareTelemetry({ recordInputs: true, recordOutputs: false });const hooks = createInstrumentationHooks([]);const runStep = createToolLoopHarness(createTestConfig("conversation", undefined, {@@ -10293,7 +10312,7 @@ describe("createToolLoopHarness", () => {},};});- mockGetInstrumentationConfig.mockReturnValue({+ declareTelemetry({events: {"step.started": resolveRuntimeContext,},@@ -10348,7 +10367,7 @@ describe("createToolLoopHarness", () => {toolResults: [],});- mockGetInstrumentationConfig.mockReturnValue({+ declareTelemetry({events: {"step.started": () => {throw new Error("runtime context resolver failed");@@ -10386,7 +10405,7 @@ describe("createToolLoopHarness", () => {"test.step": `${input.turn.id}:${input.step.index}`,},}));- mockGetInstrumentationConfig.mockReturnValue({+ declareTelemetry({events: {"step.started": resolveRuntimeContext,},diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.tsindex f0d43f46f..caee8d531 100644--- a/packages/eve/src/harness/tool-loop.ts+++ b/packages/eve/src/harness/tool-loop.ts@@ -60,7 +60,6 @@ import {createSessionWaitingEvent,type UnstampedMessageStreamEvent,} from "#protocol/message.js";-import type { InstrumentationDefinition } from "#public/instrumentation/index.js";import { ASK_QUESTION_TOOL_NAME } from "#runtime/framework-tools/ask-question.js";import { resolveAgentsAnnouncement } from "#harness/handles/prompt.js";import { getAgentHandleStore } from "#harness/handles/store.js";@@ -153,6 +152,8 @@ import {dropStaleSessionLimitContinuationResponses,} from "#harness/stale-input-responses.js";import { getInstrumentationConfig } from "#harness/instrumentation-config.js";+import { getInstrumentationRuntime } from "#harness/instrumentation-runtime.js";+import type { OtelHarnessSettings } from "#tracing/otel-declaration.js";import { normalizeUserContent, resolveAssistantStepText } from "#harness/messages.js";import { normalizeProviderToolHistory } from "#harness/provider-tool-history.js";import {@@ -241,7 +242,7 @@ import {/*** Builds the `telemetry` value for the AI SDK from authored settings.*- * Custom context (authored `InstrumentationDefinition.events` plus+ * Custom context (authored instrumentation events plus* eve-specific identifiers such as `eve.session.id`) is flowed through* {@link buildTelemetryRuntimeContext} because AI SDK v7 surfaces* per-call attributes via `runtimeContext`, not a dedicated metadata field on@@ -287,12 +288,12 @@ const MODEL_CALL_MAX_ATTEMPTS = 3;const MODEL_CALL_RETRY_BASE_DELAY_MS = 500;function enrichTelemetry(- authored: InstrumentationDefinition | undefined,+ settings: OtelHarnessSettings | undefined,agentName: string | undefined,runtimeContext?: Readonly<Record<string, unknown>>,bridgeIntegration?: Telemetry,): TelemetryOptions | undefined {- if (authored === undefined && bridgeIntegration === undefined) {+ if (settings === undefined && bridgeIntegration === undefined) {return undefined;}@@ -304,17 +305,17 @@ function enrichTelemetry(}return {- functionId: authored?.functionId ?? agentName,+ functionId: settings?.functionId ?? agentName,includeRuntimeContext,integrations:bridgeIntegration === undefined? undefined- : authored === undefined+ : getInstrumentationConfig() === undefined? [bridgeIntegration]: [bridgeIntegration, createOtelIntegration()],isEnabled: true,- recordInputs: authored?.recordInputs ?? true,- recordOutputs: authored?.recordOutputs ?? true,+ recordInputs: settings?.recordInputs ?? true,+ recordOutputs: settings?.recordOutputs ?? true,};}@@ -543,11 +544,12 @@ function buildHarnessToolsWithDynamicSubagents(export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {const baseEmit = config.handleEvent;- const telemetryConfig = getInstrumentationConfig();- if (telemetryConfig !== undefined) {+ const otelSettings = getInstrumentationRuntime()?.otelSettings;+ const authoredConfig = getInstrumentationConfig();+ if (otelSettings !== undefined) {ensureOtelIntegration();}- const tracer = telemetryConfig !== undefined ? trace.getTracer("eve") : undefined;+ const tracer = otelSettings !== undefined ? trace.getTracer("eve") : undefined;const agentName = config.runtimeIdentity?.agentName;async function runStep(@@ -560,7 +562,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {// restore the parent from session state via resolveStepOtelContext.let turnSpan: Span | undefined;if (tracer && hasStepInput(input)) {- const functionId = telemetryConfig?.functionId ?? agentName;+ const functionId = otelSettings?.functionId ?? agentName;const attributes: Record<string, string> = {"eve.version": eveVersion,"eve.environment": environment,@@ -692,7 +694,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {resolveModel: config.resolveModel,runtimeIdentity: config.runtimeIdentity,session,- telemetry: enrichTelemetry(telemetryConfig, agentName) ?? undefined,+ telemetry: enrichTelemetry(otelSettings, agentName) ?? undefined,});session = {@@ -1095,7 +1097,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {resolveModel: config.resolveModel,runtimeIdentity: config.runtimeIdentity,session,- telemetry: enrichTelemetry(telemetryConfig, agentName) ?? undefined,+ telemetry: enrichTelemetry(otelSettings, agentName) ?? undefined,}));const approvedTools = getApprovedTools(session);@@ -1167,7 +1169,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {instructions,telemetryRuntimeContext: buildTelemetryRuntimeContext({eveVersion,- authored: telemetryConfig,+ authored: authoredConfig,emissionState,environment,modelInput: {@@ -1289,7 +1291,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {: {attemptId: `${session.sessionId}:${instrumentationTurnId}:${emissionState.stepIndex}:${opts.attemptIndex}`,attemptIndex: opts.attemptIndex,- functionId: telemetryConfig?.functionId ?? agentName,+ functionId: otelSettings?.functionId ?? agentName,rootSessionId: parent?.rootSessionId ?? session.sessionId,sessionId: session.sessionId,stepIndex: emissionState.stepIndex,@@ -1335,7 +1337,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {runtimeContext: telemetryRuntimeContext,stopWhen: isStepCount(1),telemetry: enrichTelemetry(- telemetryConfig,+ otelSettings,agentName,telemetryRuntimeContext,bridgeIntegration,diff --git a/packages/eve/src/internal/application/compiled-artifacts.ts b/packages/eve/src/internal/application/compiled-artifacts.tsindex 4ce9c7e30..561d60f45 100644--- a/packages/eve/src/internal/application/compiled-artifacts.ts+++ b/packages/eve/src/internal/application/compiled-artifacts.ts@@ -391,7 +391,7 @@ function createInstrumentationPluginSource(input: {`import { registerInstrumentationConfig } from ${stringifyEsmImportSpecifier(input.registerConfigPath)};`,"","if (instrumentationModule.default != null) {",- ` registerInstrumentationConfig(instrumentationModule.default, { agentName: ${JSON.stringify(input.agentName)} });`,+ ` await registerInstrumentationConfig(instrumentationModule.default, { agentName: ${JSON.stringify(input.agentName)} });`,"}","","// Default export satisfies the Nitro plugin contract so this file",diff --git a/packages/eve/src/internal/nitro/host/local-tracing-runtime-plugin.ts b/packages/eve/src/internal/nitro/host/local-tracing-runtime-plugin.tsindex fe9833763..a17557213 100644--- a/packages/eve/src/internal/nitro/host/local-tracing-runtime-plugin.ts+++ b/packages/eve/src/internal/nitro/host/local-tracing-runtime-plugin.ts@@ -9,10 +9,14 @@ if (appRoot === undefined) {throw new Error(`${DEVELOPMENT_WORKER_APP_ROOT_ENV} is required for local tracing.`);}-installLocalInstrumentationRuntime({+const runtime = installLocalInstrumentationRuntime({appRoot,frameworkVersion: resolveInstalledPackageInfo().version,serviceName: basename(appRoot),});-export default function installLocalTracingRuntimePlugin(): void {}+export default function installLocalTracingRuntimePlugin(nitroApp?: {+ readonly hooks?: { hook(name: "close", handler: () => Promise<void>): unknown };+}): void {+ nitroApp?.hooks?.hook("close", () => runtime.shutdown());+}diff --git a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.tsindex dbe8d3472..9617df9c6 100644--- a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts+++ b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts@@ -8,7 +8,7 @@ import {type TextMapGetter,trace,} from "#compiled/@opentelemetry/api/index.js";-import { getInstrumentationConfig } from "#harness/instrumentation-config.js";+import { getInstrumentationRuntime } from "#harness/instrumentation-runtime.js";import { recordErrorOnSpan } from "#internal/logging.js";/**@@ -70,7 +70,7 @@ export async function traceChannelRequest<T extends Response>(input: TraceChannelRequestInput,handler: (span: Span | undefined) => Promise<T>,): Promise<T> {- if (getInstrumentationConfig()?.traceChannelRequests !== true) {+ if (getInstrumentationRuntime()?.otelSettings?.traceChannelRequests !== true) {return await handler(undefined);}diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex c36dfa4e6..93bd6df43 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -459,7 +459,8 @@ describe("createAgentOtelInstrumentation", () => {spanProcessors: [new SimpleSpanProcessor(exporter)],});const agentOtel = createAgentOtelInstrumentation({- captureContent: false,+ recordInputs: false,+ recordOutputs: false,frameworkVersion: "test",idGenerator,stateStore: new InMemoryAgentTraceStateStore(),diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex 84e7e7620..c4d67047b 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -57,12 +57,10 @@ interface ToolSpanState extends SpanState {}export interface AgentOtelInstrumentationInput {- /**- * Capture model prompts/responses and tool call inputs/outputs as span- * attributes. Content stays on the local machine — this provider only- * wires the dev-time local spool — but can be turned off per project.- */- readonly captureContent?: boolean;+ /** Whether any destination requested model prompts and tool inputs. */+ readonly recordInputs?: boolean;+ /** Whether any destination requested model responses and tool outputs. */+ readonly recordOutputs?: boolean;readonly frameworkVersion: string;readonly idGenerator: AgentSpanIdGenerator;readonly stateStore: AgentTraceStateStore;@@ -79,7 +77,8 @@ export interface AgentOtelInstrumentation {export function createAgentOtelInstrumentation(input: AgentOtelInstrumentationInput,): AgentOtelInstrumentation {- const captureContent = input.captureContent ?? true;+ const recordInputs = input.recordInputs ?? true;+ const recordOutputs = input.recordOutputs ?? true;const executionContexts = new WeakMap<InstrumentationAttemptScope,{ readonly models: Map<string, Context>; readonly tools: Map<string, Context> }@@ -270,7 +269,7 @@ export function createAgentOtelInstrumentation(},attempt.operation.context,);- if (captureContent) {+ if (recordInputs) {const messages = messagesContentAttribute(event.input.messages);if (messages !== undefined) span.setAttribute("ai.prompt.messages", messages);const system = systemPromptAttribute(event.input.instructions);@@ -291,7 +290,7 @@ export function createAgentOtelInstrumentation(setUsage(state.span, event.usage);const attempt = steps.get(event.scope);if (attempt !== undefined) setUsage(attempt.step.span, event.usage);- if (captureContent) {+ if (recordOutputs) {state.span.setAttribute("ai.response.finish_reason", event.finishReason);const reasoning = textContentAttribute(event.content@@ -377,7 +376,7 @@ export function createAgentOtelInstrumentation(},actionContext,);- if (captureContent) {+ if (recordInputs) {const args = contentAttribute(event.input, false);if (args !== undefined) toolSpan.setAttribute("gen_ai.tool.call.arguments", args);}@@ -400,7 +399,7 @@ export function createAgentOtelInstrumentation(} else if (event.output.type === "error") {recordError(state.toolSpan, event.output.error);recordError(state.span, event.output.error);- } else if (captureContent) {+ } else if (recordOutputs) {const result = contentAttribute(event.output.output, false);if (result !== undefined) state.toolSpan.setAttribute("gen_ai.tool.call.result", result);}@@ -499,6 +498,7 @@ export function createAgentOtelInstrumentation(return {hook: {+ name: "eve.otel",events: {"step.attempt.completed": onStepTerminal,"step.attempt.failed": onStepTerminal,diff --git a/packages/eve/src/tracing/agent-trace-span-processor.ts b/packages/eve/src/tracing/agent-trace-span-processor.tsindex 99daa8065..4b63a09e7 100644--- a/packages/eve/src/tracing/agent-trace-span-processor.ts+++ b/packages/eve/src/tracing/agent-trace-span-processor.ts@@ -11,8 +11,6 @@ export class AgentTraceSpanProcessor implements SpanProcessor {readonly #children: readonly SpanProcessor[];readonly #ownedTraceIds = new Set<string>();readonly #sessionTraceIds = new Map<string, Set<string>>();- #attached = false;-constructor(children: readonly SpanProcessor[]) {this.#children = children;}@@ -21,12 +19,7 @@ export class AgentTraceSpanProcessor implements SpanProcessor {await Promise.all(this.#children.map((child) => child.forceFlush()));}- isAttached(): boolean {- return this.#attached;- }-onStart(span: unknown, parentContext: unknown): void {- this.#attached = true;if (!isSpanLike(span)) return;const sessionId = span.attributes["agent.session.id"];if (typeof sessionId === "string") {diff --git a/packages/eve/src/tracing/batch-span-processor.test.ts b/packages/eve/src/tracing/batch-span-processor.test.tsnew file mode 100644index 000000000..36d8a3e4b--- /dev/null+++ b/packages/eve/src/tracing/batch-span-processor.test.ts@@ -0,0 +1,226 @@+import type { SpanExporter } from "#compiled/@vercel/otel/index.js";+import { context, TraceFlags } from "@opentelemetry/api";+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";+import { isTracingSuppressed } from "@opentelemetry/core";+import { afterEach, describe, expect, it, vi } from "vitest";++import { batchSpanProcessor } from "#tracing/batch-span-processor.js";++interface RecordingExporter extends SpanExporter {+ readonly batches: readonly (readonly unknown[])[];+}++function recordingExporter(behavior: { readonly fail?: boolean } = {}): RecordingExporter {+ const batches: (readonly unknown[])[] = [];+ return {+ batches,+ export: (spans, resultCallback) => {+ batches.push([...spans]);+ resultCallback(+ behavior.fail === true ? { code: 1, error: new Error("refused") } : { code: 0 },+ );+ },+ shutdown: async () => undefined,+ };+}++function span(index: number, traceFlags: TraceFlags = TraceFlags.SAMPLED): unknown {+ return { index, spanContext: () => ({ traceFlags }) };+}++function batchIndexes(batches: readonly (readonly unknown[])[]): number[][] {+ return batches.map((batch) => batch.map((item) => (item as { readonly index: number }).index));+}++describe("batchSpanProcessor", () => {+ afterEach(() => {+ vi.useRealTimers();+ context.disable();+ });++ it("exports once the batch size is reached, without waiting for the timer", async () => {+ const exporter = recordingExporter();+ const processor = batchSpanProcessor(exporter, {+ maxExportBatchSize: 2,+ scheduledDelayMillis: 60_000,+ });++ processor.onEnd(span(1));+ expect(exporter.batches).toHaveLength(0);++ processor.onEnd(span(2));+ await processor.forceFlush();++ expect(batchIndexes(exporter.batches)).toStrictEqual([[1, 2]]);+ });++ it("splits a flush larger than one batch", async () => {+ const exporter = recordingExporter();+ const processor = batchSpanProcessor(exporter, { maxExportBatchSize: 2 });++ for (const index of [1, 2, 3]) processor.onEnd(span(index));+ await processor.forceFlush();++ expect(batchIndexes(exporter.batches)).toStrictEqual([[1, 2], [3]]);+ });++ // An exporter that has stopped draining is already losing telemetry; taking+ // the agent's memory with it would turn that into an outage.+ it("drops spans past the queue limit rather than growing without bound", async () => {+ const exporter = recordingExporter();+ const processor = batchSpanProcessor(exporter, {+ maxExportBatchSize: 100,+ maxQueueSize: 2,+ });++ for (const index of [1, 2, 3, 4]) processor.onEnd(span(index));+ await processor.forceFlush();++ expect(batchIndexes(exporter.batches)).toStrictEqual([[1, 2]]);+ });++ it("logs a refused export rather than failing the turn that produced it", async () => {+ const processor = batchSpanProcessor(recordingExporter({ fail: true }), {+ maxExportBatchSize: 1,+ });++ processor.onEnd(span(1));+ await expect(processor.forceFlush()).resolves.toBeUndefined();+ });++ it("exports only sampled spans", async () => {+ const exporter = recordingExporter();+ const processor = batchSpanProcessor(exporter);++ processor.onEnd(span(1, TraceFlags.NONE));+ processor.onEnd(span(2));+ await processor.forceFlush();++ expect(batchIndexes(exporter.batches)).toStrictEqual([[2]]);+ });++ it("suppresses tracing while an exporter runs", async () => {+ const manager = new AsyncLocalStorageContextManager().enable();+ context.setGlobalContextManager(manager);+ const suppressed: boolean[] = [];+ const exporter: SpanExporter = {+ export: (_spans, resultCallback) => {+ suppressed.push(isTracingSuppressed(context.active()));+ resultCallback({ code: 0 });+ },+ shutdown: async () => undefined,+ };+ const processor = batchSpanProcessor(exporter);++ processor.onEnd(span(1));+ await processor.forceFlush();++ expect(suppressed).toEqual([true]);+ expect(isTracingSuppressed(context.active())).toBe(false);+ manager.disable();+ });++ it("does not overlap a later batch after the first times out", async () => {+ vi.useFakeTimers();+ const batches: (readonly unknown[])[] = [];+ const callbacks: Array<(result: { code: number }) => void> = [];+ const exporter: SpanExporter = {+ export: (spans, resultCallback) => {+ batches.push([...spans]);+ callbacks.push(resultCallback);+ },+ shutdown: async () => undefined,+ };+ const processor = batchSpanProcessor(exporter, {+ exportTimeoutMillis: 10,+ maxExportBatchSize: 1,+ });++ processor.onEnd(span(1));+ await vi.advanceTimersByTimeAsync(0);+ processor.onEnd(span(2));+ const flushed = processor.forceFlush();+ let flushSettled = false;+ void flushed.then(() => {+ flushSettled = true;+ });+ await vi.advanceTimersByTimeAsync(10);++ expect(batchIndexes(batches)).toStrictEqual([[1]]);+ expect(flushSettled).toBe(true);+ callbacks[0]?.({ code: 0 });+ await vi.advanceTimersByTimeAsync(0);+ expect(batchIndexes(batches)).toStrictEqual([[1], [2]]);+ callbacks[1]?.({ code: 0 });+ await flushed;+ });++ it("continues exporting after exporter forceFlush times out", async () => {+ vi.useFakeTimers();+ const exporter = recordingExporter() as RecordingExporter & {+ forceFlush: () => Promise<void>;+ };+ exporter.forceFlush = () => new Promise(() => {});+ const processor = batchSpanProcessor(exporter, {+ exportTimeoutMillis: 10,+ maxExportBatchSize: 1,+ });++ processor.onEnd(span(1));+ const flushed = processor.forceFlush();+ await vi.advanceTimersByTimeAsync(10);+ await flushed;++ processor.onEnd(span(2));+ await vi.advanceTimersByTimeAsync(0);++ expect(batchIndexes(exporter.batches)).toStrictEqual([[1], [2]]);+ });++ it("drains queued batches and closes the exporter after shutdown times out", async () => {+ vi.useFakeTimers();+ const batches: (readonly unknown[])[] = [];+ const callbacks: Array<(result: { code: number }) => void> = [];+ const shutdown = vi.fn(async () => undefined);+ const exporter: SpanExporter = {+ export: (spans, resultCallback) => {+ batches.push([...spans]);+ callbacks.push(resultCallback);+ },+ shutdown,+ };+ const processor = batchSpanProcessor(exporter, {+ exportTimeoutMillis: 10,+ maxExportBatchSize: 1,+ });++ processor.onEnd(span(1));+ await vi.advanceTimersByTimeAsync(0);+ processor.onEnd(span(2));+ const stopped = processor.shutdown();+ await vi.advanceTimersByTimeAsync(10);+ await stopped;+ expect(shutdown).not.toHaveBeenCalled();++ callbacks[0]?.({ code: 0 });+ await vi.advanceTimersByTimeAsync(0);+ expect(batchIndexes(batches)).toStrictEqual([[1], [2]]);+ callbacks[1]?.({ code: 0 });+ await vi.advanceTimersByTimeAsync(0);+ expect(shutdown).toHaveBeenCalledOnce();+ });++ it("drains and then closes the exporter on shutdown, and takes nothing after", async () => {+ const exporter = recordingExporter();+ const shutdown = vi.spyOn(exporter, "shutdown");+ const processor = batchSpanProcessor(exporter, { scheduledDelayMillis: 60_000 });++ processor.onEnd(span(1));+ await processor.shutdown();+ processor.onEnd(span(2));+ await processor.forceFlush();++ expect(batchIndexes(exporter.batches)).toStrictEqual([[1]]);+ expect(shutdown).toHaveBeenCalledOnce();+ });+});diff --git a/packages/eve/src/tracing/batch-span-processor.ts b/packages/eve/src/tracing/batch-span-processor.tsnew file mode 100644index 000000000..8e45d5402--- /dev/null+++ b/packages/eve/src/tracing/batch-span-processor.ts@@ -0,0 +1,186 @@+import { TraceFlags, context, createContextKey } from "#compiled/@opentelemetry/api/index.js";+import type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js";++import { createLogger, formatError } from "#internal/logging.js";++const log = createLogger("harness.batch-span-processor");++// This is the Symbol.for-backed key used by @opentelemetry/core's+// suppressTracing(), without pulling that package into eve's runtime.+const SUPPRESS_TRACING_KEY = createContextKey("OpenTelemetry SDK Context Key SUPPRESS_TRACING");++/** The OpenTelemetry specification's defaults for a batching processor. */+const DEFAULTS = {+ exportTimeoutMillis: 30_000,+ maxExportBatchSize: 512,+ maxQueueSize: 2048,+ scheduledDelayMillis: 5_000,+} as const;++/** @internal — exposed for tests; authors get the defaults. */+export interface BatchSpanProcessorOptions {+ readonly exportTimeoutMillis?: number;+ readonly maxExportBatchSize?: number;+ readonly maxQueueSize?: number;+ readonly scheduledDelayMillis?: number;+}++/**+ * Buffers ended spans and hands them to one exporter in batches.+ *+ * eve owns this rather than taking `@opentelemetry/sdk-trace-base` as a runtime+ * dependency: an exporter needs exactly one thing done for it, and the+ * dependency would arrive with the whole SDK behind it.+ *+ * A full queue drops the newest span rather than growing without bound. An+ * exporter that has stopped draining is already losing telemetry; taking the+ * agent's memory with it would turn that into an outage.+ */+export function batchSpanProcessor(+ exporter: SpanExporter,+ options: BatchSpanProcessorOptions = {},+): SpanProcessor {+ const exportTimeoutMillis = options.exportTimeoutMillis ?? DEFAULTS.exportTimeoutMillis;+ const maxExportBatchSize = options.maxExportBatchSize ?? DEFAULTS.maxExportBatchSize;+ const maxQueueSize = options.maxQueueSize ?? DEFAULTS.maxQueueSize;+ const scheduledDelayMillis = options.scheduledDelayMillis ?? DEFAULTS.scheduledDelayMillis;++ let buffer: unknown[] = [];+ let timer: ReturnType<typeof setTimeout> | undefined;+ // Exports are chained rather than raced so a flush cannot interleave two+ // batches into the same exporter.+ let pending: Promise<void> = Promise.resolve();+ let stopped = false;++ function cancelTimer(): void {+ if (timer === undefined) return;+ clearTimeout(timer);+ timer = undefined;+ }++ function scheduleDrain(): void {+ if (timer !== undefined) return;+ timer = setTimeout(() => {+ timer = undefined;+ void drain();+ }, scheduledDelayMillis);+ // Buffered spans must not be the reason a process stays alive; `shutdown`+ // is what settles them.+ timer.unref?.();+ }++ function drain(): Promise<void> {+ return waitForExporter(pendingDrain(), "span export").then(() => undefined);+ }++ async function exportBatch(batch: readonly unknown[]): Promise<void> {+ try {+ await new Promise<void>((resolve, reject) => {+ try {+ context.with(context.active().setValue(SUPPRESS_TRACING_KEY, true), () => {+ exporter.export(batch, (result) => {+ if (result.code === 0) {+ resolve();+ return;+ }+ reject(result.error ?? new Error("Span export failed."));+ });+ });+ } catch (error: unknown) {+ reject(error instanceof Error ? error : new Error(String(error)));+ }+ });+ } catch (error: unknown) {+ // A destination that cannot take spans is not a reason to fail the turn+ // that produced them.+ log.warn("span export failed", { error: formatError(error) });+ }+ }++ return {+ async forceFlush() {+ if (stopped) return;+ const drained = await waitForExporter(pendingDrain(), "span export");+ if (!drained || exporter.forceFlush === undefined) return;+ await waitForExporter(+ settleExporter("exporter flush", exporter.forceFlush),+ "exporter flush",+ );+ },+ onEnd(span) {+ if (stopped) return;+ if (!isSampled(span)) return;+ if (buffer.length >= maxQueueSize) return;++ buffer.push(span);+ if (buffer.length >= maxExportBatchSize) {+ void drain();+ return;+ }+ scheduleDrain();+ },+ onStart() {},+ async shutdown() {+ stopped = true;+ const draining = pendingDrain();+ const drained = await waitForExporter(draining, "span export");+ if (!drained) {+ void draining.then(() => settleExporter("exporter shutdown", exporter.shutdown));+ return;+ }+ await waitForExporter(+ settleExporter("exporter shutdown", exporter.shutdown),+ "exporter shutdown",+ );+ },+ };++ function pendingDrain(): Promise<void> {+ cancelTimer();+ pending = pending.then(async () => {+ while (buffer.length > 0) {+ const batch = buffer.splice(0, maxExportBatchSize);+ await exportBatch(batch);+ }+ });+ return pending;+ }++ async function waitForExporter(operation: Promise<void>, name: string): Promise<boolean> {+ let timer: ReturnType<typeof setTimeout> | undefined;+ const timeout = new Promise<false>((resolve) => {+ timer = setTimeout(() => {+ log.warn(`${name} timed out; preserving exporter serialization`, {+ timeoutMillis: exportTimeoutMillis,+ });+ resolve(false);+ }, exportTimeoutMillis);+ timer.unref?.();+ });+ const completed = operation.then(() => true);+ try {+ return await Promise.race([completed, timeout]);+ } finally {+ clearTimeout(timer);+ }+ }++ async function settleExporter(name: string, operation: () => Promise<void>): Promise<void> {+ try {+ await operation.call(exporter);+ } catch (error: unknown) {+ log.warn(`${name} failed`, { error: formatError(error) });+ }+ }+}++function isSampled(span: unknown): boolean {+ if (typeof span !== "object" || span === null || !("spanContext" in span)) return false;+ const spanContext = (span as { readonly spanContext?: unknown }).spanContext;+ if (typeof spanContext !== "function") return false;+ const context = Reflect.apply(spanContext, span, []) as { readonly traceFlags?: unknown };+ return (+ typeof context.traceFlags === "number" &&+ (context.traceFlags & TraceFlags.SAMPLED) === TraceFlags.SAMPLED+ );+}diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts b/packages/eve/src/tracing/install-instrumentation-runtime.test.tsnew file mode 100644index 000000000..bd0c07fb8--- /dev/null+++ b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts@@ -0,0 +1,66 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js";+import { otelIntegration, collectOtelPipeline } from "#tracing/otel-declaration.js";++const { forceFlush, shutdown } = vi.hoisted(() => ({+ forceFlush: vi.fn(async () => undefined),+ shutdown: vi.fn(async () => undefined),+}));++vi.mock("#tracing/otel-registration.js", async (importOriginal) => {+ const actual = await importOriginal<typeof import("#tracing/otel-registration.js")>();+ return {+ ...actual,+ registerOtelPipeline: () => ({+ forceFlush,+ idGenerator: {+ allocateSpanId: () => "1".repeat(16),+ withSpanId: (_spanId: string, run: () => unknown) => run(),+ },+ shutdown,+ }),+ };+});++vi.mock("#tracing/agent-otel-provider.js", () => ({+ createAgentOtelInstrumentation: () => ({+ hook: {},+ runInContext: (_operation: unknown, execute: () => PromiseLike<unknown>) => execute(),+ }),+}));++const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime");++describe("installInstrumentationRuntime", () => {+ beforeEach(() => {+ forceFlush.mockClear();+ shutdown.mockClear();+ delete (globalThis as Record<symbol, unknown>)[RUNTIME_GLOBAL_KEY];+ });++ it("flushes and shuts down the registered tracer provider", async () => {+ const providerFlush = vi.fn();+ const providerShutdown = vi.fn();+ const runtime = installInstrumentationRuntime({+ collected: collectOtelPipeline([otelIntegration()]),+ frameworkVersion: "test",+ providers: [{ flush: providerFlush, shutdown: providerShutdown }],+ serviceName: "weather",+ });++ await runtime.forceFlush();+ await runtime.shutdown();+ await runtime.shutdown();++ expect(forceFlush).toHaveBeenCalledOnce();+ expect(providerFlush).toHaveBeenCalledOnce();+ expect(runtime.otelSettings).toEqual({+ recordInputs: true,+ recordOutputs: true,+ traceChannelRequests: false,+ });+ expect(shutdown).toHaveBeenCalledOnce();+ expect(providerShutdown).toHaveBeenCalledOnce();+ });+});diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.ts b/packages/eve/src/tracing/install-instrumentation-runtime.tsnew file mode 100644index 000000000..a0dd60bbe--- /dev/null+++ b/packages/eve/src/tracing/install-instrumentation-runtime.ts@@ -0,0 +1,98 @@+import { trace } from "#compiled/@opentelemetry/api/index.js";+import type { SpanProcessor } from "#compiled/@vercel/otel/index.js";++import {+ createInstrumentationHooks,+ type InstrumentationProviderDefinition,+} from "#harness/instrumentation-lifecycle.js";+import {+ registerInstrumentationRuntime,+ type InstrumentationRuntime,+} from "#harness/instrumentation-runtime.js";+import { createLogger, formatError } from "#internal/logging.js";+import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";+import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js";+import { hasSessionRelease, type LocalTracesProcessor } from "#tracing/local-traces.js";+import type { CollectedOtel } from "#tracing/otel-declaration.js";+import { registerOtelPipeline, type RegisteredOtelPipeline } from "#tracing/otel-registration.js";++const log = createLogger("tracing.install-instrumentation-runtime");++/** Installs the bus and the one OpenTelemetry pipeline collected for this process. */+export function installInstrumentationRuntime(input: {+ readonly collected: CollectedOtel;+ readonly frameworkVersion: string;+ readonly providers: readonly InstrumentationProviderDefinition[];+ readonly serviceName: string;+}): InstrumentationRuntime {+ const providers: InstrumentationProviderDefinition[] = [...input.providers];+ let otelRuntime: RegisteredOtelPipeline | undefined;+ let runInContext: InstrumentationRuntime["runInContext"] = (_operation, execute) => execute();++ if (input.collected.declared) {+ otelRuntime = registerOtelPipeline({+ pipeline: input.collected.pipeline,+ serviceName: input.serviceName,+ });+ const agentOtel = createAgentOtelInstrumentation({+ frameworkVersion: input.frameworkVersion,+ idGenerator: otelRuntime.idGenerator,+ recordInputs: input.collected.settings.recordInputs,+ recordOutputs: input.collected.settings.recordOutputs,+ stateStore: new ContextAgentTraceStateStore(),+ tracer: trace.getTracer("eve.agent", input.frameworkVersion),+ });+ providers.unshift(agentOtel.hook);+ runInContext = agentOtel.runInContext;++ const releasable = input.collected.pipeline.spanProcessors+ .filter(isSpanProcessor)+ .filter(hasSessionRelease);+ if (releasable.length > 0) providers.push(sessionReleaseProvider(releasable));+ }++ let shutdown: Promise<void> | undefined;+ return registerInstrumentationRuntime({+ forceFlush: () =>+ settleAll([+ ...(otelRuntime === undefined ? [] : [otelRuntime.forceFlush]),+ ...providers.map((provider) => () => provider.flush?.()),+ ]),+ hooks: createInstrumentationHooks(providers),+ otelSettings: input.collected.declared ? input.collected.settings : undefined,+ runInContext,+ shutdown: () => {+ shutdown ??= settleAll([+ ...(otelRuntime === undefined ? [] : [otelRuntime.shutdown]),+ ...providers.map((provider) => () => provider.shutdown?.()),+ ]);+ return shutdown;+ },+ });+}++function isSpanProcessor(processor: SpanProcessor | "auto"): processor is SpanProcessor {+ return processor !== "auto";+}++function sessionReleaseProvider(+ processors: readonly LocalTracesProcessor[],+): InstrumentationProviderDefinition {+ const release = async (event: { readonly sessionId: string }): Promise<void> => {+ await Promise.all(processors.map((processor) => processor.releaseSession(event.sessionId)));+ };+ return {+ events: { "session.completed": release, "session.failed": release },+ name: "eve.session-release",+ };+}++/** One failing drain must not take the others or the step awaiting them with it. */+async function settleAll(operations: readonly (() => void | PromiseLike<void>)[]): Promise<void> {+ const results = await Promise.allSettled(operations.map(async (run) => run()));+ for (const result of results) {+ if (result.status === "rejected") {+ log.warn("instrumentation drain failed", { error: formatError(result.reason) });+ }+ }+}diff --git a/packages/eve/src/tracing/local-instrumentation-runtime-conflict.scenario.test.ts b/packages/eve/src/tracing/local-instrumentation-runtime-conflict.scenario.test.tsindex 5db09a7c8..6d372cf22 100644--- a/packages/eve/src/tracing/local-instrumentation-runtime-conflict.scenario.test.ts+++ b/packages/eve/src/tracing/local-instrumentation-runtime-conflict.scenario.test.ts@@ -25,6 +25,6 @@ describe("local instrumentation runtime ownership", () => {frameworkVersion: "test",serviceName: "test-agent",}),- ).toThrow(/another runtime already exists/u);+ ).toThrow(/another runtime already owns the global tracer provider/u);});});diff --git a/packages/eve/src/tracing/local-instrumentation-runtime.ts b/packages/eve/src/tracing/local-instrumentation-runtime.tsindex 47201de28..5c4d8a5e1 100644--- a/packages/eve/src/tracing/local-instrumentation-runtime.ts+++ b/packages/eve/src/tracing/local-instrumentation-runtime.ts@@ -1,24 +1,10 @@-import { context, trace } from "#compiled/@opentelemetry/api/index.js";-import { registerOTel } from "#compiled/@vercel/otel/index.js";--import { ContextAgentTraceStateStore } from "#tracing/agent-trace-context-store.js";-import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js";-import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";-import { AgentTraceSpanProcessor } from "#tracing/agent-trace-span-processor.js";-import {- createInstrumentationHooks,- type InstrumentationProviderDefinition,-} from "#harness/instrumentation-lifecycle.js";import {getInstrumentationRuntime,- registerInstrumentationRuntime,type InstrumentationRuntime,} from "#harness/instrumentation-runtime.js";-import {- requestLocalTraceStorePrune,- resolveLocalTraceRetentionSettings,-} from "#tracing/local-trace-retention.js";-import { LocalTraceSpanProcessor } from "#tracing/local-trace-span-processor.js";+import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js";+import { createLocalTracesProcessor } from "#tracing/local-traces.js";+import { collectOtelPipeline, otel, otelIntegration } from "#tracing/otel-declaration.js";/** Installs the zero-config local OTel runtime once in an `eve dev` worker. */export function installLocalInstrumentationRuntime(input: {@@ -29,68 +15,26 @@ export function installLocalInstrumentationRuntime(input: {const existing = getInstrumentationRuntime();if (existing !== undefined) return existing;- const retention = resolveLocalTraceRetentionSettings();- // `EVE_TRACES=off` removes the writer but keeps the runtime: agent context- // still has to propagate so AI SDK and user spans nest correctly.- const processor = new AgentTraceSpanProcessor(- retention.enabled ? [new LocalTraceSpanProcessor(input.appRoot)] : [],- );- const idGenerator = new AgentSpanIdGenerator();- registerOTel({- autoDetectResources: false,- idGenerator,- instrumentations: [],- propagators: ["none"],- serviceName: input.serviceName,- spanProcessors: [processor],- });- const probe = trace.getTracer("eve.registration").startSpan("eve.otel.registration");- const activeContext = trace.setSpan(context.active(), probe);- const contextAttached = context.with(activeContext, () => trace.getActiveSpan() === probe);- probe.end();- if (!processor.isAttached() || !contextAttached) {- throw new Error("eve could not register OpenTelemetry because another runtime already exists.");- }- const agentOtel = createAgentOtelInstrumentation({- captureContent: process.env.EVE_TRACES_CONTENT !== "off",- frameworkVersion: input.frameworkVersion,- idGenerator,- stateStore: new ContextAgentTraceStateStore(),- tracer: trace.getTracer("eve.agent", input.frameworkVersion),- });- const releaseTrace: InstrumentationProviderDefinition = {- events: {- "session.completed": releaseSessionTrace,- "session.failed": releaseSessionTrace,+ // The zero-config default expressed with the same values an authored+ // `agent/instrumentation/` would declare, so this path exercises them.+ const spool = createLocalTracesProcessor({ appRoot: input.appRoot });+ const collectedPipeline = collectOtelPipeline([+ otel(),+ otelIntegration({ spanProcessors: [spool] }),+ ]);+ const captureContent = process.env.EVE_TRACES_CONTENT !== "off";+ const collected = {+ ...collectedPipeline,+ settings: {+ ...collectedPipeline.settings,+ recordInputs: captureContent,+ recordOutputs: captureContent,},};- // Startup sweep: a store left oversized by a killed dev server is bounded- // before this worker adds to it.- requestPrune();-- return registerInstrumentationRuntime({- forceFlush: () => processor.forceFlush(),- hooks: createInstrumentationHooks([agentOtel.hook, releaseTrace]),- runInContext: agentOtel.runInContext,+ return installInstrumentationRuntime({+ collected,+ frameworkVersion: input.frameworkVersion,+ providers: [],+ serviceName: input.serviceName,});-- async function releaseSessionTrace(event: { readonly sessionId: string }): Promise<void> {- // Settle pending segment writes before dropping liveness: a sweep already- // running reads the same live set, so releasing first would expose the- // trace to eviction while it is still being written.- await processor.forceFlush();- if (!processor.releaseSession(event.sessionId)) return;- requestPrune();- }-- function requestPrune(): void {- if (!retention.enabled) return;- requestLocalTraceStorePrune({- activeTraceIds: processor.activeTraceIds(),- appRoot: input.appRoot,- maxAgeMs: retention.maxAgeMs,- maxTotalBytes: retention.maxTotalBytes,- retainCount: retention.retainCount,- });- }}diff --git a/packages/eve/src/tracing/local-traces.test.ts b/packages/eve/src/tracing/local-traces.test.tsnew file mode 100644index 000000000..76557d037--- /dev/null+++ b/packages/eve/src/tracing/local-traces.test.ts@@ -0,0 +1,63 @@+import { afterEach, describe, expect, it, vi } from "vitest";++import { createLocalTracesProcessor } from "#tracing/local-traces.js";++vi.mock("#tracing/local-trace-span-processor.js", () => ({+ LocalTraceSpanProcessor: class {+ async forceFlush(): Promise<void> {}+ onEnd(): void {}+ onStart(): void {}+ async shutdown(): Promise<void> {}+ },+}));++vi.mock("#tracing/local-trace-retention.js", () => ({+ requestLocalTraceStorePrune: vi.fn(),+ resolveLocalTraceRetentionSettings: () => ({+ enabled: true,+ maxAgeMs: 1,+ maxTotalBytes: 1,+ retainCount: 1,+ }),+}));++function agentSpan(sessionId: string, traceId: string): unknown {+ return {+ attributes: { "agent.session.id": sessionId },+ spanContext: () => ({ traceId }),+ };+}++describe("createLocalTracesProcessor", () => {+ afterEach(() => {+ vi.unstubAllEnvs();+ });++ it("reports whether the released session owned any traces", async () => {+ const spool = createLocalTracesProcessor({ appRoot: "/tmp/eve-local-traces-test" });+ spool.onStart(agentSpan("session-one", "a".repeat(32)), undefined);++ // A subagent child owns none, so releasing it leaves the trace pinned.+ await expect(spool.releaseSession("child-one")).resolves.toBe(false);+ await expect(spool.releaseSession("session-one")).resolves.toBe(true);+ // Releasing twice is not an error, it just owns nothing the second time.+ await expect(spool.releaseSession("session-one")).resolves.toBe(false);+ });++ it("is a span processor, so it composes wherever one goes", () => {+ const spool = createLocalTracesProcessor({ appRoot: "/tmp/eve-local-traces-test" });+ expect(typeof spool.onStart).toBe("function");+ expect(typeof spool.onEnd).toBe("function");+ expect(typeof spool.forceFlush).toBe("function");+ expect(typeof spool.shutdown).toBe("function");+ });++ it("is inert outside a development worker", async () => {+ vi.stubEnv("EVE_DEV_WORKER_APP_ROOT", undefined);+ const processor = createLocalTracesProcessor();++ expect(() => processor.onEnd(agentSpan("session-one", "a".repeat(32)))).not.toThrow();+ await expect(processor.forceFlush()).resolves.toBeUndefined();+ await expect(processor.shutdown()).resolves.toBeUndefined();+ });+});diff --git a/packages/eve/src/tracing/local-traces.ts b/packages/eve/src/tracing/local-traces.tsnew file mode 100644index 000000000..7cf456c7e--- /dev/null+++ b/packages/eve/src/tracing/local-traces.ts@@ -0,0 +1,92 @@+import type { SpanProcessor } from "#compiled/@vercel/otel/index.js";++import { AgentTraceSpanProcessor } from "#tracing/agent-trace-span-processor.js";+import { LocalTraceSpanProcessor } from "#tracing/local-trace-span-processor.js";+import {+ requestLocalTraceStorePrune,+ resolveLocalTraceRetentionSettings,+} from "#tracing/local-trace-retention.js";++/**+ * The local spool, as a span processor.+ *+ * Session liveness and retention live in here rather than in the runtime that+ * installs it, so nothing an author puts in the same `spanProcessors` list can+ * see them — and eve's accept filter never reaches an author's exporters.+ */+export interface LocalTracesProcessor extends SpanProcessor {+ /**+ * Settles pending writes and drops one root session's liveness, then bounds+ * the store. A subagent child owns no traces, so releasing one is a no-op+ * and leaves the shared trace pinned until its root finishes.+ */+ releaseSession(sessionId: string): Promise<boolean>;+}++/** Whether a processor still exposes the local spool's session lifecycle. */+export function hasSessionRelease(processor: SpanProcessor): processor is LocalTracesProcessor {+ return typeof (processor as Partial<LocalTracesProcessor>).releaseSession === "function";+}++/**+ * Writes the OTLP/JSON spool under `.eve/traces/v1`.+ *+ * `EVE_TRACES=off` removes the writer but keeps the processor: eve still has+ * to observe spans to track which session owns which trace.+ *+ * Internal because of `releaseSession`, which eve's runtime drives off session+ * lifecycle.+ */+export function createLocalTracesProcessor(+ input: { readonly appRoot?: string } = {},+): LocalTracesProcessor {+ const appRoot = input.appRoot ?? process.env["EVE_DEV_WORKER_APP_ROOT"];+ if (appRoot === undefined) return inertLocalTracesProcessor();++ const retention = resolveLocalTraceRetentionSettings();+ const processor = new AgentTraceSpanProcessor(+ retention.enabled ? [new LocalTraceSpanProcessor(appRoot)] : [],+ );++ const requestPrune = (): void => {+ if (!retention.enabled) return;+ requestLocalTraceStorePrune({+ activeTraceIds: processor.activeTraceIds(),+ appRoot,+ maxAgeMs: retention.maxAgeMs,+ maxTotalBytes: retention.maxTotalBytes,+ retainCount: retention.retainCount,+ });+ };++ // Startup sweep: a store left oversized by a killed dev server is bounded+ // before this worker adds to it.+ requestPrune();++ return {+ forceFlush: () => processor.forceFlush(),+ onEnd: (span) => processor.onEnd(span),+ onStart: (span, parentContext) => processor.onStart(span, parentContext),+ async releaseSession(sessionId) {+ // Settle pending segment writes before dropping liveness: a sweep already+ // running reads the same live set, so releasing first would expose the+ // trace to eviction while it is still being written.+ await processor.forceFlush();+ if (!processor.releaseSession(sessionId)) return false;+ requestPrune();+ return true;+ },+ shutdown: () => processor.shutdown(),+ };+}++/** A production-authored `localTraces()` has no local development store. */+function inertLocalTracesProcessor(): LocalTracesProcessor {+ return {+ forceFlush: async () => undefined,+ onEnd: () => undefined,+ onStart: () => undefined,+ releaseSession: async () => false,+ shutdown: async () => undefined,+ };+}diff --git a/packages/eve/src/tracing/otel-declaration.test.ts b/packages/eve/src/tracing/otel-declaration.test.tsnew file mode 100644index 000000000..7067b72fd--- /dev/null+++ b/packages/eve/src/tracing/otel-declaration.test.ts@@ -0,0 +1,130 @@+import type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js";+import { describe, expect, it } from "vitest";++import {+ agentRunsIntegration,+ collectOtelPipeline,+ isOtelDeclaration,+ isOtelIntegration,+ otel,+ otelIntegration,+} from "#tracing/otel-declaration.js";++/** Collection only ever moves processors, so a fresh no-op is identity enough. */+function processor(): SpanProcessor {+ return {+ forceFlush: async () => undefined,+ onEnd: () => undefined,+ onStart: () => undefined,+ shutdown: async () => undefined,+ };+}++function exporter(): SpanExporter {+ return {+ export: (_spans, resultCallback) => {+ resultCallback({ code: 0 });+ },+ shutdown: async () => undefined,+ };+}++describe("otel", () => {+ it("declares settings without registering anything", () => {+ const declaration = otel({ sampler: "always_on" });+ expect(isOtelDeclaration(declaration)).toBe(true);+ expect(isOtelDeclaration({ options: {} })).toBe(false);+ });+});++describe("otelIntegration", () => {+ it("passes declared processors through untouched", () => {+ const first = processor();+ const integration = otelIntegration({ spanProcessors: [first] });++ expect(isOtelIntegration(integration)).toBe(true);+ expect(integration.spanProcessors).toStrictEqual([first]);+ });++ it("wraps an exporter in a batching processor, after any declared ones", () => {+ const first = processor();+ const integration = otelIntegration({+ spanProcessors: [first],+ traceExporter: exporter(),+ });++ expect(integration.spanProcessors).toHaveLength(2);+ expect(integration.spanProcessors[0]).toBe(first);+ });+});++describe("agentRunsIntegration", () => {+ it("uses Vercel's automatic processor by default", () => {+ const integration = agentRunsIntegration();++ expect(integration.spanProcessors).toStrictEqual(["auto"]);+ });+});++describe("collectOtelPipeline", () => {+ it("reports nothing declared when nothing did", () => {+ expect(collectOtelPipeline([]).declared).toBe(false);+ expect(collectOtelPipeline([{ not: "a declaration" }]).declared).toBe(false);+ });++ it("concatenates destinations in declaration order", () => {+ const [first, second, third] = [processor(), processor(), processor()];+ const collected = collectOtelPipeline([+ otelIntegration({ spanProcessors: [first, second] }),+ otelIntegration({ spanProcessors: [third] }),+ otel(),+ ]);++ expect(collected.declared).toBe(true);+ expect(collected.pipeline.spanProcessors).toStrictEqual([first, second, third]);+ });++ it("is declared by a destination alone, with no otel() beside it", () => {+ const collected = collectOtelPipeline([otelIntegration({ spanProcessors: [processor()] })]);++ expect(collected.declared).toBe(true);+ expect(collected.settings).toStrictEqual({+ functionId: undefined,+ recordInputs: true,+ recordOutputs: true,+ traceChannelRequests: false,+ });+ });++ it("carries the singletons onto the pipeline and the rest onto the settings", () => {+ const collected = collectOtelPipeline([+ otel({+ functionId: "weather",+ propagators: ["tracecontext"],+ resource: { "service.version": "abc" },+ sampler: "always_on",+ traceChannelRequests: true,+ }),+ ]);++ expect(collected.pipeline).toMatchObject({+ propagators: ["tracecontext"],+ resource: { "service.version": "abc" },+ sampler: "always_on",+ });+ expect(collected.settings).toStrictEqual({+ functionId: "weather",+ recordInputs: false,+ recordOutputs: false,+ traceChannelRequests: true,+ });+ });++ // A process has one tracer provider, so letting the first declaration win+ // would silently discard the second — the failure this throw exists to stop.+ it("refuses a second otel() rather than picking one", () => {+ expect(() =>+ collectOtelPipeline([otel({ sampler: "always_on" }), otel({ sampler: "always_off" })]),+ ).toThrow(/declares `otel\(\)` more than once/u);+ });+});diff --git a/packages/eve/src/tracing/otel-declaration.ts b/packages/eve/src/tracing/otel-declaration.tsnew file mode 100644index 000000000..d47bab81b--- /dev/null+++ b/packages/eve/src/tracing/otel-declaration.ts@@ -0,0 +1,209 @@+import type {+ PropagatorOrName,+ SamplerOrName,+ SpanExporter,+ SpanProcessor,+ SpanProcessorOrName,+} from "#compiled/@vercel/otel/index.js";++import { batchSpanProcessor } from "#tracing/batch-span-processor.js";++/**+ * The process-wide OpenTelemetry settings, declared by `otel()`.+ *+ * Everything here is a singleton, which is why it is one file: a process has+ * one tracer provider, so it has one resource, one sampler, and one propagator+ * set. Destinations are the plural half and live in `otelIntegration()`.+ *+ * `contextManager` and `instrumentations` are deliberately absent: eve's span+ * nesting depends on the former, and the latter cannot reach anything eve+ * imported before setup ran.+ */+export interface OtelOptions {+ /**+ * The function identifier attached to telemetry spans+ * (`ai.telemetry.functionId`). Defaults to the agent name.+ */+ readonly functionId?: string;+ /**+ * Whether to emit the inbound HTTP `SERVER` span that wraps each channel+ * request — the parent of the turn trace and of any `hook.resume` or+ * outgoing HTTP spans. Defaults to `false`.+ */+ readonly traceChannelRequests?: boolean;+ /**+ * Resource attributes merged into eve's own, which already carry the+ * service name.+ */+ readonly resource?: Readonly<Record<string, unknown>>;+ /**+ * Head sampling, and it is global: it decides whether a span is created at+ * all, so it thins eve's own sinks and the `traceparent` eve propagates+ * along with your exporters. To thin one backend only, drop spans in a+ * processor.+ */+ readonly sampler?: SamplerOrName;+ /** Composed into one propagator. All inject; the first to extract wins. */+ readonly propagators?: readonly PropagatorOrName[];+}++/** Where one `otelIntegration()` sends spans. */+export interface OtelIntegrationOptions {+ /** Merged into the pipeline in declaration order. */+ readonly spanProcessors?: readonly SpanProcessor[];+ /** Wrapped in eve's batching processor and appended after `spanProcessors`. */+ readonly traceExporter?: SpanExporter;+}++const OTEL_DECLARATION = Symbol.for("eve.instrumentation.otel");+const OTEL_INTEGRATION = Symbol.for("eve.instrumentation.otel-integration");++/**+ * The declared OpenTelemetry pipeline settings. eve collects this before+ * building the tracer provider, so it is a value rather than a side effect.+ */+export interface OtelDeclaration {+ readonly [OTEL_DECLARATION]: true;+ readonly options: OtelOptions;+}++/** One declared destination. A process may have as many as it has files. */+export interface OtelIntegration {+ readonly [OTEL_INTEGRATION]: true;+ readonly spanProcessors: readonly SpanProcessorOrName[];+}++/**+ * Declares the process-wide OpenTelemetry settings.+ *+ * This remains internal until the provider authoring API exposes it.+ */+export function otel(options: OtelOptions = {}): OtelDeclaration {+ return { [OTEL_DECLARATION]: true, options };+}++/**+ * Declares one destination for this agent's traces.+ *+ * A `traceExporter` is wrapped in eve's batching processor, which is what makes+ * the one-line form of a hosted backend enough. Pass `spanProcessors` instead+ * when the destination needs its own batching, sampling, or filtering.+ */+export function otelIntegration(options: OtelIntegrationOptions = {}): OtelIntegration {+ const declared = options.spanProcessors ?? [];+ return {+ [OTEL_INTEGRATION]: true,+ spanProcessors:+ options.traceExporter === undefined+ ? declared+ : [...declared, batchSpanProcessor(options.traceExporter)],+ };+}++/** Vercel Agent Runs through @vercel/otel's automatic processor. @internal */+export function agentRunsIntegration(): OtelIntegration {+ return {+ [OTEL_INTEGRATION]: true,+ spanProcessors: ["auto"],+ };+}++export function isOtelDeclaration(value: unknown): value is OtelDeclaration {+ return (+ typeof value === "object" &&+ value !== null &&+ (value as Partial<OtelDeclaration>)[OTEL_DECLARATION] === true+ );+}++export function isOtelIntegration(value: unknown): value is OtelIntegration {+ return (+ typeof value === "object" &&+ value !== null &&+ (value as Partial<OtelIntegration>)[OTEL_INTEGRATION] === true+ );+}++/** The one pipeline a process can register. @internal */+export interface OtelPipeline {+ readonly propagators?: readonly PropagatorOrName[];+ readonly resource?: Readonly<Record<string, unknown>>;+ readonly sampler?: SamplerOrName;+ readonly spanProcessors: readonly SpanProcessorOrName[];+}++/** What the harness reads at turn time, as opposed to at registration. @internal */+export interface OtelHarnessSettings {+ readonly functionId?: string;+ readonly traceChannelRequests: boolean;+ /**+ * What to materialize on spans at all. Each destination independently strips+ * anything it declined before export.+ */+ readonly recordInputs?: boolean;+ readonly recordOutputs?: boolean;+}++/** @internal */+export interface CollectedOtel {+ /**+ * Whether anything declared OpenTelemetry. False means eve should leave the+ * global tracer provider slot alone rather than register an empty pipeline.+ */+ readonly declared: boolean;+ readonly pipeline: OtelPipeline;+ readonly settings: OtelHarnessSettings;+}++/**+ * Folds the declared values into the one pipeline a process can register.+ *+ * Destinations concatenate in declaration order. The singletons cannot: two+ * `otel()` values is a boot error rather than a silent win for whichever eve+ * happened to visit first. With one declaration per file that collision needs+ * two files both exporting `otel()`, which is the only way to reach it.+ *+ * @internal+ */+export function collectOtelPipeline(values: readonly unknown[]): CollectedOtel {+ const spanProcessors: SpanProcessorOrName[] = [];+ let declaration: OtelDeclaration | undefined;+ let declared = false;+ let recordInputs = false;+ let recordOutputs = false;++ for (const value of values) {+ if (isOtelIntegration(value)) {+ declared = true;+ recordInputs = true;+ recordOutputs = true;+ spanProcessors.push(...value.spanProcessors);+ continue;+ }+ if (!isOtelDeclaration(value)) continue;+ if (declaration !== undefined) {+ throw new Error(+ "Instrumentation declares `otel()` more than once. One process has one OpenTelemetry tracer provider, so it has one resource, one sampler, and one propagator set — declare them in a single `otel()`.",+ );+ }+ declared = true;+ declaration = value;+ }++ const options = declaration?.options ?? {};+ return {+ declared,+ pipeline: {+ propagators: options.propagators,+ resource: options.resource,+ sampler: options.sampler,+ spanProcessors,+ },+ settings: {+ functionId: options.functionId,+ recordInputs,+ recordOutputs,+ traceChannelRequests: options.traceChannelRequests === true,+ },+ };+}diff --git a/packages/eve/src/tracing/otel-registration.scenario.test.ts b/packages/eve/src/tracing/otel-registration.scenario.test.tsnew file mode 100644index 000000000..85d5e3d6f--- /dev/null+++ b/packages/eve/src/tracing/otel-registration.scenario.test.ts@@ -0,0 +1,101 @@+import { context, propagation, trace, type Context } from "@opentelemetry/api";+import {+ BasicTracerProvider,+ InMemorySpanExporter,+ SimpleSpanProcessor,+} from "@opentelemetry/sdk-trace-base";+import { afterEach, describe, expect, it, vi } from "vitest";++import { registerOtelPipeline } from "#tracing/otel-registration.js";++afterEach(() => {+ context.disable();+ propagation.disable();+ trace.disable();+});++describe("registerOtelPipeline", () => {+ it("verifies tracer ownership when the sampler records nothing", () => {+ expect(() =>+ registerOtelPipeline({+ pipeline: { sampler: "always_off", spanProcessors: [] },+ serviceName: "weather",+ }),+ ).not.toThrow();+ });++ it("fails without replacing another runtime's global propagator", async () => {+ let foreignInjections = 0;+ const shutdown = vi.fn(async () => undefined);+ expect(+ propagation.setGlobalPropagator({+ extract: (carrierContext: Context) => carrierContext,+ fields: () => [],+ inject: () => {+ foreignInjections += 1;+ },+ }),+ ).toBe(true);+ const tracerDelegate = currentTracerDelegate();++ expect(() =>+ registerOtelPipeline({+ pipeline: {+ spanProcessors: [+ { forceFlush: async () => {}, onEnd: () => {}, onStart: () => {}, shutdown },+ ],+ },+ serviceName: "weather",+ }),+ ).toThrow(/another runtime already owns the global propagator/u);++ const injectionsAfterFailure = foreignInjections;+ propagation.inject(context.active(), {}, { set: () => {} });+ expect(foreignInjections).toBe(injectionsAfterFailure + 1);+ expect(currentTracerDelegate()).toBe(tracerDelegate);+ await vi.waitFor(() => expect(shutdown).toHaveBeenCalledOnce());+ });++ it("leaves an existing tracer provider untouched when registration fails", () => {+ const provider = new BasicTracerProvider();+ const shutdown = vi.spyOn(provider, "shutdown");+ expect(trace.setGlobalTracerProvider(provider)).toBe(true);++ expect(() =>+ registerOtelPipeline({+ pipeline: { spanProcessors: [] },+ serviceName: "weather",+ }),+ ).toThrow(/another runtime already owns the global tracer provider/u);++ expect(currentTracerDelegate()).toBe(provider);+ expect(shutdown).not.toHaveBeenCalled();+ expect(+ propagation.setGlobalPropagator({+ extract: (carrierContext: Context) => carrierContext,+ fields: () => [],+ inject: () => {},+ }),+ ).toBe(true);+ });++ it("does not export the private registration span", async () => {+ const exporter = new InMemorySpanExporter();+ const processor = new SimpleSpanProcessor(exporter);+ registerOtelPipeline({+ pipeline: { spanProcessors: [processor] },+ serviceName: "weather",+ });++ trace.getTracer("test").startSpan("user.work").end();+ await processor.forceFlush();++ expect(exporter.getFinishedSpans().map((span) => span.name)).toEqual(["user.work"]);+ await processor.shutdown();+ });+});++function currentTracerDelegate(): unknown {+ const provider = trace.getTracerProvider() as { getDelegate?: () => unknown };+ return provider.getDelegate?.();+}diff --git a/packages/eve/src/tracing/otel-registration.test.ts b/packages/eve/src/tracing/otel-registration.test.tsnew file mode 100644index 000000000..9e49ca976--- /dev/null+++ b/packages/eve/src/tracing/otel-registration.test.ts@@ -0,0 +1,102 @@+import { describe, expect, it, vi } from "vitest";++import { registerOtelPipeline } from "#tracing/otel-registration.js";++const { registerOTel } = vi.hoisted(() => ({ registerOTel: vi.fn() }));++vi.mock("#compiled/@vercel/otel/index.js", () => ({ registerOTel }));++describe("registerOtelPipeline", () => {+ it("maps eve's option names onto the ones @vercel/otel accepts", () => {+ registerOTel.mockImplementation(() => undefined);++ expect(() =>+ registerOtelPipeline({+ pipeline: {+ propagators: ["tracecontext"],+ resource: { "service.version": "abc" },+ sampler: "always_on",+ spanProcessors: [],+ },+ serviceName: "weather",+ }),+ ).toThrow();++ expect(registerOTel).toHaveBeenCalledWith(+ expect.objectContaining({+ attributes: { "service.version": "abc" },+ autoDetectResources: false,+ instrumentations: [],+ propagators: expect.arrayContaining(["tracecontext", expect.any(Object)]),+ serviceName: "weather",+ traceSampler: "always_on",+ }),+ );+ });++ it("omits the sampler entirely rather than passing undefined", () => {+ registerOTel.mockImplementation(() => undefined);++ expect(() =>+ registerOtelPipeline({ pipeline: { spanProcessors: [] }, serviceName: "weather" }),+ ).toThrow(/already owns the global tracer provider/u);++ const configuration = registerOTel.mock.calls.at(-1)?.[0] as Record<string, unknown>;+ expect("traceSampler" in configuration).toBe(false);+ // Absent propagators keep `"none"`; eve's private ownership marker follows it.+ expect(configuration["propagators"]).toEqual(["none", expect.any(Object)]);+ });++ it("filters the private registration span from authored processors", () => {+ const downstream = {+ forceFlush: vi.fn(async () => {}),+ onEnd: vi.fn(),+ onStart: vi.fn(),+ shutdown: vi.fn(async () => {}),+ };+ registerOTel.mockImplementation(() => undefined);++ expect(() =>+ registerOtelPipeline({+ pipeline: { spanProcessors: [downstream] },+ serviceName: "weather",+ }),+ ).toThrow();++ const configuration = registerOTel.mock.calls.at(-1)?.[0] as {+ spanProcessors: {+ onEnd(span: unknown): void;+ onStart(span: unknown, parentContext: unknown): void;+ }[];+ };+ const processor = configuration.spanProcessors[0]!;+ processor.onStart({ name: "eve.otel.registration" }, {});+ processor.onEnd({ name: "eve.otel.registration" });+ processor.onStart({ name: "agent.turn" }, {});+ processor.onEnd({ name: "agent.turn" });++ expect(downstream.onStart).toHaveBeenCalledExactlyOnceWith({ name: "agent.turn" }, {});+ expect(downstream.onEnd).toHaveBeenCalledExactlyOnceWith({ name: "agent.turn" });+ });++ it("passes Vercel's automatic processor through", () => {+ registerOTel.mockImplementation(() => undefined);++ expect(() =>+ registerOtelPipeline({ pipeline: { spanProcessors: ["auto"] }, serviceName: "weather" }),+ ).toThrow();++ const configuration = registerOTel.mock.calls.at(-1)?.[0] as {+ spanProcessors: unknown[];+ };+ expect(configuration.spanProcessors[0]).toBe("auto");+ });++ it("throws when the registration never reached a processor", () => {+ registerOTel.mockImplementation(() => undefined);++ expect(() =>+ registerOtelPipeline({ pipeline: { spanProcessors: [] }, serviceName: "weather" }),+ ).toThrow(/another runtime already owns/u);+ });+});diff --git a/packages/eve/src/tracing/otel-registration.ts b/packages/eve/src/tracing/otel-registration.tsnew file mode 100644index 000000000..e2337f4fb--- /dev/null+++ b/packages/eve/src/tracing/otel-registration.ts@@ -0,0 +1,186 @@+import { context, propagation, trace, type Context } from "#compiled/@opentelemetry/api/index.js";+import {+ registerOTel,+ type Configuration,+ type SpanProcessor,+ type SpanProcessorOrName,+} from "#compiled/@vercel/otel/index.js";++import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";+import type { OtelPipeline } from "#tracing/otel-declaration.js";++const REGISTRATION_SPAN_NAME = "eve.otel.registration";++class RegistrationMarkerPropagator {+ #injected = false;++ extract(carrierContext: Context): Context {+ return carrierContext;+ }++ fields(): string[] {+ return [];+ }++ inject(): void {+ this.#injected = true;+ }++ isInstalled(): boolean {+ this.#injected = false;+ propagation.inject(context.active(), {}, { set: () => {} });+ return this.#injected;+ }+}++/** Keeps eve's ownership check out of every authored destination. */+class PrivateSpanFilteringProcessor implements SpanProcessor {+ private readonly processors: readonly SpanProcessor[];++ constructor(processors: readonly SpanProcessor[]) {+ this.processors = processors;+ }++ async forceFlush(): Promise<void> {+ await Promise.all(this.processors.map((processor) => processor.forceFlush()));+ }++ onEnd(span: unknown): void {+ if (isRegistrationSpan(span)) return;+ for (const processor of this.processors) processor.onEnd(span);+ }++ onStart(span: unknown, parentContext: unknown): void {+ if (isRegistrationSpan(span)) return;+ for (const processor of this.processors) processor.onStart(span, parentContext);+ }++ async shutdown(): Promise<void> {+ await Promise.all(this.processors.map((processor) => processor.shutdown()));+ }+}++/**+ * Builds the process's one OpenTelemetry tracer provider from a merged+ * declaration, then proves it took the global slot.+ *+ * `registerOTel` reports a refused registration only through `diag`, which+ * goes nowhere unless `OTEL_LOG_LEVEL` is set, so a second caller would export+ * nothing and say nothing. eve primes its id generator and installs a private+ * propagator, then verifies both through the global APIs.+ */+export function registerOtelPipeline(input: {+ readonly pipeline: OtelPipeline;+ readonly serviceName: string;+}): RegisteredOtelPipeline {+ const { pipeline } = input;+ const idGenerator = new AgentSpanIdGenerator();+ const markerPropagator = new RegistrationMarkerPropagator();+ const configuration: Configuration = {+ attributes: pipeline.resource,+ autoDetectResources: false,+ idGenerator,+ // eve imports the model SDK before any of this runs, so an auto+ // instrumentation registered here could not patch it anyway.+ instrumentations: [],+ propagators: [...(pipeline.propagators ?? ["none"]), markerPropagator],+ serviceName: input.serviceName,+ spanProcessors: pipeline.spanProcessors.map((processor) =>+ isSpanProcessor(processor) ? new PrivateSpanFilteringProcessor([processor]) : processor,+ ),+ };+ registerOTel(+ // Absent means "let `@vercel/otel` decide", which is not the same as+ // passing an explicit `undefined` sampler.+ pipeline.sampler === undefined+ ? configuration+ : { ...configuration, traceSampler: pipeline.sampler },+ );++ const ownsTracer = globalTracerUses(idGenerator);+ const ownsPropagator = markerPropagator.isInstalled();+ if (!ownsTracer || !ownsPropagator) {+ rollbackRegistration({ ownsPropagator, ownsTracer });+ }+ if (!ownsTracer) {+ throw new Error(+ "eve could not register OpenTelemetry because another runtime already owns the global tracer provider. Remove the other `registerOTel` call, or move its exporters into eve's `otelIntegration({ spanProcessors: [...] })`.",+ );+ }+ if (!ownsPropagator) {+ throw new Error(+ "eve could not register OpenTelemetry because another runtime already owns the global propagator. Remove the other global propagator registration and declare propagators through eve's `otel()` instead.",+ );+ }+ const provider = runtimeTracerProvider();+ if (typeof provider.forceFlush !== "function" || typeof provider.shutdown !== "function") {+ rollbackRegistration({ ownsPropagator, ownsTracer });+ throw new Error("The registered OpenTelemetry tracer provider has no lifecycle methods.");+ }+ return {+ forceFlush: () => provider.forceFlush!(),+ idGenerator,+ shutdown: () => provider.shutdown!(),+ };+}++/** Lifecycle retained from the tracer provider that owns every destination. */+export interface RegisteredOtelPipeline {+ readonly forceFlush: () => Promise<void>;+ readonly idGenerator: AgentSpanIdGenerator;+ readonly shutdown: () => Promise<void>;+}++interface RuntimeTracerProvider {+ forceFlush(): Promise<void>;+ shutdown(): Promise<void>;+}++interface ProxyTracerProvider {+ getDelegate(): unknown;+}++function rollbackRegistration(input: {+ readonly ownsPropagator: boolean;+ readonly ownsTracer: boolean;+}): void {+ if (input.ownsPropagator) {+ (propagation as typeof propagation & { disable(): void }).disable();+ }+ if (!input.ownsTracer) return;+ const provider = runtimeTracerProvider();+ if (typeof provider.shutdown === "function") {+ try {+ void provider.shutdown().catch(() => {});+ } catch {}+ }+ (trace as typeof trace & { disable(): void }).disable();+}++function runtimeTracerProvider(): Partial<RuntimeTracerProvider> {+ const globalProvider = trace.getTracerProvider() as Partial<ProxyTracerProvider>;+ return (globalProvider.getDelegate?.() ?? globalProvider) as Partial<RuntimeTracerProvider>;+}++function globalTracerUses(idGenerator: AgentSpanIdGenerator): boolean {+ const spanId = idGenerator.allocateSpanId();+ const probe = idGenerator.withSpanId(spanId, () =>+ trace.getTracer("eve.registration").startSpan(REGISTRATION_SPAN_NAME),+ );+ // Deliberately not ended: named processors are resolved inside @vercel/otel+ // and cannot be wrapped, while an unended span is never exported.+ return probe.spanContext().spanId === spanId;+}++function isRegistrationSpan(span: unknown): boolean {+ return (+ typeof span === "object" &&+ span !== null &&+ "name" in span &&+ span.name === REGISTRATION_SPAN_NAME+ );+}++function isSpanProcessor(processor: SpanProcessorOrName): processor is SpanProcessor {+ return processor !== "auto";+}diff --git a/packages/eve/src/tracing/vercel-runtime-span-exporter.test.ts b/packages/eve/src/tracing/vercel-runtime-span-exporter.test.tsnew file mode 100644index 000000000..f291fe934--- /dev/null+++ b/packages/eve/src/tracing/vercel-runtime-span-exporter.test.ts@@ -0,0 +1,42 @@+import { afterEach, describe, expect, it, vi } from "vitest";++import { vercelRuntimeSpanExporter } from "#tracing/vercel-runtime-span-exporter.js";++const { serializeRequest } = vi.hoisted(() => ({+ serializeRequest: vi.fn(() => new TextEncoder().encode('{"resourceSpans":[]}')),+}));++vi.mock("#compiled/@opentelemetry/otlp-transformer/index.js", () => ({+ JsonTraceSerializer: { serializeRequest },+}));++const REQUEST_CONTEXT = Symbol.for("@vercel/request-context");++afterEach(() => {+ delete (globalThis as Record<symbol, unknown>)[REQUEST_CONTEXT];+ serializeRequest.mockClear();+});++describe("vercelRuntimeSpanExporter", () => {+ it("reports through the current Vercel request context", () => {+ const reportSpans = vi.fn();+ (globalThis as Record<symbol, unknown>)[REQUEST_CONTEXT] = {+ get: () => ({ telemetry: { reportSpans } }),+ };+ const result = vi.fn();++ vercelRuntimeSpanExporter().export([{}], result);++ expect(reportSpans).toHaveBeenCalledExactlyOnceWith({ resourceSpans: [] });+ expect(result).toHaveBeenCalledExactlyOnceWith({ code: 0 });+ });++ it("is a successful no-op without request telemetry", () => {+ const result = vi.fn();++ vercelRuntimeSpanExporter().export([{}], result);++ expect(serializeRequest).not.toHaveBeenCalled();+ expect(result).toHaveBeenCalledExactlyOnceWith({ code: 0 });+ });+});diff --git a/packages/eve/src/tracing/vercel-runtime-span-exporter.ts b/packages/eve/src/tracing/vercel-runtime-span-exporter.tsnew file mode 100644index 000000000..49d1519ba--- /dev/null+++ b/packages/eve/src/tracing/vercel-runtime-span-exporter.ts@@ -0,0 +1,81 @@+import { JsonTraceSerializer } from "#compiled/@opentelemetry/otlp-transformer/index.js";+import { TraceFlags } from "#compiled/@opentelemetry/api/index.js";+import type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js";++import { createLogger, formatError } from "#internal/logging.js";++const VERCEL_REQUEST_CONTEXT = Symbol.for("@vercel/request-context");+const log = createLogger("tracing.vercel-runtime-span-exporter");++interface VercelRequestContextReader {+ get():+ | {+ readonly telemetry?: { readonly reportSpans: (data: unknown) => void };+ }+ | undefined;+}++/** The request-context transport behind @vercel/otel's automatic processor. */+export function vercelRuntimeSpanExporter(): SpanExporter {+ return {+ export(spans, resultCallback) {+ const reader = (globalThis as Record<symbol, unknown>)[VERCEL_REQUEST_CONTEXT] as+ | VercelRequestContextReader+ | undefined;+ const telemetry = reader?.get()?.telemetry;+ if (telemetry === undefined) {+ resultCallback({ code: 0 });+ return;+ }++ try {+ const serialized = JsonTraceSerializer.serializeRequest([...spans]);+ if (serialized === undefined) throw new Error("Failed to serialize spans.");+ telemetry.reportSpans(JSON.parse(new TextDecoder().decode(serialized)) as unknown);+ resultCallback({ code: 0 });+ } catch (error: unknown) {+ resultCallback({+ code: 1,+ error: error instanceof Error ? error : new Error(String(error)),+ });+ }+ },+ forceFlush: async () => undefined,+ shutdown: async () => undefined,+ };+}++/** Reports immediately so the span stays attached to its request context. */+export function vercelRuntimeSpanProcessor(): SpanProcessor {+ const exporter = vercelRuntimeSpanExporter();+ let stopped = false;+ return {+ forceFlush: () => exporter.forceFlush?.() ?? Promise.resolve(),+ onEnd(span) {+ if (stopped || !isSampled(span)) return;+ exporter.export([span], (result) => {+ if (result.code !== 0) {+ log.warn("Agent Runs export failed", {+ error: formatError(result.error ?? new Error("Span export failed.")),+ });+ }+ });+ },+ onStart() {},+ async shutdown() {+ stopped = true;+ await exporter.shutdown();+ },+ };+}++function isSampled(span: unknown): boolean {+ if (typeof span !== "object" || span === null || !("spanContext" in span)) return false;+ const spanContext = (span as { readonly spanContext?: unknown }).spanContext;+ if (typeof spanContext !== "function") return false;+ const value = Reflect.apply(spanContext, span, []) as { readonly traceFlags?: unknown };+ return (+ typeof value.traceFlags === "number" &&+ (value.traceFlags & TraceFlags.SAMPLED) === TraceFlags.SAMPLED+ );+}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 681083bd0..ed213bbf4 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1286,6 +1286,9 @@ importers:'@opentelemetry/context-async-hooks':specifier: 'catalog:'version: 2.6.1(@opentelemetry/[email protected])+ '@opentelemetry/core':+ specifier: 'catalog:'+ version: 2.6.1(@opentelemetry/[email protected])'@opentelemetry/otlp-transformer':specifier: 0.214.0version: 0.214.0(@opentelemetry/[email protected])