The Open Framework for Building Agents
feat(eve): add the instrumentation provider layout (#1863)
Chad Hietalac90a459104e3
diff --git a/.changeset/instrumentation-provider-layout.md b/.changeset/instrumentation-provider-layout.mdnew file mode 100644index 000000000..1e40d0691--- /dev/null+++ b/.changeset/instrumentation-provider-layout.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Add the experimental `agent/instrumentation/` provider layout with durable lifecycle handlers, including user input boundaries and action settlement-time, outcome, error-code, and usage metadata, final setup context, reserved OpenTelemetry destinations, and coordinated flush and shutdown. OpenTelemetry singleton settings and destinations are exposed through `eve/instrumentation/otel`, and eve's AI SDK bridge composes with registered integrations.diff --git a/packages/eve/package.json b/packages/eve/package.jsonindex 5346416a2..2cf228c5b 100644--- a/packages/eve/package.json+++ b/packages/eve/package.json@@ -206,6 +206,11 @@"import": "./dist/src/public/instrumentation/index.js","default": "./dist/src/public/instrumentation/index.js"},+ "./instrumentation/otel": {+ "types": "./dist/src/public/instrumentation/otel.d.ts",+ "import": "./dist/src/public/instrumentation/otel.js",+ "default": "./dist/src/public/instrumentation/otel.js"+ },"./schedules": {"types": "./dist/src/public/schedules/index.d.ts","import": "./dist/src/public/schedules/index.js",diff --git a/packages/eve/src/compiler/manifest.ts b/packages/eve/src/compiler/manifest.tsindex fd5a9f432..21926310b 100644--- a/packages/eve/src/compiler/manifest.ts+++ b/packages/eve/src/compiler/manifest.ts@@ -439,6 +439,7 @@ const compiledAgentConfigBaseFields = {description: z.string().optional(),experimental: z.object({+ instrumentationProviders: z.boolean().optional(),subagentPersistentSessions: z.boolean().optional(),workflow: compiledAgentWorkflowDefinitionSchema.optional(),})@@ -917,6 +918,7 @@ function cloneCompiledAgentDefinition(config: CompiledAgentDefinition): Compiledconfig.experimental === undefined? undefined: {+ instrumentationProviders: config.experimental.instrumentationProviders,subagentPersistentSessions: config.experimental.subagentPersistentSessions,workflow:config.experimental.workflow === undefineddiff --git a/packages/eve/src/compiler/normalize-agent-config.ts b/packages/eve/src/compiler/normalize-agent-config.tsindex 8faf3dfb5..0c0bc81b7 100644--- a/packages/eve/src/compiler/normalize-agent-config.ts+++ b/packages/eve/src/compiler/normalize-agent-config.ts@@ -185,6 +185,10 @@ function normalizeExperimentalDefinition(const compiledExperimental: Mutable<NonNullable<CompiledAgentDefinition["experimental"]>> = {};+ if (experimental.instrumentationProviders !== undefined) {+ compiledExperimental.instrumentationProviders = experimental.instrumentationProviders;+ }+if (experimental.subagentPersistentSessions !== undefined) {compiledExperimental.subagentPersistentSessions = experimental.subagentPersistentSessions;}diff --git a/packages/eve/src/discover/agent.integration.test.ts b/packages/eve/src/discover/agent.integration.test.tsindex fb726983c..39b6daf10 100644--- a/packages/eve/src/discover/agent.integration.test.ts+++ b/packages/eve/src/discover/agent.integration.test.ts@@ -511,6 +511,23 @@ describe("discoverAgent (memory)", () => {]);});+ it("recognizes the instrumentation provider directory", async () => {+ const project = buildMemoryAgentProject({+ agentDirectories: ["instrumentation"],+ agentFiles: {+ "instructions.md": "You are a precise assistant.",+ },+ });++ const result = await discoverAgent({+ agentRoot: project.agentRoot,+ appRoot: project.appRoot,+ source: project.source,+ });++ expect(result.diagnostics).toEqual([]);+ });+it("rejects authored tool filenames that violate the tool-name charset", async () => {const project = buildMemoryAgentProject({agentFiles: {diff --git a/packages/eve/src/discover/filesystem.ts b/packages/eve/src/discover/filesystem.tsindex 814233d3c..2e96b3d04 100644--- a/packages/eve/src/discover/filesystem.ts+++ b/packages/eve/src/discover/filesystem.ts@@ -44,6 +44,7 @@ export type AgentRootEntryKind =| "extensions-directory"| "hooks-directory"| "ignored-directory"+ | "instrumentation-directory"| "instructions-directory"| "instructions-markdown"| "instructions-module"@@ -179,6 +180,10 @@ export function classifyAgentRootEntry(return "instructions-directory";}+ if (name === "instrumentation") {+ return "instrumentation-directory";+ }+if (name === "lib") {return "lib-directory";}diff --git a/packages/eve/src/evals/cli/eval.ts b/packages/eve/src/evals/cli/eval.tsindex 26a709075..2ed4991ac 100644--- a/packages/eve/src/evals/cli/eval.ts+++ b/packages/eve/src/evals/cli/eval.ts@@ -1,8 +1,13 @@+import { randomUUID } from "node:crypto";import { readFile } from "node:fs/promises";import { basename, join } from "node:path";import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js";import { shutdownActiveSandboxHandles } from "#execution/sandbox/active-handles.js";+import {+ EVE_EVALUATION_ENV_FLAG,+ EVE_EVALUATION_RUN_ID_ENV,+} from "#internal/application/dev-environment.js";import { resolveApplicationRoot } from "#internal/application/paths.js";import { createDevelopmentServer, type DevelopmentServer } from "#internal/nitro/host.js";import { createEvalClient } from "#evals/cli/eval-client.js";@@ -141,6 +146,10 @@ export async function runEvalCommand(url: options.url,});} else {+ // Set before the server boots, because a provider's `setup` reads it+ // once at startup and never again.+ process.env[EVE_EVALUATION_ENV_FLAG] = "1";+ process.env[EVE_EVALUATION_RUN_ID_ENV] = randomUUID();devServer = createDevelopmentServer(appRoot, { host: "127.0.0.1", port: 0 });const started = await devServer.start();client = await createEvalClient({ kind: "local", url: started.url });diff --git a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts b/packages/eve/src/harness/ai-sdk-hook-bridge.test.tsindex 0a51032d6..f0fbea575 100644--- a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts+++ b/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts@@ -94,7 +94,10 @@ describe("createAiSdkHookBridge", () => {it("passes the identity captured at model-call start to the context runner", async () => {const ids: string[] = [];const hooks = createInstrumentationHooks([- { events: { "model.call.started": (event) => void ids.push(event.idempotencyKey) } },+ {+ events: { "model.call.started": (event) => void ids.push(event.idempotencyKey) },+ name: "identity",+ },]);const bridge = createAiSdkHookBridge(scope, hooks, (operation, execute) => {ids.push(operation.idempotencyKey);@@ -132,7 +135,10 @@ describe("createAiSdkHookBridge", () => {it("derives replay-stable model identity without the AI SDK call ID", async () => {const keys: string[] = [];const hooks = createInstrumentationHooks([- { events: { "model.call.started": (event) => void keys.push(event.idempotencyKey) } },+ {+ events: { "model.call.started": (event) => void keys.push(event.idempotencyKey) },+ name: "identity",+ },]);for (const callId of ["sdk-random-1", "sdk-random-2"]) {@@ -155,6 +161,7 @@ describe("createAiSdkHookBridge", () => {events.push(event);},},+ name: "metadata",},]);const bridge = createAiSdkHookBridge(scope, hooks);@@ -183,11 +190,13 @@ describe("createAiSdkHookBridge", () => {throw new Error("provider failed");},},+ name: "failing",},{events: {"model.call.completed": after,},+ name: "observer",},]);const bridge = createAiSdkHookBridge(scope, hooks);@@ -211,7 +220,9 @@ describe("createAiSdkHookBridge", () => {it("terminalizes started operations when the attempt errors", async () => {const after = vi.fn();- const hooks = createInstrumentationHooks([{ events: { "model.call.failed": after } }]);+ const hooks = createInstrumentationHooks([+ { events: { "model.call.failed": after }, name: "terminal" },+ ]);const bridge = createAiSdkHookBridge(scope, hooks);await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [@@ -252,8 +263,8 @@ describe("createAiSdkHookBridge", () => {});const started = vi.fn();const hooks = createInstrumentationHooks([- { events: { "step.attempt.started": mutator } },- { events: { "step.attempt.started": started } },+ { events: { "step.attempt.started": mutator }, name: "mutator" },+ { events: { "step.attempt.started": started }, name: "observer" },]);const bridge = createAiSdkHookBridge(scope, hooks);@@ -288,7 +299,10 @@ describe("createAiSdkHookBridge", () => {expect(Object.isFrozen(event.usage.inputTokenDetails)).toBe(true);});const hooks = createInstrumentationHooks([- { events: { "model.call.completed": after, "model.call.started": before } },+ {+ events: { "model.call.completed": after, "model.call.started": before },+ name: "model",+ },]);const bridge = createAiSdkHookBridge(scope, hooks);@@ -404,7 +418,10 @@ describe("createAiSdkHookBridge", () => {expect(Object.isFrozen(event.output)).toBe(true);});const hooks = createInstrumentationHooks([- { events: { "tool.call.completed": after, "tool.call.started": before } },+ {+ events: { "tool.call.completed": after, "tool.call.started": before },+ name: "tool",+ },]);const bridge = createAiSdkHookBridge(scope, hooks);const toolCall = { input: { q: "eve" }, toolCallId: "tool-1", toolName: "search" };@@ -478,7 +495,9 @@ describe("createAiSdkHookBridge", () => {it("skips a terminal handler when the operation never started", async () => {const completed = vi.fn();- const hooks = createInstrumentationHooks([{ events: { "model.call.completed": completed } }]);+ const hooks = createInstrumentationHooks([+ { events: { "model.call.completed": completed }, name: "terminal" },+ ]);const bridge = createAiSdkHookBridge(scope, hooks);// No onLanguageModelCallStart, so the bridge holds no id and publishes@@ -516,6 +535,7 @@ describe("createAiSdkHookBridge", () => {terminalStates.set(event.idempotencyKey, started.get(event.idempotencyKey));},},+ name: "parallel",},]);const bridge = createAiSdkHookBridge(scope, hooks);diff --git a/packages/eve/src/harness/ai-sdk-telemetry.test.ts b/packages/eve/src/harness/ai-sdk-telemetry.test.tsnew file mode 100644index 000000000..6049c90ca--- /dev/null+++ b/packages/eve/src/harness/ai-sdk-telemetry.test.ts@@ -0,0 +1,25 @@+import type { Telemetry } from "ai";+import { afterEach, describe, expect, it } from "vitest";++import { getRegisteredTelemetryIntegrations } from "#harness/ai-sdk-telemetry.js";++describe("getRegisteredTelemetryIntegrations", () => {+ const original = globalThis.AI_SDK_TELEMETRY_INTEGRATIONS;++ afterEach(() => {+ globalThis.AI_SDK_TELEMETRY_INTEGRATIONS = original;+ });++ it("is empty when nothing has registered", () => {+ globalThis.AI_SDK_TELEMETRY_INTEGRATIONS = undefined;+ expect(getRegisteredTelemetryIntegrations()).toEqual([]);+ });++ it("reports the integrations in registration order", () => {+ const first: Telemetry = { onStart() {} };+ const second: Telemetry = { onStart() {} };+ globalThis.AI_SDK_TELEMETRY_INTEGRATIONS = [first, second];++ expect(getRegisteredTelemetryIntegrations()).toEqual([first, second]);+ });+});diff --git a/packages/eve/src/harness/otel-integration.ts b/packages/eve/src/harness/ai-sdk-telemetry.tssimilarity index 50%rename from packages/eve/src/harness/otel-integration.tsrename to packages/eve/src/harness/ai-sdk-telemetry.tsindex d34108a79..02954907f 100644--- a/packages/eve/src/harness/otel-integration.ts+++ b/packages/eve/src/harness/ai-sdk-telemetry.ts@@ -16,10 +16,17 @@ export function ensureOtelIntegration(): void {return;}registered = true;- registerTelemetry(createOtelIntegration());+ registerTelemetry(new OpenTelemetry({ runtimeContext: true }));}-/** Creates the existing OTel integration for explicit per-call composition. */-export function createOtelIntegration(): Telemetry {- return new OpenTelemetry({ runtimeContext: true });+/**+ * Every integration currently registered with the AI SDK — eve's own, plus any+ * an authored instrumentation module added with `registerTelemetry`.+ *+ * A per-call `integrations` list replaces the registered ones rather than+ * adding to them, so anything that passes integrations per call has to carry+ * these forward or they stop receiving events.+ */+export function getRegisteredTelemetryIntegrations(): readonly Telemetry[] {+ return globalThis.AI_SDK_TELEMETRY_INTEGRATIONS ?? [];}diff --git a/packages/eve/src/harness/instrumentation-config.test.ts b/packages/eve/src/harness/instrumentation-config.test.tsindex ef907d37a..fc027202c 100644--- a/packages/eve/src/harness/instrumentation-config.test.ts+++ b/packages/eve/src/harness/instrumentation-config.test.ts@@ -1,5 +1,7 @@import { beforeEach, describe, expect, it, vi } from "vitest";+import type { InstrumentationSetupContext } from "#public/instrumentation/index.js";+const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime");beforeEach(() => {@@ -31,7 +33,7 @@ describe("instrumentation-config chunk-isolation regression", () => {vi.resetModules();const moduleA = await import("#harness/instrumentation-config.js");const config = { functionId: "test.instrumentation.cross-module.alice" };- moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" });+ await moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" });vi.resetModules();const moduleB = await import("#harness/instrumentation-config.js");@@ -46,7 +48,7 @@ describe("instrumentation-config chunk-isolation regression", () => {const { registerInstrumentationConfig } = await import("#harness/instrumentation-config.js");const canary = { functionId: "test.instrumentation.global-mount.canary" };- registerInstrumentationConfig(canary, { agentName: "test-agent" });+ await registerInstrumentationConfig(canary, { agentName: "test-agent" });expect((globalThis as Record<symbol, unknown>)[globalKey]).toBe(canary);});@@ -57,7 +59,7 @@ describe("instrumentation-config chunk-isolation regression", () => {vi.resetModules();const moduleA = await import("#harness/instrumentation-config.js");const config = { functionId: "test.instrumentation.reimport.canary" };- moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" });+ await moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" });const firstRef = (globalThis as Record<symbol, unknown>)[globalKey];vi.resetModules();@@ -89,13 +91,23 @@ describe("instrumentation-config chunk-isolation regression", () => {});});- it("invokes the setup callback with the supplied context", async () => {+ it("awaits the setup callback with the resolved context", async () => {vi.resetModules();const { registerInstrumentationConfig } = await import("#harness/instrumentation-config.js");- const setup = vi.fn();- registerInstrumentationConfig({ setup }, { agentName: "weather-agent" });+ const contexts: InstrumentationSetupContext[] = [];+ await registerInstrumentationConfig(+ {+ setup: (context) => {+ contexts.push(context);+ },+ },+ { agentName: "weather-agent" },+ );- expect(setup).toHaveBeenCalledExactlyOnceWith({ agentName: "weather-agent" });+ expect(contexts).toHaveLength(1);+ expect(contexts[0]?.agentName).toBe("weather-agent");+ expect(contexts[0]?.environment).toMatch(/^(development|production)$/);+ expect(contexts[0]?.frameworkVersion).toEqual(expect.any(String));});});diff --git a/packages/eve/src/harness/instrumentation-config.ts b/packages/eve/src/harness/instrumentation-config.tsindex fbe66be55..541ad1f15 100644--- a/packages/eve/src/harness/instrumentation-config.ts+++ b/packages/eve/src/harness/instrumentation-config.ts@@ -1,9 +1,7 @@-import type {- InstrumentationDefinition,- InstrumentationSetupContext,-} from "#public/instrumentation/index.js";import { createInstrumentationHooks } from "#harness/instrumentation-lifecycle.js";import { registerInstrumentationRuntime } from "#harness/instrumentation-runtime.js";+import { createInstrumentationSetupContext } from "#harness/instrumentation-setup-context.js";+import type { InstrumentationDefinition } from "#public/instrumentation/index.js";/*** Process-global store for the authored instrumentation config.@@ -30,19 +28,24 @@ interface InstrumentationConfigGlobal {const globalContainer = globalThis as typeof globalThis & InstrumentationConfigGlobal;/**- * Registers the authored instrumentation config and invokes its `setup`- * callback with the resolved agent name.+ * Registers the authored instrumentation config and awaits its `setup`+ * callback.** Called once by the generated instrumentation Nitro plugin at server* startup. Subsequent calls overwrite the previous value.*+ * The store write lands before `setup` runs so a synchronous caller sees the+ * config without waiting on the returned promise.+ ** @internal — not part of the public API.*/export async function registerInstrumentationConfig(config: InstrumentationDefinition,- context: InstrumentationSetupContext,+ input: { readonly agentName: string },): Promise<void> {globalContainer[INSTRUMENTATION_CONFIG_GLOBAL_KEY] = config;+ // This legacy layout leaves `registerOTel` to `setup`, so install only the+ // runtime projection consumed by the harness.registerInstrumentationRuntime({forceFlush: async () => undefined,hooks: createInstrumentationHooks([]),@@ -55,7 +58,7 @@ export async function registerInstrumentationConfig(runInContext: (_operation, execute) => execute(),shutdown: async () => undefined,});- await config.setup?.(context);+ await config.setup?.(createInstrumentationSetupContext(input.agentName));}/**diff --git a/packages/eve/src/harness/instrumentation-dispatch.ts b/packages/eve/src/harness/instrumentation-dispatch.tsindex 5125fd738..fffbf3c2b 100644--- a/packages/eve/src/harness/instrumentation-dispatch.ts+++ b/packages/eve/src/harness/instrumentation-dispatch.ts@@ -155,17 +155,20 @@ async function dispatchToProvider(const startedBoundary = event.type.endsWith(".started") || event.type === "input.requested";const owner = stateOwner(event);const providerName = provider.name;- if (isInstrumentationStateAbandoned(providerName, event.idempotencyKey)) return;+ const stateNamespace = provider.stateNamespace ?? providerName;+ if (isInstrumentationStateAbandoned(stateNamespace, event.idempotencyKey)) return;const handler = provider.events?.[event.type];if (handler === undefined) return;- const state = instrumentationStateSlot(providerName, event.idempotencyKey, owner);+ const state = instrumentationStateSlot(stateNamespace, event.idempotencyKey, owner);try {const settled = await withTimeout(() => (handler as InstrumentationEventHandler<InstrumentationEvent>)(event, { state }),handlerTimeoutMs,() => {state.revoke();- if (startedBoundary) abandonInstrumentationState(providerName, event.idempotencyKey, owner);+ if (startedBoundary) {+ abandonInstrumentationState(stateNamespace, event.idempotencyKey, owner);+ }},);if (!settled) {diff --git a/packages/eve/src/harness/instrumentation-lifecycle.ts b/packages/eve/src/harness/instrumentation-lifecycle.tsindex c8a7bd022..f9cbce54a 100644--- a/packages/eve/src/harness/instrumentation-lifecycle.ts+++ b/packages/eve/src/harness/instrumentation-lifecycle.ts@@ -445,6 +445,8 @@ export type InstrumentationEventHandler<TEvent> = (/** Internal provider shape mirrored by the future public hook contract. */export interface InstrumentationProviderDefinition {readonly name: string;+ /** Durable state identity, separate from the human-readable log name. */+ readonly stateNamespace?: string;readonly events?: {readonly "step.attempt.started"?: InstrumentationEventHandler<InstrumentationStepAttemptStartedEvent>;readonly "step.attempt.completed"?: InstrumentationEventHandler<InstrumentationStepAttemptCompletedEvent>;diff --git a/packages/eve/src/harness/instrumentation-native-events.test.ts b/packages/eve/src/harness/instrumentation-native-events.test.tsindex afa99f2fd..16a5e19fc 100644--- a/packages/eve/src/harness/instrumentation-native-events.test.ts+++ b/packages/eve/src/harness/instrumentation-native-events.test.ts@@ -160,6 +160,7 @@ describe("createInstrumentationHandleEvent", () => {},]);});+it("publishes every runtime action and settles it in a replacement worker", async () => {const events: unknown[] = [];const scope = {diff --git a/packages/eve/src/harness/instrumentation-native-events.ts b/packages/eve/src/harness/instrumentation-native-events.tsindex bfe369b6e..477d2469e 100644--- a/packages/eve/src/harness/instrumentation-native-events.ts+++ b/packages/eve/src/harness/instrumentation-native-events.ts@@ -50,9 +50,9 @@ export function createInstrumentationHandleEvent(const handleEvent = input.handleEvent;const hooks = input.hooks;- let activeTurnId = input.turnId;const publishedActions = new Set<string>();const publishedInputs = new Set<string>();+ let activeTurnId = input.turnId;return async (event, messages) => {await handleEvent(event, messages);const lifecycleEvent = toLifecycleEvent(event, input, activeTurnId);diff --git a/packages/eve/src/harness/instrumentation-providers-local.scenario.test.ts b/packages/eve/src/harness/instrumentation-providers-local.scenario.test.tsnew file mode 100644index 000000000..ca81922e4--- /dev/null+++ b/packages/eve/src/harness/instrumentation-providers-local.scenario.test.ts@@ -0,0 +1,45 @@+import { mkdtemp, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";++import { afterEach, describe, expect, it, vi } from "vitest";++import {+ finalizeInstrumentationProviders,+ getInstrumentationProviders,+ registerInstrumentationProvider,+ seedInstrumentationProviders,+} from "#harness/instrumentation-providers.js";+import { DEVELOPMENT_WORKER_APP_ROOT_ENV } from "#internal/workflow/development-world-protocol.js";+import { otelIntegration } from "#public/instrumentation/otel.js";++const temporaryDirectories: string[] = [];++afterEach(async () => {+ vi.unstubAllEnvs();+ await Promise.all(+ temporaryDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true })),+ );+});++describe("instrumentation provider local default", () => {+ it("registers default local traces and an authored destination in one pipeline", async () => {+ const appRoot = await mkdtemp(join(tmpdir(), "eve-provider-local-"));+ temporaryDirectories.push(appRoot);+ vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, appRoot);+ vi.stubEnv("EVE_TRACES", "off");++ seedInstrumentationProviders();+ await registerInstrumentationProvider({+ agentName: "weather",+ slot: "backend",+ value: otelIntegration(),+ });++ const runtime = finalizeInstrumentationProviders({ serviceName: "weather" });+ await runtime.forceFlush();+ await runtime.shutdown();++ expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["backend", "local"]);+ });+});diff --git a/packages/eve/src/harness/instrumentation-providers-production.scenario.test.ts b/packages/eve/src/harness/instrumentation-providers-production.scenario.test.tsnew file mode 100644index 000000000..1afb44245--- /dev/null+++ b/packages/eve/src/harness/instrumentation-providers-production.scenario.test.ts@@ -0,0 +1,33 @@+import { afterEach, describe, expect, it, vi } from "vitest";++import {+ finalizeInstrumentationProviders,+ getInstrumentationProviders,+ registerInstrumentationProvider,+ seedInstrumentationProviders,+} from "#harness/instrumentation-providers.js";+import { localTraces } from "#public/instrumentation/otel.js";++afterEach(() => {+ vi.unstubAllEnvs();+});++describe("instrumentation provider production defaults", () => {+ it("keeps authored local traces inert beside Agent Runs", async () => {+ vi.stubEnv("EVE_DEV_WORKER_APP_ROOT", undefined);+ vi.stubEnv("VERCEL_ENV", "production");++ seedInstrumentationProviders();+ await registerInstrumentationProvider({+ agentName: "weather",+ slot: "local",+ value: localTraces(),+ });++ const runtime = finalizeInstrumentationProviders({ serviceName: "weather" });+ await runtime.forceFlush();+ await runtime.shutdown();++ expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["agent-runs", "local"]);+ });+});diff --git a/packages/eve/src/harness/instrumentation-providers.integration.test.ts b/packages/eve/src/harness/instrumentation-providers.integration.test.tsnew file mode 100644index 000000000..dfb6cd089--- /dev/null+++ b/packages/eve/src/harness/instrumentation-providers.integration.test.ts@@ -0,0 +1,92 @@+import { beforeEach, describe, expect, it } from "vitest";++import { turnIdempotencyKey } from "#harness/instrumentation-lifecycle.js";+import {+ finalizeInstrumentationProviders,+ registerInstrumentationProvider,+} from "#harness/instrumentation-providers.js";+import { defineInstrumentation } from "#public/instrumentation/index.js";++const REGISTRY_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers");+const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime");++function deferred(): { readonly promise: Promise<void>; readonly resolve: () => void } {+ let resolve!: () => void;+ const promise = new Promise<void>((complete) => {+ resolve = complete;+ });+ return { promise, resolve };+}++describe("authored instrumentation provider dispatch", () => {+ beforeEach(() => {+ delete (globalThis as Record<symbol, unknown>)[REGISTRY_GLOBAL_KEY];+ delete (globalThis as Record<symbol, unknown>)[RUNTIME_GLOBAL_KEY];+ });++ it("awaits setup in slot order while event handlers overlap", async () => {+ const firstSetup = deferred();+ const firstHandler = deferred();+ const secondHandler = deferred();+ const order: string[] = [];++ const firstRegistration = registerInstrumentationProvider({+ agentName: "weather",+ slot: "first",+ value: defineInstrumentation({+ events: {+ "turn.started": async () => {+ order.push("first:handler");+ await firstHandler.promise;+ },+ },+ setup: async () => {+ order.push("first:setup");+ await firstSetup.promise;+ order.push("first:setup-complete");+ },+ }),+ });++ expect(order).toEqual(["first:setup"]);+ firstSetup.resolve();+ await firstRegistration;++ await registerInstrumentationProvider({+ agentName: "weather",+ slot: "second",+ value: defineInstrumentation({+ events: {+ "turn.started": async () => {+ order.push("second:handler");+ await secondHandler.promise;+ },+ },+ setup: () => void order.push("second:setup"),+ }),+ });++ expect(order).toEqual(["first:setup", "first:setup-complete", "second:setup"]);++ const runtime = finalizeInstrumentationProviders({ serviceName: "weather" });+ const publication = runtime.hooks.publish({+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),+ rootSessionId: "session-1",+ sequence: 0,+ sessionId: "session-1",+ turnId: "turn-1",+ type: "turn.started",+ });++ expect(order).toEqual([+ "first:setup",+ "first:setup-complete",+ "second:setup",+ "first:handler",+ "second:handler",+ ]);+ firstHandler.resolve();+ secondHandler.resolve();+ await publication;+ });+});diff --git a/packages/eve/src/harness/instrumentation-providers.test.ts b/packages/eve/src/harness/instrumentation-providers.test.tsnew file mode 100644index 000000000..1b2842869--- /dev/null+++ b/packages/eve/src/harness/instrumentation-providers.test.ts@@ -0,0 +1,255 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++import { turnIdempotencyKey } from "#harness/instrumentation-lifecycle.js";+import {+ EVE_EVALUATION_ENV_FLAG,+ EVE_EVALUATION_RUN_ID_ENV,+} from "#internal/application/dev-environment.js";+import {+ finalizeInstrumentationProviders,+ getInstrumentationProviders,+ registerInstrumentationProvider,+ seedInstrumentationProviders,+ shutdownInstrumentationProviders,+} from "#harness/instrumentation-providers.js";+import { DEVELOPMENT_WORKER_APP_ROOT_ENV } from "#internal/workflow/development-world-protocol.js";+import { defineInstrumentation } from "#public/instrumentation/index.js";+import { agentRuns, localTraces, otelIntegration } from "#public/instrumentation/otel.js";+import {+ disableInstrumentation,+ type ProviderSetupContext,+} from "#public/instrumentation/provider.js";++const REGISTRY_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers");+const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime");++function register(slot: string, value: unknown): Promise<void> {+ return registerInstrumentationProvider({ agentName: "weather-agent", slot, value });+}++describe("registerInstrumentationProvider", () => {+ beforeEach(() => {+ vi.unstubAllEnvs();+ delete (globalThis as Record<symbol, unknown>)[REGISTRY_GLOBAL_KEY];+ delete (globalThis as Record<symbol, unknown>)[RUNTIME_GLOBAL_KEY];+ });++ it("registers a provider under its slot", async () => {+ const provider = defineInstrumentation({ events: {} });+ await register("otel", provider);++ expect(getInstrumentationProviders()).toEqual([{ provider, slot: "otel" }]);+ });++ it("preserves registration order across slots", async () => {+ await register("agent-runs", defineInstrumentation({}));+ await register("local", defineInstrumentation({}));++ expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["agent-runs", "local"]);+ });++ it("awaits setup with the resolved provider context", async () => {+ const contexts: ProviderSetupContext[] = [];+ await register(+ "otel",+ defineInstrumentation({+ setup: (context) => {+ contexts.push(context);+ },+ shutdown: () => {},+ }),+ );++ expect(contexts).toHaveLength(1);+ expect(contexts[0]?.agentName).toBe("weather-agent");+ expect(contexts[0]?.environment).toMatch(/^(development|preview|production)$/);+ expect(contexts[0]?.evaluation).toBeUndefined();+ expect(contexts[0]?.frameworkVersion).toEqual(expect.any(String));+ });++ it("reports Vercel preview separately from production", async () => {+ vi.stubEnv("VERCEL_ENV", "preview");+ const contexts: ProviderSetupContext[] = [];++ await register(+ "otel",+ defineInstrumentation({ setup: (context) => void contexts.push(context) }),+ );++ expect(contexts[0]?.environment).toBe("preview");+ });++ it("reports an evaluation server to setup", async () => {+ vi.stubEnv(EVE_EVALUATION_ENV_FLAG, "1");+ vi.stubEnv(EVE_EVALUATION_RUN_ID_ENV, "eval-run-1");++ const contexts: ProviderSetupContext[] = [];+ await register(+ "otel",+ defineInstrumentation({+ setup: (context) => {+ contexts.push(context);+ },+ }),+ );++ expect(contexts[0]?.evaluation).toEqual({ runId: "eval-run-1" });+ });++ it("registers nothing for a disabled slot", async () => {+ await register("local", disableInstrumentation());++ expect(getInstrumentationProviders()).toEqual([]);+ });++ it("removes an already-registered provider when a later file disables the slot", async () => {+ await register("local", defineInstrumentation({}));+ await register("local", disableInstrumentation());++ expect(getInstrumentationProviders()).toEqual([]);+ });++ // A slot that registers nothing is telemetry that silently does nothing —+ // the failure this surface exists to prevent — so the shape check throws+ // rather than skipping the file.+ it.each([+ ["a bare object", { events: {} }],+ ["a function", () => {}],+ ["undefined", undefined],+ ["null", null],+ ])("throws when the default export is %s", async (_label, value) => {+ await expect(register("otel", value)).rejects.toThrow(+ /The default export of "instrumentation\/otel" is not an instrumentation provider/,+ );+ });+});++describe("seedInstrumentationProviders", () => {+ beforeEach(() => {+ vi.unstubAllEnvs();+ delete (globalThis as Record<symbol, unknown>)[REGISTRY_GLOBAL_KEY];+ delete (globalThis as Record<symbol, unknown>)[RUNTIME_GLOBAL_KEY];+ vi.stubEnv("EVE_TRACES", "off");+ vi.stubEnv("VERCEL_ENV", undefined);+ vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, "/tmp/eve-seed-test");+ });++ it("sorts default local traces with authored destinations", async () => {+ seedInstrumentationProviders();+ await register("backend", otelIntegration());++ expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["backend", "local"]);+ });++ it("seeds Agent Runs only in Vercel production", () => {+ vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, undefined);+ vi.stubEnv("VERCEL_ENV", "production");++ seedInstrumentationProviders();++ expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["agent-runs"]);+ });++ it("lets an authored reserved slot reconfigure or disable its default", async () => {+ seedInstrumentationProviders();+ const authored = localTraces({ recordInputs: false });+ await register("local", authored);+ expect(getInstrumentationProviders()).toEqual([{ provider: authored, slot: "local" }]);++ await register("local", disableInstrumentation());+ expect(getInstrumentationProviders()).toEqual([]);+ });++ it("lets an authored Agent Runs slot narrow the production default", async () => {+ vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, undefined);+ vi.stubEnv("VERCEL_ENV", "production");+ seedInstrumentationProviders();+ const authored = agentRuns({ recordOutputs: false });++ await register("agent-runs", authored);++ expect(getInstrumentationProviders()).toEqual([{ provider: authored, slot: "agent-runs" }]);+ });++ it("sorts built-ins, authored slots, and reserved-slot replacements together", async () => {+ vi.stubEnv("VERCEL_ENV", "production");+ seedInstrumentationProviders();+ await register("zeta", defineInstrumentation({}));+ await register("audit", defineInstrumentation({}));+ await register("local", localTraces({ recordInputs: false }));++ expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual([+ "agent-runs",+ "audit",+ "local",+ "zeta",+ ]);+ });+});++describe("finalizeInstrumentationProviders", () => {+ const turnStarted = {+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),+ rootSessionId: "session-1",+ sequence: 0,+ sessionId: "session-1",+ turnId: "turn-1",+ type: "turn.started",+ } as const;++ beforeEach(() => {+ vi.unstubAllEnvs();+ delete (globalThis as Record<symbol, unknown>)[REGISTRY_GLOBAL_KEY];+ delete (globalThis as Record<symbol, unknown>)[RUNTIME_GLOBAL_KEY];+ });++ it("publishes to an authored handler", async () => {+ const started = vi.fn();+ await register("rows", defineInstrumentation({ events: { "turn.started": started } }));++ const runtime = finalizeInstrumentationProviders({ serviceName: "weather-agent" });+ await runtime.hooks.publish(turnStarted);++ expect(started).toHaveBeenCalledOnce();+ expect(started.mock.calls[0]?.[0]).toMatchObject({ turnId: "turn-1" });+ });++ it("still runs execution when no destination was declared", async () => {+ // A directory with no `otel()` has nothing to hang a span on, so+ // `runInContext` degrades to running the work directly rather than+ // going missing.+ await register("rows", defineInstrumentation({}));++ const runtime = finalizeInstrumentationProviders({ serviceName: "weather-agent" });+ const result = await runtime.runInContext(+ {+ idempotencyKey: "tool:session-1:turn-1:0:0:call-1:0",+ scope: {+ attemptId: "session-1:turn-1:0:0",+ attemptIndex: 0,+ sessionId: "session-1",+ stepIndex: 0,+ turnId: "turn-1",+ },+ type: "tool.call",+ },+ () => Promise.resolve("ran"),+ );++ expect(result).toBe("ran");+ });++ it("drains and releases every provider", async () => {+ const flush = vi.fn();+ const shutdown = vi.fn();+ await register("rows", defineInstrumentation({ flush, shutdown }));++ const runtime = finalizeInstrumentationProviders({ serviceName: "weather-agent" });+ await runtime.forceFlush();+ await shutdownInstrumentationProviders();+ await shutdownInstrumentationProviders();++ expect(flush).toHaveBeenCalledOnce();+ expect(shutdown).toHaveBeenCalledOnce();+ });+});diff --git a/packages/eve/src/harness/instrumentation-providers.ts b/packages/eve/src/harness/instrumentation-providers.tsnew file mode 100644index 000000000..13fb2a3eb--- /dev/null+++ b/packages/eve/src/harness/instrumentation-providers.ts@@ -0,0 +1,153 @@+import type { InstrumentationProviderDefinition } from "#harness/instrumentation-lifecycle.js";+import {+ getInstrumentationRuntime,+ type InstrumentationRuntime,+} from "#harness/instrumentation-runtime.js";+import { createInstrumentationSetupContext } from "#harness/instrumentation-setup-context.js";+import { resolveInstalledPackageInfo } from "#internal/application/package.js";+import { DEVELOPMENT_WORKER_APP_ROOT_ENV } from "#internal/workflow/development-world-protocol.js";+import { agentRuns, localTraces } from "#public/instrumentation/otel.js";+import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js";+import { collectOtelPipeline } from "#tracing/otel-declaration.js";+import {+ isInstrumentationDisabled,+ isInstrumentationProvider,+ type InstrumentationProvider,+} from "#public/instrumentation/provider.js";++/**+ * Process-global registry of the providers authored under+ * `agent/instrumentation/`.+ *+ * Rooted on `globalThis` for the same reason the single-config store is: the+ * generated Nitro plugin stays external by `file://` URL while the harness+ * chunk is inlined, so the two resolve to distinct ESM module instances and+ * need one shared source of truth.+ */+const INSTRUMENTATION_PROVIDERS_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers");++/** One provider and the `instrumentation/<slot>.ts` file it came from. */+export interface RegisteredInstrumentationProvider {+ readonly provider: InstrumentationProvider;+ readonly slot: string;+}++interface InstrumentationProvidersGlobal {+ [INSTRUMENTATION_PROVIDERS_GLOBAL_KEY]?: Map<string, InstrumentationProvider>;+}++const globalContainer = globalThis as typeof globalThis & InstrumentationProvidersGlobal;++function providerRegistry(): Map<string, InstrumentationProvider> {+ const existing = globalContainer[INSTRUMENTATION_PROVIDERS_GLOBAL_KEY];+ if (existing !== undefined) {+ return existing;+ }++ const created = new Map<string, InstrumentationProvider>();+ globalContainer[INSTRUMENTATION_PROVIDERS_GLOBAL_KEY] = created;+ return created;+}++/** Fills reserved slots before authored files may reconfigure or disable them. */+export function seedInstrumentationProviders(): void {+ const registry = providerRegistry();+ if (process.env.VERCEL_ENV === "production") registry.set("agent-runs", agentRuns());+ if (process.env[DEVELOPMENT_WORKER_APP_ROOT_ENV] !== undefined) {+ registry.set("local", localTraces());+ }+}++/**+ * Registers one authored provider and awaits its `setup`.+ *+ * Called once per `instrumentation/<slot>.ts` by the generated Nitro plugin at+ * server startup, before any event is published. A default export that is not+ * a `defineInstrumentation` result throws rather than being skipped: a slot+ * that registers nothing is telemetry that silently does nothing, which is the+ * failure this surface exists to prevent.+ *+ * @internal — not part of the public API.+ */+export async function registerInstrumentationProvider(input: {+ readonly agentName: string;+ readonly slot: string;+ readonly value: unknown;+}): Promise<void> {+ if (isInstrumentationDisabled(input.value)) {+ providerRegistry().delete(input.slot);+ return;+ }++ if (!isInstrumentationProvider(input.value)) {+ throw new Error(+ `The default export of "instrumentation/${input.slot}" is not an instrumentation provider. Return the result of \`defineInstrumentation\` or \`disableInstrumentation\` from it.`,+ );+ }++ providerRegistry().set(input.slot, input.value);+ await input.value.setup?.(createInstrumentationSetupContext(input.agentName));+}++/** Registered providers in slot order. @internal */+export function getInstrumentationProviders(): readonly RegisteredInstrumentationProvider[] {+ return [...providerRegistry()]+ .sort(([left], [right]) => left.localeCompare(right))+ .map(([slot, provider]) => ({ provider, slot }));+}++/**+ * Installs the process instrumentation runtime from the registered providers.+ *+ * Called once by the generated Nitro plugin after every slot has registered,+ * which is also why it cannot happen inside `setup`: the OpenTelemetry pipeline+ * is the union of every destination declared in the directory, so no single+ * file knows enough to build it. A `setup` that reaches for a tracer therefore+ * gets the no-op one; declare destinations as values and let this assemble+ * them.+ *+ * A directory that declared no OpenTelemetry at all still gets a bus. Its+ * providers see every event; they just have no spans to hang them on.+ *+ * @internal — not part of the public API.+ */+export function finalizeInstrumentationProviders(input: {+ readonly serviceName: string;+}): InstrumentationRuntime {+ const registered = getInstrumentationProviders();+ return installInstrumentationRuntime({+ collected: collectOtelPipeline(registered.map((entry) => entry.provider)),+ frameworkVersion: resolveInstalledPackageInfo().version,+ providers: registered.map(toProviderDefinition),+ serviceName: input.serviceName,+ });+}++/**+ * Releases every registered provider and OTel processor from Nitro's close+ * hook, the last point a buffered exporter can still reach the network.+ */+export async function shutdownInstrumentationProviders(): Promise<void> {+ await getInstrumentationRuntime()?.shutdown();+}++/**+ * Adapts an authored provider onto the internal bus contract.+ *+ * The event maps are the same shape — the public one is derived from the+ * internal union and both handlers take `(event, ctx)` — so only the name has+ * to be supplied.+ */+function toProviderDefinition(+ entry: RegisteredInstrumentationProvider,+): InstrumentationProviderDefinition {+ return {+ events: entry.provider.events as InstrumentationProviderDefinition["events"],+ flush: entry.provider.flush,+ // The file the provider came from, which is the only name an author can+ // recognize in a log line about it.+ name: entry.slot,+ shutdown: entry.provider.shutdown,+ stateNamespace: `authored:${entry.slot}`,+ };+}diff --git a/packages/eve/src/harness/instrumentation-setup-context.ts b/packages/eve/src/harness/instrumentation-setup-context.tsnew file mode 100644index 000000000..b23dfe575--- /dev/null+++ b/packages/eve/src/harness/instrumentation-setup-context.ts@@ -0,0 +1,31 @@+import {+ isEveDevEnvironment,+ resolveEveEvaluationRunId,+} from "#internal/application/dev-environment.js";+import { resolveInstalledPackageInfo } from "#internal/application/package.js";+import type { ProviderSetupContext } from "#public/instrumentation/provider.js";++/**+ * Builds the context handed to an authored `setup` at server startup.+ *+ * Shared by both layouts so the two cannot drift: a divergent context type+ * would leave `defineInstrumentation`'s union without a contextual signature+ * for `setup`, silently making every authored `setup(context)` parameter an+ * implicit `any`.+ *+ * @internal — not part of the public API.+ */+export function createInstrumentationSetupContext(agentName: string): ProviderSetupContext {+ const evaluationRunId = resolveEveEvaluationRunId();+ return {+ agentName,+ environment: resolveInstrumentationEnvironment(),+ evaluation: evaluationRunId === undefined ? undefined : { runId: evaluationRunId },+ frameworkVersion: resolveInstalledPackageInfo().version,+ };+}++function resolveInstrumentationEnvironment(): ProviderSetupContext["environment"] {+ if (isEveDevEnvironment() || process.env.VERCEL_ENV === "development") return "development";+ return process.env.VERCEL_ENV === "preview" ? "preview" : "production";+}diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex d3d84af5a..81fb1f608 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -77,18 +77,25 @@ vi.mock("ai", () => ({tool: vi.fn((t: unknown) => t),}));-const { existingOtelIntegration, mockCreateAiSdkHookBridge } = vi.hoisted(() => ({- existingOtelIntegration: { onStart: vi.fn() },+const {+ mockCreateAiSdkHookBridge,+ mockGetRegisteredTelemetryIntegrations,+ registeredAuthorIntegration,+ registeredOtelIntegration,+} = vi.hoisted(() => ({mockCreateAiSdkHookBridge: vi.fn((..._args: unknown[]) => ({ onStart: vi.fn() })),+ mockGetRegisteredTelemetryIntegrations: vi.fn((): unknown[] => []),+ registeredAuthorIntegration: { onStart: vi.fn() },+ registeredOtelIntegration: { onStart: vi.fn() },}));vi.mock("./ai-sdk-hook-bridge.js", () => ({createAiSdkHookBridge: (...args: unknown[]) => mockCreateAiSdkHookBridge(...args),}));-vi.mock("./otel-integration.js", () => ({- createOtelIntegration: vi.fn(() => existingOtelIntegration),+vi.mock("./ai-sdk-telemetry.js", () => ({ensureOtelIntegration: vi.fn(),+ getRegisteredTelemetryIntegrations: () => mockGetRegisteredTelemetryIntegrations(),}));const mockGetInstrumentationConfig = vi.fn().mockReturnValue(undefined);@@ -101,6 +108,10 @@ vi.mock("./instrumentation-runtime.js", () => ({getInstrumentationRuntime: (...args: unknown[]) => mockGetInstrumentationRuntime(...args),}));+/**+ * Registering an authored config writes both stores, so the tests toggle+ * telemetry through one call rather than keeping two mocks in step by hand.+ */function declareTelemetry(config: Readonly<Record<string, unknown>> | undefined): void {mockGetInstrumentationConfig.mockReturnValue(config);mockGetInstrumentationRuntime.mockReturnValue(@@ -135,6 +146,7 @@ afterEach(() => {vi.clearAllMocks();vi.unstubAllEnvs();declareTelemetry(undefined);+ mockGetRegisteredTelemetryIntegrations.mockReturnValue([]);});function createTestSession(overrides?: Partial<HarnessSession>): HarnessSession {@@ -10204,7 +10216,7 @@ describe("createToolLoopHarness", () => {});const attemptCompleted = vi.fn();const hooks = createInstrumentationHooks([- { events: { "step.attempt.completed": attemptCompleted } },+ { events: { "step.attempt.completed": attemptCompleted }, name: "attempt" },]);const runInContext: InstrumentationContextRunner = (_operation, execute) => execute();const config = createTestConfig("conversation", undefined, {@@ -10278,7 +10290,7 @@ describe("createToolLoopHarness", () => {});const started = vi.fn();const hooks = createInstrumentationHooks([- { events: { "action.started": started }, name: "test" },+ { events: { "action.started": started }, name: "action" },]);const { emit } = createEventCollector();const runStep = createToolLoopHarness(@@ -10301,7 +10313,7 @@ describe("createToolLoopHarness", () => {);});- it("composes lifecycle hooks with existing authored OTel", async () => {+ it("composes the bridge with every registered integration", async () => {setupMockAgent({finishReason: "stop",response: { messages: [{ content: "Hello!", role: "assistant" }] },@@ -10310,6 +10322,11 @@ describe("createToolLoopHarness", () => {toolResults: [],});declareTelemetry({ recordInputs: true, recordOutputs: false });+ // An authored module can register its own integration alongside eve's.+ mockGetRegisteredTelemetryIntegrations.mockReturnValue([+ registeredOtelIntegration,+ registeredAuthorIntegration,+ ]);const hooks = createInstrumentationHooks([]);const runStep = createToolLoopHarness(createTestConfig("conversation", undefined, {@@ -10331,7 +10348,7 @@ describe("createToolLoopHarness", () => {};};expect(agentCall.telemetry).toMatchObject({- integrations: [bridge, existingOtelIntegration],+ integrations: [bridge, registeredOtelIntegration, registeredAuthorIntegration],recordInputs: true,recordOutputs: false,});diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.tsindex 0409fedc2..5e99ea925 100644--- a/packages/eve/src/harness/tool-loop.ts+++ b/packages/eve/src/harness/tool-loop.ts@@ -191,7 +191,10 @@ import {hasEmptyDeliverySentinel,} from "#shared/empty-delivery.js";import { extractWorkflowStreamWriteErrorDetails } from "#harness/workflow-stream-error.js";-import { createOtelIntegration, ensureOtelIntegration } from "#harness/otel-integration.js";+import {+ ensureOtelIntegration,+ getRegisteredTelemetryIntegrations,+} from "#harness/ai-sdk-telemetry.js";import { getAdvertisedTools } from "#harness/advertised-tools.js";import {applyLastToolCacheBreakpoint,@@ -246,7 +249,7 @@ import {/*** Builds the `telemetry` value for the AI SDK from authored settings.*- * Custom context (authored instrumentation events plus+ * Custom context (authored `InstrumentationDefinition.events` plus* eve-specific identifiers such as `eve.session.id`) is flowed through* {@link buildTelemetryRuntimeContext} because AI SDK v7 surfaces* per-call attributes via `runtimeContext`, not a dedicated metadata field on@@ -311,12 +314,12 @@ function enrichTelemetry(return {functionId: settings?.functionId ?? agentName,includeRuntimeContext,+ // Passing integrations replaces the registered ones for this call, so the+ // bridge has to be composed with them rather than handed over on its own.integrations:bridgeIntegration === undefined? undefined- : getInstrumentationConfig() === undefined- ? [bridgeIntegration]- : [bridgeIntegration, createOtelIntegration()],+ : [bridgeIntegration, ...getRegisteredTelemetryIntegrations()],isEnabled: true,recordInputs: settings?.recordInputs ?? true,recordOutputs: settings?.recordOutputs ?? true,@@ -549,6 +552,9 @@ function buildHarnessToolsWithDynamicSubagents(export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {const baseEmit = config.handleEvent;const otelSettings = getInstrumentationRuntime()?.otelSettings;+ // The custom-context enrichment below still reads the authored config object+ // directly. Its replacement on the provider surface is unresolved, so a+ // provider directory contributes no custom context yet.const authoredConfig = getInstrumentationConfig();if (otelSettings !== undefined) {ensureOtelIntegration();diff --git a/packages/eve/src/internal/application/compiled-artifacts.ts b/packages/eve/src/internal/application/compiled-artifacts.tsindex 561d60f45..a79a9ea0f 100644--- a/packages/eve/src/internal/application/compiled-artifacts.ts+++ b/packages/eve/src/internal/application/compiled-artifacts.ts@@ -1,4 +1,3 @@-import { existsSync } from "node:fs";import { copyFile, mkdir, writeFile } from "node:fs/promises";import { join } from "node:path";@@ -12,12 +11,23 @@ import {} from "#internal/application/package.js";import { buildPackageUserAgent } from "#internal/user-agent.js";import type { AgentWorkflowWorldDefinition } from "#shared/agent-definition.js";-import { readMaterializedAuthoredModuleIndex } from "#internal/materialized-authored-modules.js";+import {+ readMaterializedAuthoredModuleIndex,+ type MaterializedInstrumentation,+} from "#internal/materialized-authored-modules.js";+import {+ resolveInstrumentationLayout,+ type InstrumentationLayout,+} from "#internal/instrumentation-layout.js";import { usesParentDevelopmentWorkflowWorld } from "#internal/workflow/development-world-protocol.js";import { resolveWorkflowWorldImport } from "#internal/workflow/world-target.js";export type BuiltInWorkflowWorldTarget = "local" | "vercel";+export type GeneratedInstrumentationLayout =+ | { readonly kind: "file" }+ | { readonly kind: "directory"; readonly slots: readonly string[] };+/*** Paths to the generated compiled-artifacts files shared by Nitro and the* vendored workflow bundles for one application.@@ -31,15 +41,18 @@ export interface GeneratedCompiledArtifactsFiles {/** Nitro plugin that installs the selected vendored Workflow world. */workflowWorldPluginPath: string;/**- * Optional Nitro plugin that imports the authored instrumentation module+ * Optional Nitro plugin that imports the authored instrumentation modules* from the application when present.*/instrumentationPluginPath?: string;+ /** Layout identity used by dev plugin selection and structural fingerprinting. */+ instrumentationLayout?: GeneratedInstrumentationLayout;/**- * Absolute path to the authored instrumentation module when present.- * Nitro uses this to preserve the module's side effects during bundling.+ * Absolute paths to the authored instrumentation modules when present — one+ * for the file layout, one per provider in the directory layout. Nitro uses these+ * to preserve each module's side effects during bundling.*/- instrumentationSourcePath?: string;+ instrumentationSourcePaths?: readonly string[];}/**@@ -57,7 +70,11 @@ export async function writeCompiledArtifactsFiles(input: {const bootstrapPath = join(input.outDir, "compiled-artifacts-bootstrap.mjs");const instrumentationPluginPath = join(input.outDir, "compiled-artifacts-instrumentation.mjs");const workflowWorldPluginPath = join(input.outDir, "compiled-artifacts-workflow-world.mjs");- const instrumentationPath = resolveInstrumentationModule(input.compileResult.manifest.agentRoot);+ const layout = resolveInstrumentationLayout({+ agentRoot: input.compileResult.manifest.agentRoot,+ providersEnabled:+ input.compileResult.manifest.config.experimental?.instrumentationProviders ?? false,+ });await mkdir(input.outDir, { recursive: true });await writeFile(@@ -78,30 +95,31 @@ export async function writeCompiledArtifactsFiles(input: {}),);- if (instrumentationPath !== undefined) {+ const generatedArtifacts: GeneratedCompiledArtifactsFiles = {+ bootstrapPath,+ workflowWorldPluginPath,+ };++ if (layout !== undefined) {await writeFile(instrumentationPluginPath,createInstrumentationPluginSource({agentName: input.compileResult.manifest.config.name,- instrumentationPath,- registerConfigPath: resolvePackageSourceFilePath("src/harness/instrumentation-config.ts"),+ layout,}),);- }-- const generatedArtifacts: GeneratedCompiledArtifactsFiles = {- bootstrapPath,- workflowWorldPluginPath,- };-- if (instrumentationPath !== undefined) {generatedArtifacts.instrumentationPluginPath = instrumentationPluginPath;- generatedArtifacts.instrumentationSourcePath = instrumentationPath;+ generatedArtifacts.instrumentationLayout = generatedInstrumentationLayout(layout);+ generatedArtifacts.instrumentationSourcePaths = instrumentationSourcePathsOf(layout);}return generatedArtifacts;}+function instrumentationSourcePathsOf(layout: InstrumentationLayout): readonly string[] {+ return layout.kind === "file" ? [layout.modulePath] : Object.values(layout.modulePathsBySlot);+}+// The dev host's Nitro inputs outlive any single generation, so nothing// written here may point into authored source or a prunable snapshot: the// bootstrap references no authored module, and the instrumentation bundle is@@ -113,10 +131,6 @@ export async function writeDevelopmentCompiledArtifactsFiles(input: {}): Promise<GeneratedCompiledArtifactsFiles> {const bootstrapPath = join(input.outDir, "compiled-artifacts-bootstrap.mjs");const instrumentationPluginPath = join(input.outDir, "compiled-artifacts-instrumentation.mjs");- const instrumentationSourcePath = join(- input.outDir,- "compiled-artifacts-instrumentation-source.mjs",- );const workflowWorldPluginPath = join(input.outDir, "compiled-artifacts-workflow-world.mjs");const materializedIndex = await readMaterializedAuthoredModuleIndex(input.runtimeAppRoot);@@ -143,25 +157,66 @@ export async function writeDevelopmentCompiledArtifactsFiles(input: {};if (materializedIndex.instrumentation !== undefined) {- await copyFile(- join(input.runtimeAppRoot, ".eve", "compile", materializedIndex.instrumentation),- instrumentationSourcePath,- );+ const layout = await copyMaterializedInstrumentation({+ instrumentation: materializedIndex.instrumentation,+ outDir: input.outDir,+ runtimeAppRoot: input.runtimeAppRoot,+ });+await writeFile(instrumentationPluginPath,createInstrumentationPluginSource({agentName: input.compileResult.manifest.config.name,- instrumentationPath: instrumentationSourcePath,- registerConfigPath: resolvePackageSourceFilePath("src/harness/instrumentation-config.ts"),+ layout,}),);generatedArtifacts.instrumentationPluginPath = instrumentationPluginPath;- generatedArtifacts.instrumentationSourcePath = instrumentationSourcePath;+ generatedArtifacts.instrumentationLayout = generatedInstrumentationLayout(layout);+ generatedArtifacts.instrumentationSourcePaths = instrumentationSourcePathsOf(layout);}return generatedArtifacts;}+function generatedInstrumentationLayout(+ layout: InstrumentationLayout,+): GeneratedInstrumentationLayout {+ return layout.kind === "file"+ ? { kind: "file" }+ : { kind: "directory", slots: Object.keys(layout.modulePathsBySlot) };+}++/**+ * Copies each materialized instrumentation bundle out of the prunable+ * generation and into the stable dev-host directory, returning the layout in+ * terms of the copies.+ */+async function copyMaterializedInstrumentation(input: {+ readonly instrumentation: MaterializedInstrumentation;+ readonly outDir: string;+ readonly runtimeAppRoot: string;+}): Promise<InstrumentationLayout> {+ const compileRoot = join(input.runtimeAppRoot, ".eve", "compile");+ const copyOne = async (sourceId: string, modulePath: string): Promise<string> => {+ const destination = join(input.outDir, `compiled-artifacts-instrumentation-${sourceId}.mjs`);+ await copyFile(join(compileRoot, modulePath), destination);+ return destination;+ };++ if (input.instrumentation.kind === "file") {+ return {+ kind: "file",+ modulePath: await copyOne("source", input.instrumentation.modulePath),+ };+ }++ const modulePathsBySlot: Record<string, string> = {};+ for (const [slot, modulePath] of Object.entries(input.instrumentation.modulePathsBySlot)) {+ modulePathsBySlot[slot] = await copyOne(slot, modulePath);+ }+ return { kind: "directory", modulePathsBySlot };+}+function createDevelopmentCompiledArtifactsBootstrapSource(agentName: string): string {return ["// Generated by eve. Do not edit by hand.",@@ -174,23 +229,6 @@ function createDevelopmentCompiledArtifactsBootstrapSource(agentName: string): s].join("\n");}-const INSTRUMENTATION_EXTENSIONS = [".ts", ".mts", ".js", ".mjs"];--/**- * Resolves the optional `agent/instrumentation` module from the agent root- * directory. Returns the absolute path if found, `undefined` otherwise.- */-function resolveInstrumentationModule(agentRoot: string): string | undefined {- for (const ext of INSTRUMENTATION_EXTENSIONS) {- const candidate = join(agentRoot, `instrumentation${ext}`);- if (existsSync(candidate)) {- return candidate;- }- }-- return undefined;-}-function stripCompiledModuleMapExports(source: string): string {return source.replace(/^export const moduleMap = /m, "const moduleMap = ")@@ -380,23 +418,73 @@ export function createDevelopmentWorkflowWorldPluginSource(input: {].join("\n");}+/**+ * Generates the Nitro plugin that registers the authored instrumentation.+ *+ * The file layout registers one default export; the directory layout+ * registers one per file, in slot order, awaiting each `setup` so a provider+ * cannot miss an event published while it is still starting up, then builds the+ * one OpenTelemetry pipeline their declarations add up to.+ */function createInstrumentationPluginSource(input: {agentName: string;- instrumentationPath: string;- registerConfigPath: string;+ layout: InstrumentationLayout;}): string {+ const agentName = JSON.stringify(input.agentName);++ if (input.layout.kind === "file") {+ const registerConfigPath = resolvePackageSourceFilePath(+ "src/harness/instrumentation-config.ts",+ );+ return [+ "// Generated by eve. Do not edit by hand.",+ `import * as instrumentationModule from ${stringifyEsmImportSpecifier(input.layout.modulePath)};`,+ `import { registerInstrumentationConfig } from ${stringifyEsmImportSpecifier(registerConfigPath)};`,+ "",+ "if (instrumentationModule.default != null) {",+ ` await registerInstrumentationConfig(instrumentationModule.default, { agentName: ${agentName} });`,+ "}",+ "",+ "// Default export satisfies the Nitro plugin contract so this file",+ "// can be used directly as a Nitro plugin without a separate wrapper.",+ "export default function installInstrumentationPlugin() {}",+ "",+ ].join("\n");+ }++ const registerProviderPath = resolvePackageSourceFilePath(+ "src/harness/instrumentation-providers.ts",+ );+ const slots = Object.entries(input.layout.modulePathsBySlot);+return ["// Generated by eve. Do not edit by hand.",- `import * as instrumentationModule from ${stringifyEsmImportSpecifier(input.instrumentationPath)};`,- `import { registerInstrumentationConfig } from ${stringifyEsmImportSpecifier(input.registerConfigPath)};`,+ ...slots.map(+ ([slot, modulePath], index) =>+ `import * as provider${index} from ${stringifyEsmImportSpecifier(modulePath)}; // ${slot}`,+ ),+ `import { finalizeInstrumentationProviders, registerInstrumentationProvider, seedInstrumentationProviders, shutdownInstrumentationProviders } from ${stringifyEsmImportSpecifier(registerProviderPath)};`,"",- "if (instrumentationModule.default != null) {",- ` await registerInstrumentationConfig(instrumentationModule.default, { agentName: ${JSON.stringify(input.agentName)} });`,- "}",+ "seedInstrumentationProviders();",+ "",+ ...slots.flatMap(([slot], index) => [+ "await registerInstrumentationProvider({",+ ` agentName: ${agentName},`,+ ` slot: ${JSON.stringify(slot)},`,+ ` value: provider${index}.default,`,+ "});",+ ]),+ "",+ `finalizeInstrumentationProviders({ serviceName: ${agentName} });`,"","// Default export satisfies the Nitro plugin contract so this file","// can be used directly as a Nitro plugin without a separate wrapper.",- "export default function installInstrumentationPlugin() {}",+ "export default function installInstrumentationPlugin(nitroApp) {",+ " // The last point a buffered exporter can still reach the network.",+ " nitroApp?.hooks?.hook('close', async () => {",+ " await shutdownInstrumentationProviders();",+ " });",+ "}","",].join("\n");}diff --git a/packages/eve/src/internal/application/dev-environment.ts b/packages/eve/src/internal/application/dev-environment.tsindex acbd09cfc..daa2ff244 100644--- a/packages/eve/src/internal/application/dev-environment.ts+++ b/packages/eve/src/internal/application/dev-environment.ts@@ -5,3 +5,25 @@ export const EVE_DEV_ENV_FLAG = "EVE_DEV";export function isEveDevEnvironment(): boolean {return process.env[EVE_DEV_ENV_FLAG] === "1";}++/** Environment flag set for a server `eve eval` started to run against. */+export const EVE_EVALUATION_ENV_FLAG = "EVE_EVALUATION";++/** Stable identifier for the local eval run this server was started to serve. */+export const EVE_EVALUATION_RUN_ID_ENV = "EVE_EVALUATION_RUN_ID";++/**+ * Reports whether this process exists to serve an eval run.+ *+ * False for a server that `eve eval --url` merely points at: that process was+ * started to serve ordinary traffic and cannot know an eval is among it.+ */+export function isEveEvaluationEnvironment(): boolean {+ return process.env[EVE_EVALUATION_ENV_FLAG] === "1";+}++export function resolveEveEvaluationRunId(): string | undefined {+ if (!isEveEvaluationEnvironment()) return undefined;+ const runId = process.env[EVE_EVALUATION_RUN_ID_ENV];+ return runId === undefined || runId.length === 0 ? undefined : runId;+}diff --git a/packages/eve/src/internal/authored-definition/core.ts b/packages/eve/src/internal/authored-definition/core.tsindex 93ce950ae..633970fe1 100644--- a/packages/eve/src/internal/authored-definition/core.ts+++ b/packages/eve/src/internal/authored-definition/core.ts@@ -259,9 +259,20 @@ function normalizeAgentExperimentalDefinition(message: string,): NonNullable<NormalizedAgentDefinition["experimental"]> {const record = expectObjectRecord(value, message);- expectOnlyKnownKeys(record, ["subagentPersistentSessions", "workflow"], message);+ expectOnlyKnownKeys(+ record,+ ["instrumentationProviders", "subagentPersistentSessions", "workflow"],+ message,+ );const normalizedDefinition: Mutable<NonNullable<NormalizedAgentDefinition["experimental"]>> = {};+ if (record.instrumentationProviders !== undefined) {+ if (typeof record.instrumentationProviders !== "boolean") {+ throw new Error(`${message} "experimental.instrumentationProviders" must be a boolean.`);+ }+ normalizedDefinition.instrumentationProviders = record.instrumentationProviders;+ }+if (record.subagentPersistentSessions !== undefined) {if (typeof record.subagentPersistentSessions !== "boolean") {throw new Error(`${message} "experimental.subagentPersistentSessions" must be a boolean.`);diff --git a/packages/eve/src/internal/instrumentation-layout.integration.test.ts b/packages/eve/src/internal/instrumentation-layout.integration.test.tsnew file mode 100644index 000000000..eca880b2b--- /dev/null+++ b/packages/eve/src/internal/instrumentation-layout.integration.test.ts@@ -0,0 +1,126 @@+import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { beforeEach, describe, expect, it } from "vitest";++import { resolveInstrumentationLayout } from "#internal/instrumentation-layout.js";++let agentRoot: string;++beforeEach(() => {+ agentRoot = mkdtempSync(join(tmpdir(), "eve-instrumentation-layout-"));+});++function writeInstrumentationFile(extension = ".ts"): string {+ const path = join(agentRoot, `instrumentation${extension}`);+ writeFileSync(path, "export default {};\n");+ return path;+}++function writeInstrumentationProvider(fileName: string): string {+ const directory = join(agentRoot, "instrumentation");+ mkdirSync(directory, { recursive: true });+ const path = join(directory, fileName);+ writeFileSync(path, "export default {};\n");+ return path;+}++describe("resolveInstrumentationLayout with providers off", () => {+ it("returns nothing when the agent authored no instrumentation", () => {+ expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: false })).toBeUndefined();+ });++ it("resolves the single instrumentation file", () => {+ const modulePath = writeInstrumentationFile();++ expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: false })).toEqual({+ kind: "file",+ modulePath,+ });+ });++ it.each([[".mts"], [".js"], [".mjs"]])("resolves a %s instrumentation file", (extension) => {+ const modulePath = writeInstrumentationFile(extension);++ expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: false })).toEqual({+ kind: "file",+ modulePath,+ });+ });++ it("ignores a providers directory", () => {+ writeInstrumentationProvider("otel.ts");++ expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: false })).toBeUndefined();+ });+});++describe("resolveInstrumentationLayout with providers on", () => {+ it("returns an empty directory layout for eve's built-in destinations", () => {+ expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toEqual({+ kind: "directory",+ modulePathsBySlot: {},+ });+ });++ it("keys each file by the slot its name derives", () => {+ const otel = writeInstrumentationProvider("otel.ts");+ const local = writeInstrumentationProvider("local.mts");++ expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toEqual({+ kind: "directory",+ modulePathsBySlot: { local, otel },+ });+ });++ it("orders slots independently of directory enumeration", () => {+ writeInstrumentationProvider("otel.ts");+ writeInstrumentationProvider("agent-runs.ts");+ writeInstrumentationProvider("local.ts");++ const layout = resolveInstrumentationLayout({ agentRoot, providersEnabled: true });++ expect(Object.keys(layout?.kind === "directory" ? layout.modulePathsBySlot : {})).toEqual([+ "agent-runs",+ "local",+ "otel",+ ]);+ });++ it("ignores files that are not instrumentation modules", () => {+ writeInstrumentationProvider("otel.ts");+ writeInstrumentationProvider("README.md");++ const layout = resolveInstrumentationLayout({ agentRoot, providersEnabled: true });++ expect(Object.keys(layout?.kind === "directory" ? layout.modulePathsBySlot : {})).toEqual([+ "otel",+ ]);+ });++ it("rejects two files claiming one slot", () => {+ writeInstrumentationProvider("otel.ts");+ writeInstrumentationProvider("otel.js");++ expect(() => resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toThrow(+ /Two files declare the "otel" instrumentation provider/,+ );+ });++ it("rejects a single instrumentation file, naming the flag", () => {+ writeInstrumentationFile();++ expect(() => resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toThrow(+ /experimental\.instrumentationProviders/,+ );+ });++ it("prefers the file error when both layouts are present", () => {+ writeInstrumentationFile();+ writeInstrumentationProvider("otel.ts");++ expect(() => resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toThrow(+ /Move it into the "instrumentation\/" directory/,+ );+ });+});diff --git a/packages/eve/src/internal/instrumentation-layout.ts b/packages/eve/src/internal/instrumentation-layout.tsnew file mode 100644index 000000000..6c46082d7--- /dev/null+++ b/packages/eve/src/internal/instrumentation-layout.ts@@ -0,0 +1,119 @@+import { existsSync, readdirSync, statSync } from "node:fs";+import { join } from "node:path";++import { createLogger } from "#internal/logging.js";++const INSTRUMENTATION_EXTENSIONS = [".ts", ".mts", ".js", ".mjs"] as const;++const INSTRUMENTATION_DIRECTORY = "instrumentation";++const PROVIDERS_FLAG = "experimental.instrumentationProviders";+const log = createLogger("internal.instrumentation-layout");++/**+ * How instrumentation is authored for one agent.+ *+ * `file` is the single `agent/instrumentation.ts` default export. `directory`+ * contains one provider per file, keyed by the slot name the+ * file derives (`instrumentation/otel.ts` → `otel`). Which one an agent may use+ * is decided by `experimental.instrumentationProviders`, never by what happens+ * to be on disk.+ */+export type InstrumentationLayout =+ | { readonly kind: "file"; readonly modulePath: string }+ | { readonly kind: "directory"; readonly modulePathsBySlot: Readonly<Record<string, string>> };++/**+ * Resolves the instrumentation layout for one agent root.+ *+ * With providers on, an empty directory layout still installs eve's built-in+ * destinations. Throws when the layout on disk is not the one the flag selects:+ * the wrong layout would otherwise be skipped silently, and telemetry that+ * quietly does nothing is the failure this whole surface exists to prevent.+ */+export function resolveInstrumentationLayout(input: {+ readonly agentRoot: string;+ readonly providersEnabled: boolean;+}): InstrumentationLayout | undefined {+ const filePath = resolveInstrumentationFile(input.agentRoot);+ const directoryPath = join(input.agentRoot, INSTRUMENTATION_DIRECTORY);+ const hasDirectory = existsSync(directoryPath) && statSync(directoryPath).isDirectory();++ if (!input.providersEnabled) {+ if (hasDirectory) {+ log.warn("ignoring instrumentation provider directory because providers are off", {+ agentRoot: input.agentRoot,+ flag: PROVIDERS_FLAG,+ });+ }++ return filePath === undefined ? undefined : { kind: "file", modulePath: filePath };+ }++ if (filePath !== undefined) {+ throw new Error(+ `Found "${filePath}", but \`${PROVIDERS_FLAG}\` is on. Move it into the "${INSTRUMENTATION_DIRECTORY}/" directory as one file per provider.`,+ );+ }++ if (!hasDirectory) {+ return { kind: "directory", modulePathsBySlot: {} };+ }++ return {+ kind: "directory",+ modulePathsBySlot: collectInstrumentationProviderModules(directoryPath),+ };+}++/**+ * Maps each `instrumentation/<slot>.<ext>` file to its absolute path.+ *+ * Slots are sorted so the registration order a provider sees does not depend on+ * how the filesystem happens to enumerate the directory.+ */+function collectInstrumentationProviderModules(+ directoryPath: string,+): Readonly<Record<string, string>> {+ const modulePathsBySlot = new Map<string, string>();++ for (const entry of readdirSync(directoryPath, { withFileTypes: true })) {+ if (!entry.isFile()) continue;++ const extension = INSTRUMENTATION_EXTENSIONS.find((candidate) =>+ entry.name.endsWith(candidate),+ );+ if (extension === undefined) continue;++ const slot = entry.name.slice(0, -extension.length);+ if (slot === "") continue;++ const existing = modulePathsBySlot.get(slot);+ if (existing !== undefined) {+ throw new Error(+ `Two files declare the "${slot}" instrumentation provider in "${directoryPath}". Keep one of them.`,+ );+ }++ modulePathsBySlot.set(slot, join(directoryPath, entry.name));+ }++ return Object.fromEntries(+ [...modulePathsBySlot].sort(([left], [right]) => left.localeCompare(right)),+ );+}++/**+ * Resolves the single `agent/instrumentation` module, ignoring any directory of+ * the same name.+ */+function resolveInstrumentationFile(agentRoot: string): string | undefined {+ for (const extension of INSTRUMENTATION_EXTENSIONS) {+ const candidate = join(agentRoot, `${INSTRUMENTATION_DIRECTORY}${extension}`);+ if (existsSync(candidate)) {+ return candidate;+ }+ }++ return undefined;+}diff --git a/packages/eve/src/internal/materialized-authored-modules.ts b/packages/eve/src/internal/materialized-authored-modules.tsindex 7c1376080..6967733cf 100644--- a/packages/eve/src/internal/materialized-authored-modules.ts+++ b/packages/eve/src/internal/materialized-authored-modules.ts@@ -10,16 +10,24 @@ import {bundleAuthoredModuleMapForGeneration,} from "#internal/authored-module-loader.js";import { serializeCompiledManifestForFingerprint } from "#internal/compiled-manifest-fingerprint.js";+import { resolveInstrumentationLayout } from "#internal/instrumentation-layout.js";const MATERIALIZED_MODULES_DIRECTORY = "authored-modules";const MATERIALIZED_MODULES_INDEX = "authored-modules.json";-const INSTRUMENTATION_EXTENSIONS = [".ts", ".mts", ".js", ".mjs"] as const;++/**+ * The materialized instrumentation modules, mirroring the layout they were+ * authored in. Paths are relative to `.eve/compile`.+ */+export type MaterializedInstrumentation =+ | { readonly kind: "file"; readonly modulePath: string }+ | { readonly kind: "directory"; readonly modulePathsBySlot: Readonly<Record<string, string>> };export interface MaterializedAuthoredModuleIndex {readonly fingerprint: string;- readonly instrumentation?: string;+ readonly instrumentation?: MaterializedInstrumentation;readonly moduleMap: string;- readonly version: 2;+ readonly version: 3;}export async function materializeAuthoredModules(input: {@@ -54,22 +62,40 @@ export async function materializeAuthoredModules(input: {await writeFile(join(modulesRoot, moduleMapFileName), moduleMapCode);fingerprint.update("module-map\0").update(moduleMapCode).update("\0");- const instrumentation = resolveInstrumentationModule(manifest.agentRoot);- let instrumentationPath: string | undefined;-- if (instrumentation !== undefined) {- const code = await bundleAuthoredModuleForGeneration(instrumentation, {- externalDependencies: manifest.config.build?.externalDependencies ?? [],- });+ const layout = resolveInstrumentationLayout({+ agentRoot: manifest.agentRoot,+ providersEnabled: manifest.config.experimental?.instrumentationProviders ?? false,+ });+ const externalDependencies = manifest.config.build?.externalDependencies ?? [];+ const materializeInstrumentationModule = async (+ sourceId: string,+ sourcePath: string,+ ): Promise<string> => {+ const code = await bundleAuthoredModuleForGeneration(sourcePath, { externalDependencies });const fileName = createMaterializedModuleFileName(ROOT_COMPILED_AGENT_NODE_ID,- "instrumentation",+ `instrumentation:${sourceId}`,code,);await writeFile(join(modulesRoot, fileName), code);- instrumentationPath = join(MATERIALIZED_MODULES_DIRECTORY, fileName);- fingerprint.update("instrumentation\0").update(code).update("\0");+ fingerprint.update(`instrumentation:${sourceId}\0`).update(code).update("\0");+ return join(MATERIALIZED_MODULES_DIRECTORY, fileName);+ };++ let instrumentation: MaterializedInstrumentation | undefined;++ if (layout?.kind === "file") {+ instrumentation = {+ kind: "file",+ modulePath: await materializeInstrumentationModule("file", layout.modulePath),+ };+ } else if (layout?.kind === "directory") {+ const modulePathsBySlot: Record<string, string> = {};+ for (const [slot, sourcePath] of Object.entries(layout.modulePathsBySlot)) {+ modulePathsBySlot[slot] = await materializeInstrumentationModule(slot, sourcePath);+ }+ instrumentation = { kind: "directory", modulePathsBySlot };}await hashDirectoryIfPresent({@@ -79,16 +105,16 @@ export async function materializeAuthoredModules(input: {});const index: {fingerprint: string;- instrumentation?: string;+ instrumentation?: MaterializedInstrumentation;moduleMap: string;- version: 2;+ version: 3;} = {fingerprint: fingerprint.digest("hex"),moduleMap: moduleMapPath,- version: 2,+ version: 3,};- if (instrumentationPath !== undefined) {- index.instrumentation = instrumentationPath;+ if (instrumentation !== undefined) {+ index.instrumentation = instrumentation;}await writeFile(join(compileRoot, MATERIALIZED_MODULES_INDEX), `${JSON.stringify(index)}\n`);return index;@@ -106,12 +132,12 @@ export async function readMaterializedAuthoredModuleIndex(await readFile(indexPath, "utf8"),) as Partial<MaterializedAuthoredModuleIndex>;if (- parsed.version !== 2 ||+ parsed.version !== 3 ||typeof parsed.fingerprint !== "string" ||parsed.fingerprint.length === 0 ||typeof parsed.moduleMap !== "string" ||parsed.moduleMap.length === 0 ||- (parsed.instrumentation !== undefined && typeof parsed.instrumentation !== "string")+ !isMaterializedInstrumentation(parsed.instrumentation)) {throw new Error(`Invalid materialized authored module index at "${indexPath}".`);}@@ -119,6 +145,26 @@ export async function readMaterializedAuthoredModuleIndex(return parsed as MaterializedAuthoredModuleIndex;}+function isMaterializedInstrumentation(value: unknown): boolean {+ if (value === undefined) return true;+ if (typeof value !== "object" || value === null) return false;++ const candidate = value as Partial<MaterializedInstrumentation>;+ if (candidate.kind === "file") {+ return typeof (candidate as { modulePath?: unknown }).modulePath === "string";+ }+ if (candidate.kind === "directory") {+ const paths = (candidate as { modulePathsBySlot?: unknown }).modulePathsBySlot;+ return (+ typeof paths === "object" &&+ paths !== null &&+ Object.values(paths).every((path) => typeof path === "string")+ );+ }++ return false;+}+async function readCompiledManifest(path: string): Promise<CompiledAgentManifest> {const manifest = JSON.parse(await readFile(path, "utf8")) as CompiledAgentManifest;@@ -129,17 +175,6 @@ async function readCompiledManifest(path: string): Promise<CompiledAgentManifestreturn manifest;}-function resolveInstrumentationModule(agentRoot: string): string | undefined {- for (const extension of INSTRUMENTATION_EXTENSIONS) {- const candidate = join(agentRoot, `instrumentation${extension}`);- if (existsSync(candidate)) {- return candidate;- }- }-- return undefined;-}-function createMaterializedModuleFileName(nodeId: string, sourceId: string, code: string): string {return `${createHash("sha256").update(nodeId)diff --git a/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts b/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.tsindex 8887d0de8..f1add13b6 100644--- a/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts+++ b/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts@@ -13,6 +13,7 @@ import {stageDevelopmentGeneration,} from "#internal/nitro/development-generation.js";import { createAuthoredSourceRuntimeCompiledArtifactsSource } from "#internal/application/runtime-compiled-artifacts-source.js";+import type { MaterializedInstrumentation } from "#internal/materialized-authored-modules.js";import { useScenarioApp } from "#internal/testing/scenario-app.js";describe("development generation artifacts", () => {@@ -489,9 +490,12 @@ describe("development generation artifacts", () => {join(first.runtimeAppRoot, ".eve", "compile", "authored-modules.json"),"utf8",),- ) as { readonly instrumentation?: string };+ ) as { readonly instrumentation?: MaterializedInstrumentation };+ if (firstIndex.instrumentation?.kind !== "file") {+ throw new Error("expected materialized file instrumentation");+ }const materializedInstrumentation = await readFile(- join(first.runtimeAppRoot, ".eve", "compile", firstIndex.instrumentation!),+ join(first.runtimeAppRoot, ".eve", "compile", firstIndex.instrumentation.modulePath),"utf8",);diff --git a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.tsindex 3af429a9b..808b1da10 100644--- a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts+++ b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts@@ -216,6 +216,7 @@ describe("application Nitro creation", () => {const { createDevelopmentApplicationNitro } =await import("#internal/nitro/host/create-application-nitro.js");const preparedHost = createPreparedHost();+ preparedHost.compiledArtifacts.instrumentationLayout = { kind: "file" };preparedHost.compiledArtifacts.instrumentationPluginPath = "/app/instrumentation.mjs";await createDevelopmentApplicationNitro(preparedHost);@@ -227,6 +228,29 @@ describe("application Nitro creation", () => {);});+ it("lets the provider pipeline own default local tracing", async () => {+ const { createDevelopmentApplicationNitro } =+ await import("#internal/nitro/host/create-application-nitro.js");++ for (const slots of ["rows", "local"] as const) {+ const nitroStub = createNitroStub();+ createNitroMock.mockResolvedValueOnce(nitroStub.nitro);+ const preparedHost = createPreparedHost();+ preparedHost.compiledArtifacts.instrumentationLayout = {+ kind: "directory",+ slots: [slots],+ };+ preparedHost.compiledArtifacts.instrumentationPluginPath = "/app/instrumentation.mjs";++ await createDevelopmentApplicationNitro(preparedHost);++ const plugins = createNitroMock.mock.calls.at(-1)?.[0].plugins as string[];+ expect(plugins).not.toEqual(+ expect.arrayContaining([expect.stringContaining("local-tracing-runtime-plugin.ts")]),+ );+ }+ });+it("preserves workflow bundle side effects and skips workflow transform for cached bundles", async () => {const nitroStub = createNitroStub();createNitroMock.mockResolvedValueOnce(nitroStub.nitro);diff --git a/packages/eve/src/internal/nitro/host/create-application-nitro.ts b/packages/eve/src/internal/nitro/host/create-application-nitro.tsindex 06bae42bf..a34a0b577 100644--- a/packages/eve/src/internal/nitro/host/create-application-nitro.ts+++ b/packages/eve/src/internal/nitro/host/create-application-nitro.ts@@ -520,9 +520,11 @@ function addDynamicCapabilityTransformPlugin(nitro: Nitro): void {*/function addInstrumentationModuleSideEffectsPlugin(nitro: Nitro,- instrumentationModulePath: string,+ instrumentationModulePaths: readonly string[],): void {- const normalizedInstrumentationModulePath = normalizePath(instrumentationModulePath);+ const normalizedInstrumentationModulePaths = new Set(+ instrumentationModulePaths.map(normalizePath),+ );nitro.hooks.hook("rollup:before", (_nitro, config) => {if (!Array.isArray(config.plugins)) {@@ -532,7 +534,7 @@ function addInstrumentationModuleSideEffectsPlugin(config.plugins.unshift({name: "eve:instrumentation-module-side-effects",resolveId(source: string) {- if (normalizePath(source) !== normalizedInstrumentationModulePath) {+ if (!normalizedInstrumentationModulePaths.has(normalizePath(source))) {return null;}@@ -682,10 +684,10 @@ function configureSharedApplicationNitro(addDynamicCapabilityTransformPlugin(nitro);- if (preparedHost.compiledArtifacts.instrumentationSourcePath !== undefined) {+ if (preparedHost.compiledArtifacts.instrumentationSourcePaths !== undefined) {addInstrumentationModuleSideEffectsPlugin(nitro,- preparedHost.compiledArtifacts.instrumentationSourcePath,+ preparedHost.compiledArtifacts.instrumentationSourcePaths,);}}diff --git a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.tsindex 62e5d3621..b8d096c8b 100644--- a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts+++ b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts@@ -20,6 +20,7 @@ afterEach(async () => {interface HostVariant {readonly channels?: CompiledAgentManifest["channels"];+ readonly instrumentationSlot?: string;readonly instrumentationSource?: string;readonly schedules?: CompiledAgentManifest["schedules"];readonly workflowWorld?: "local" | "vercel";@@ -55,7 +56,15 @@ async function createHost(variant: HostVariant = {}): Promise<PreparedDevelopmenappRoot,compiledArtifacts: {bootstrapPath: join(appRoot, "bootstrap.mjs"),- instrumentationSourcePath,+ ...(instrumentationSourcePath === undefined+ ? {}+ : {+ instrumentationLayout:+ variant.instrumentationSlot === undefined+ ? ({ kind: "file" } as const)+ : ({ kind: "directory", slots: [variant.instrumentationSlot] } as const),+ instrumentationSourcePaths: [instrumentationSourcePath],+ }),workflowWorldPluginPath: join(appRoot, "workflow-world.mjs"),},compileResult: { manifest } as PreparedDevelopmentApplicationHost["compileResult"],@@ -81,6 +90,18 @@ describe("computeDevelopmentHostFingerprint", () => {expect(changed).not.toBe(base);});+ it("treats provider slot names as structural", async () => {+ const source = "export default {}\n";+ const first = await computeDevelopmentHostFingerprint(+ await createHost({ instrumentationSlot: "a", instrumentationSource: source }),+ );+ const renamed = await computeDevelopmentHostFingerprint(+ await createHost({ instrumentationSlot: "b", instrumentationSource: source }),+ );++ expect(renamed).not.toBe(first);+ });+it("treats channel route topology as structural", async () => {const base = await computeDevelopmentHostFingerprint(await createHost());const withRoute = await computeDevelopmentHostFingerprint(diff --git a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.tsindex 5206c23f5..2b27a8a02 100644--- a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts+++ b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts@@ -50,12 +50,21 @@ export async function computeDevelopmentHostFingerprint(return createHash("sha256").update(JSON.stringify(payload)).digest("hex");}-async function readInstrumentationSource(- host: PreparedDevelopmentApplicationHost,-): Promise<string | null> {- const path = host.compiledArtifacts.instrumentationSourcePath;- if (path === undefined) {+async function readInstrumentationSource(host: PreparedDevelopmentApplicationHost): Promise<{+ readonly kind: "directory" | "file";+ readonly modules: readonly { readonly slot: string | null; readonly source: string }[];+} | null> {+ const paths = host.compiledArtifacts.instrumentationSourcePaths;+ const layout = host.compiledArtifacts.instrumentationLayout;+ if (paths === undefined || layout === undefined) {return null;}- return await readFile(path, "utf8");+ const sources = await Promise.all(paths.map(async (path) => await readFile(path, "utf8")));+ return {+ kind: layout.kind,+ modules: sources.map((source, index) => ({+ slot: layout.kind === "directory" ? (layout.slots[index] ?? null) : null,+ source,+ })),+ };}diff --git a/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts b/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.tsindex 4f0612f9e..e36d53b59 100644--- a/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts+++ b/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts@@ -108,9 +108,9 @@ describe("application host preparation", () => {join(firstHostDirectory, "compiled-artifacts-workflow-world.mjs"),);expect(firstHost.compiledArtifacts.bootstrapPath).not.toContain("/.eve/dev-runtime/snapshots/");- expect(firstHost.compiledArtifacts.instrumentationSourcePath).toBe(+ expect(firstHost.compiledArtifacts.instrumentationSourcePaths).toEqual([join(firstHostDirectory, "compiled-artifacts-instrumentation-source.mjs"),- );+ ]);expect(await readFile(firstBootstrapPath, "utf8")).not.toContain(normalizeEsmImportSpecifier(agentModulePath),);diff --git a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.tsindex 9617df9c6..48752f804 100644--- a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts+++ b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts@@ -58,9 +58,8 @@ export interface TraceChannelRequestInput {* waiting for `event.waitUntil()` work or streamed response bodies.** Emitting these spans is opt-in: unless authored instrumentation enables it- * via `defineInstrumentation({ traceChannelRequests: true })`, the handler runs- * with no span (`undefined`) and no context extraction — a true bypass, not a- * non-recording span.+ * via `traceChannelRequests: true`, the handler runs with no span (`undefined`)+ * and no context extraction — a true bypass, not a non-recording span.** This is observability-only: it never changes the response and performs no* synchronous span export in the request path, adding only minimal in-processdiff --git a/packages/eve/src/public/definitions/exact.test.ts b/packages/eve/src/public/definitions/exact.test.tsindex d6aa982e6..dd1bf1d4e 100644--- a/packages/eve/src/public/definitions/exact.test.ts+++ b/packages/eve/src/public/definitions/exact.test.ts@@ -221,11 +221,18 @@ function typeOnlyFixtures(): void {// @ts-expect-error Instructions identity is path-derived.defineInstructions(instructionsWithName);+ defineInstrumentation({+ isEnabled: true,+ recordInputs: true,+ });++ // Unlike the helpers above, `defineInstrumentation` takes a generic union — a+ // config and a provider overlap on `events` and `setup` — so it cannot use+ // `ExactDefinition`. Excess keys reach `eve build` instead.const instrumentationWithEnabled = {isEnabled: true,recordInputs: true,};- // @ts-expect-error Instrumentation has no separate enable toggle.defineInstrumentation(instrumentationWithEnabled);defineInstrumentation({diff --git a/packages/eve/src/public/instrumentation/index.ts b/packages/eve/src/public/instrumentation/index.tsindex a5d38f09d..85be1b8da 100644--- a/packages/eve/src/public/instrumentation/index.ts+++ b/packages/eve/src/public/instrumentation/index.ts@@ -1,15 +1,21 @@-import type { ExactDefinition } from "#public/definitions/exact.js";-/**- * Instrumentation authoring helpers for `agent/instrumentation.ts`.+ * Instrumentation authoring helpers for `agent/instrumentation.ts` and, with+ * `experimental.instrumentationProviders` on, `agent/instrumentation/`.*/import type { ModelMessage, SystemModelMessage } from "ai";import type { SessionAuthContext, SessionParent } from "#channel/types.js";import type { InstrumentationChannel } from "#public/channels/index.js";+import {+ PROVIDER,+ type ProviderDefinition,+ type ProviderSetupContext,+} from "#public/instrumentation/provider.js";import type { JsonObject } from "#shared/json.js";+export * from "#public/instrumentation/provider.js";+// Re-export channel metadata types so existing `eve/instrumentation`// imports continue to work. The canonical home is `eve/channels`.export {@@ -23,14 +29,13 @@ export {/*** Context passed to the {@link InstrumentationDefinition.setup} callback.+ *+ * The same context both layouts receive. Keeping one type is what gives+ * {@link defineInstrumentation}'s union a contextual signature for `setup`;+ * two divergent ones would leave every authored `setup(context)` parameter an+ * implicit `any`.*/-export interface InstrumentationSetupContext {- /**- * The agent name declared by `defineAgent`. Use as the `serviceName` for- * `registerOTel` instead of a hard-coded string.- */- readonly agentName: string;-}+export interface InstrumentationSetupContext extends ProviderSetupContext {}/*** User-authored runtime context values attached to AI SDK telemetry spans.@@ -161,21 +166,39 @@ export interface InstrumentationDefinition {*/readonly traceChannelRequests?: boolean;/**- * Setup callback invoked at server startup with the resolved agent name.- * Use it to call `registerOTel` or other OTel provider setup;- * `context.agentName` comes from `defineAgent`.+ * Setup callback invoked at server startup, before the first request. Use it+ * to call `registerOTel` or other OTel provider setup; `context.agentName`+ * comes from `defineAgent`. A returned promise is awaited.*/- readonly setup?: (context: InstrumentationSetupContext) => void;+ readonly setup?: (context: InstrumentationSetupContext) => void | PromiseLike<void>;}/**- * Export the result as the default export of `agent/instrumentation.ts`. eve- * reads these settings at server startup and applies them to every AI SDK- * model call. The `setup` callback runs later with the resolved agent name,- * not during `defineInstrumentation` itself.+ * Declares instrumentation, in either of eve's two layouts.+ *+ * Export the result as the default export of `agent/instrumentation.ts`, or —+ * with `experimental.instrumentationProviders` on — of one file under+ * `agent/instrumentation/`. The layout decides how eve reads the value; the two+ * are mutually exclusive builds, so only one can apply. `setup` runs at server+ * startup, not during this call.+ *+ * The parameter is a union because a provider and a legacy config overlap on+ * `events` and `setup`, so no value-level check separates them. One consequence+ * is that excess-property checking is weaker here than it was against the+ * config shape alone, and a misspelled key can reach `eve build` rather than+ * failing at `tsc`.*/-export function defineInstrumentation<T extends InstrumentationDefinition>(- definition: ExactDefinition<T, InstrumentationDefinition>,-): T {- return definition;+export function defineInstrumentation<+ const TDefinition extends InstrumentationDefinition | ProviderDefinition,+>(definition: TDefinition): InstrumentationDeclaration<TDefinition> {+ return { ...definition, [PROVIDER]: true };}++/** The branded result of {@link defineInstrumentation}. */+export type InstrumentationDeclaration<+ TDefinition extends InstrumentationDefinition | ProviderDefinition =+ | InstrumentationDefinition+ | ProviderDefinition,+> = TDefinition & {+ readonly [PROVIDER]: true;+};diff --git a/packages/eve/src/public/instrumentation/otel.ts b/packages/eve/src/public/instrumentation/otel.tsnew file mode 100644index 000000000..f5beb31ed--- /dev/null+++ b/packages/eve/src/public/instrumentation/otel.ts@@ -0,0 +1,59 @@+/**+ * The OpenTelemetry authoring surface for `agent/instrumentation/`.+ *+ * Two halves, because OpenTelemetry has two: `otel()` is the settings a process+ * can only hold one of, and an integration is a destination, of which there may+ * be as many as there are files.+ *+ * Reachable only with `experimental.instrumentationProviders` on. With the flag+ * off nothing discovers that directory, so these compile but never run.+ */++import { createLocalTracesProcessor, resolveLocalTracesContent } from "#tracing/local-traces.js";+import {+ agentRunsIntegration,+ otelIntegration,+ type ContentOptions,+ type OtelIntegration,+} from "#tracing/otel-declaration.js";++export {+ isOtelDeclaration,+ isOtelIntegration,+ otel,+ otelIntegration,+ type ContentOptions,+ type OtelDeclaration,+ type OtelIntegration,+ type OtelIntegrationOptions,+ type OtelOptions,+} from "#tracing/otel-declaration.js";++export type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js";++/**+ * Vercel Agent Runs, enabled by default in production.+ *+ * Export it from `agent/instrumentation/agent-runs.ts` to narrow content, or+ * export `disableInstrumentation()` from that file to turn it off.+ */+export function agentRuns(options: ContentOptions = {}): OtelIntegration {+ return agentRunsIntegration(options);+}++/**+ * The local trace spool `eve dev` reads, as a destination.+ *+ * Export it from `agent/instrumentation/local.ts` to keep it alongside a hosted+ * backend, or export `disableInstrumentation()` from that file to turn it off.+ * Omitting the file leaves eve's default in place.+ *+ * `EVE_TRACES_CONTENT=off` narrows this destination and no other, so declining+ * content locally leaves what a hosted backend receives alone.+ */+export function localTraces(options: ContentOptions = {}): OtelIntegration {+ return otelIntegration({+ ...resolveLocalTracesContent(options),+ spanProcessors: [createLocalTracesProcessor()],+ });+}diff --git a/packages/eve/src/public/instrumentation/provider.test.ts b/packages/eve/src/public/instrumentation/provider.test.tsnew file mode 100644index 000000000..0fe3e5108--- /dev/null+++ b/packages/eve/src/public/instrumentation/provider.test.ts@@ -0,0 +1,92 @@+import { describe, expect, it } from "vitest";++import {+ defineInstrumentation,+ disableInstrumentation,+ isInstrumentationDisabled,+ isInstrumentationProvider,+ type InstrumentationSetupContext,+} from "#public/instrumentation/index.js";++describe("defineInstrumentation", () => {+ it("keeps the legacy setup context constructible with only the agent name", () => {+ const context: InstrumentationSetupContext = { agentName: "weather" };++ expect(context).toEqual({ agentName: "weather" });+ });++ it("brands a provider-shaped declaration", () => {+ const provider = defineInstrumentation({+ events: {+ "session.started"(event) {+ void event.sessionId;+ },+ },+ });++ expect(isInstrumentationProvider(provider)).toBe(true);+ });++ it("brands a legacy config-shaped declaration", () => {+ const config = defineInstrumentation({ functionId: "support", recordInputs: false });++ expect(isInstrumentationProvider(config)).toBe(true);+ expect(config.functionId).toBe("support");+ expect(config).toMatchObject({ functionId: "support", recordInputs: false });+ });++ it("infers terminal handler events from union-typed discriminants", () => {+ const provider = defineInstrumentation({+ events: {+ "action.completed": (event) => void [event.acceptedAtMs, event.outcome, event.usage],+ "action.failed": (event) => void [event.acceptedAtMs, event.errorCode, event.outcome],+ "session.completed": (event) => void event.sessionId,+ "step.attempt.completed": (event) => void event.scope,+ "turn.failed": (event) => void event.error,+ },+ });++ expect(isInstrumentationProvider(provider)).toBe(true);+ });++ it("exposes durable input request and resolution events", () => {+ const provider = defineInstrumentation({+ events: {+ "input.requested": (event) => void event.action.callId,+ "input.resolved": (event) => void event.outcome,+ },+ });++ expect(isInstrumentationProvider(provider)).toBe(true);+ });++ it("preserves the authored fields", () => {+ const setup = (): void => {};+ const declaration = defineInstrumentation({ setup });++ expect(declaration).toMatchObject({ setup });+ });+});++describe("isInstrumentationProvider", () => {+ it("rejects a value that never went through defineInstrumentation", () => {+ expect(isInstrumentationProvider({ events: {} })).toBe(false);+ });++ it.each([[null], [undefined], ["provider"], [42]])("rejects %p", (value) => {+ expect(isInstrumentationProvider(value)).toBe(false);+ });+});++describe("disableInstrumentation", () => {+ it("is recognizable as a disabled slot rather than a provider", () => {+ const disabled = disableInstrumentation();++ expect(isInstrumentationDisabled(disabled)).toBe(true);+ expect(isInstrumentationProvider(disabled)).toBe(false);+ });++ it("does not treat a provider as a disabled slot", () => {+ expect(isInstrumentationDisabled(defineInstrumentation({}))).toBe(false);+ });+});diff --git a/packages/eve/src/public/instrumentation/provider.ts b/packages/eve/src/public/instrumentation/provider.tsnew file mode 100644index 000000000..dac98d3a7--- /dev/null+++ b/packages/eve/src/public/instrumentation/provider.ts@@ -0,0 +1,189 @@+/**+ * The provider contract authored under `agent/instrumentation/`.+ *+ * Reachable only with `experimental.instrumentationProviders` on. With the flag+ * off nothing discovers that directory, so these types compile but never run.+ */++// Type-only, so nothing couples the public entrypoint to the harness at+// runtime. The event shapes are eve's own vocabulary; deriving the handler map+// from the union below is what keeps the public contract from drifting away+// from the bus that feeds it.+import type { InstrumentationEvent } from "#harness/instrumentation-lifecycle.js";+import type { JsonValue } from "#public/types/json.js";++export type { JsonValue } from "#public/types/json.js";++export type {+ InstrumentationActionCompletedEvent,+ InstrumentationActionFailedEvent,+ InstrumentationActionKind,+ InstrumentationActionOutcome,+ InstrumentationActionOutput,+ InstrumentationActionStartedEvent,+ InstrumentationAttemptScope,+ InstrumentationContentPart,+ InstrumentationEvent,+ InstrumentationInputKind,+ InstrumentationInputOption,+ InstrumentationInputOutcome,+ InstrumentationInputRequest,+ InstrumentationInputRequestedEvent,+ InstrumentationInputResolvedEvent,+ InstrumentationInputResponse,+ InstrumentationModelCallCompletedEvent,+ InstrumentationModelCallFailedEvent,+ InstrumentationModelCallStartedEvent,+ InstrumentationModelRef,+ InstrumentationOperationRef,+ InstrumentationParentLineage,+ InstrumentationSessionFailedEvent,+ InstrumentationSessionSettledEvent,+ InstrumentationSessionStartedEvent,+ InstrumentationSessionTransitionEvent,+ InstrumentationStepAttemptMetadataEvent,+ InstrumentationStepAttemptCompletedEvent,+ InstrumentationStepAttemptFailedEvent,+ InstrumentationStepAttemptStartedEvent,+ InstrumentationStepAttemptTerminalEvent,+ InstrumentationToolCallCompletedEvent,+ InstrumentationToolCallFailedEvent,+ InstrumentationToolCallStartedEvent,+ InstrumentationToolOutput,+ InstrumentationTraceContext,+ InstrumentationTurnFailedEvent,+ InstrumentationTurnSettledEvent,+ InstrumentationTurnStartedEvent,+ InstrumentationTurnTerminalEvent,+ InstrumentationUsage,+} from "#harness/instrumentation-lifecycle.js";++/**+ * Marks a value as having come from `defineInstrumentation` or a built-in+ * factory.+ *+ * It does not say which layout the value belongs to: a provider and a legacy+ * config both carry `events` and `setup`, so no value-level check separates+ * them. The layout decides — `agent/instrumentation.ts` is read as a config and+ * `agent/instrumentation/*.ts` as providers, and the two are mutually exclusive+ * builds. The brand's job is only to catch a default export that never went+ * through eve at all.+ */+export const PROVIDER = Symbol.for("eve.instrumentation.provider");++/** Marks a slot the author turned off rather than configured. */+export const DISABLED = Symbol.for("eve.instrumentation.disabled");++/** Where the agent is running when `setup` fires. */+export type InstrumentationEnvironment = "development" | "preview" | "production";++/** The local eval run this server was started to serve. */+export interface EvaluationRef {+ readonly runId: string;+}++/**+ * Passed to {@link InstrumentationProvider.setup} once at server startup,+ * before any event is published.+ */+export interface ProviderSetupContext {+ /** The agent name declared by `defineAgent`. */+ readonly agentName: string;+ /** Always supplied at runtime; optional for legacy setup-context compatibility. */+ readonly environment?: InstrumentationEnvironment;+ /** Present only when this server was started for a local `eve eval` run. */+ readonly evaluation?: EvaluationRef;+ /** The eve version running the agent. */+ /** Always supplied at runtime; optional for legacy setup-context compatibility. */+ readonly frameworkVersion?: string;+}++export interface ProviderState {+ get(): JsonValue | undefined;+ /** Stages a JSON value; `undefined` releases this operation's slot. */+ set(value: JsonValue | undefined): void;+}++export interface ProviderContext {+ readonly state: ProviderState;+}++/**+ * One event handler.+ *+ * A handler can carry durable JSON state from a start to its terminal through+ * `ctx.state`. eve scopes and releases that state by provider and operation.+ */+export type Handler<TEvent> = (event: TEvent, ctx: ProviderContext) => void | PromiseLike<void>;++type EventForType<TEvent, TType> = TEvent extends { readonly type: infer TEventType }+ ? TType extends TEventType+ ? TEvent & { readonly type: TType }+ : never+ : never;++/**+ * The events a provider may handle, one optional handler per event type.+ *+ * Derived from the event union rather than written out, so a new event reaches+ * providers the moment the bus can publish it.+ */+export type ProviderEvents = {+ readonly [TType in InstrumentationEvent["type"]]?: Handler<+ EventForType<InstrumentationEvent, TType>+ >;+};++/**+ * What an author writes for one file under `agent/instrumentation/`.+ *+ * Setup runs in slot order, but event handlers across authored providers run+ * concurrently and are failure-isolated. Do not coordinate providers through+ * completion order.+ */+export interface ProviderDefinition {+ readonly events?: ProviderEvents;+ /** Runs once at server startup, before any event is published. */+ readonly setup?: (context: ProviderSetupContext) => void | PromiseLike<void>;+ /** Drains anything buffered. eve calls this before a session goes idle. */+ readonly flush?: () => void | PromiseLike<void>;+ /** Releases resources when the process is going away. */+ readonly shutdown?: () => void | PromiseLike<void>;+}++/** A {@link ProviderDefinition} that has been through `defineInstrumentation`. */+export type InstrumentationProvider = ProviderDefinition & {+ readonly [PROVIDER]: true;+};++/** A slot the author turned off. eve registers nothing for it. */+export interface InstrumentationDisabled {+ readonly [DISABLED]: true;+}++/**+ * Turns off the slot the file it is exported from names.+ *+ * Export it as the default of `agent/instrumentation/local.ts` to stop eve+ * spooling local traces, for instance. Omitting the file entirely leaves eve's+ * default in place, which is why turning one off takes a value.+ */+export function disableInstrumentation(): InstrumentationDisabled {+ return { [DISABLED]: true };+}++export function isInstrumentationProvider(value: unknown): value is InstrumentationProvider {+ return (+ typeof value === "object" &&+ value !== null &&+ (value as Partial<InstrumentationProvider>)[PROVIDER] === true+ );+}++export function isInstrumentationDisabled(value: unknown): value is InstrumentationDisabled {+ return (+ typeof value === "object" &&+ value !== null &&+ (value as Partial<InstrumentationDisabled>)[DISABLED] === true+ );+}diff --git a/packages/eve/src/runtime/resolve-agent.ts b/packages/eve/src/runtime/resolve-agent.tsindex dae3eaf2e..d26449eb9 100644--- a/packages/eve/src/runtime/resolve-agent.ts+++ b/packages/eve/src/runtime/resolve-agent.ts@@ -204,6 +204,7 @@ function createResolvedAgentConfig(if (manifest.config.experimental !== undefined) {config.experimental = {+ instrumentationProviders: manifest.config.experimental.instrumentationProviders,subagentPersistentSessions: manifest.config.experimental.subagentPersistentSessions,workflow:manifest.config.experimental.workflow === undefineddiff --git a/packages/eve/src/shared/agent-definition.ts b/packages/eve/src/shared/agent-definition.tsindex a42a5c941..a403b7b4b 100644--- a/packages/eve/src/shared/agent-definition.ts+++ b/packages/eve/src/shared/agent-definition.ts@@ -191,6 +191,15 @@ export interface AgentLimitsDefinition {* These options are unstable and may change or be removed in any release.*/export interface AgentExperimentalDefinition {+ /**+ * Reads instrumentation from an `instrumentation/` directory of providers+ * rather than a single `agent/instrumentation.ts` config object.+ *+ * The two layouts are mutually exclusive: with this on, an+ * `agent/instrumentation.ts` is a build error, and with it off, an+ * `instrumentation/` directory is.+ */+ readonly instrumentationProviders?: boolean;/*** Keeps this agent's delegated subagent sessions alive after they answer.* The model can pass `agentId` to a subagent tool to continue a previousdiff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex e5611b1cc..2db1b4ad2 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -938,10 +938,10 @@ describe("createAgentOtelInstrumentation", () => {spanProcessors: [new SimpleSpanProcessor(exporter)],});const agentOtel = createAgentOtelInstrumentation({- recordInputs: false,- recordOutputs: false,frameworkVersion: "test",idGenerator,+ recordInputs: false,+ recordOutputs: false,stateStore: new InMemoryAgentTraceStateStore(),tracer: provider.getTracer("eve.agent"),});diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex 9ff4b7280..b7347a35d 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -58,9 +58,13 @@ interface AttemptSpanState {}export interface AgentOtelInstrumentationInput {- /** Whether any destination requested model prompts and tool inputs. */+ /**+ * Whether to write model prompts and tool call inputs onto spans at all.+ * This is the union across destinations, not one destination's policy: a+ * destination that declined drops these on its way out instead.+ */readonly recordInputs?: boolean;- /** Whether any destination requested model responses and tool outputs. */+ /** The same, for model responses and tool call outputs. */readonly recordOutputs?: boolean;readonly frameworkVersion: string;readonly idGenerator: AgentSpanIdGenerator;@@ -509,7 +513,6 @@ export function createAgentOtelInstrumentation(return {hook: {- name: "eve.otel",events: {"action.completed": actions.events["action.completed"],"action.failed": actions.events["action.failed"],@@ -535,6 +538,7 @@ export function createAgentOtelInstrumentation("turn.failed": onTurnTerminal,"turn.started": onTurnStarted,},+ name: "eve.otel",},runInContext(operation, execute) {const scope = attemptScopes.get(operation.scope.attemptId) ?? operation.scope;diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts b/packages/eve/src/tracing/install-instrumentation-runtime.test.tsindex 4d65b92a0..ea6824f8d 100644--- a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts+++ b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts@@ -1,10 +1,13 @@import { beforeEach, describe, expect, it, vi } from "vitest";+import { ContextContainer, contextStorage } from "#context/container.js";+import { turnIdempotencyKey } from "#harness/instrumentation-lifecycle.js";import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js";import { otelIntegration, collectOtelPipeline } from "#tracing/otel-declaration.js";-const { forceFlush, shutdown } = vi.hoisted(() => ({+const { forceFlush, internalTerminalState, shutdown } = vi.hoisted(() => ({forceFlush: vi.fn(async () => undefined),+ internalTerminalState: vi.fn(),shutdown: vi.fn(async () => undefined),}));@@ -25,7 +28,17 @@ vi.mock("#tracing/otel-registration.js", async (importOriginal) => {vi.mock("#tracing/agent-otel-provider.js", () => ({createAgentOtelInstrumentation: () => ({- hook: {},+ hook: {+ events: {+ "turn.completed": (_event: unknown, ctx: { state: { get(): unknown } }) => {+ internalTerminalState(ctx.state.get());+ },+ "turn.started": (_event: unknown, ctx: { state: { set(value: string): void } }) => {+ ctx.state.set("framework");+ },+ },+ name: "eve.otel",+ },runInContext: (_operation: unknown, execute: () => PromiseLike<unknown>) => execute(),}),}));@@ -35,6 +48,7 @@ const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime");describe("installInstrumentationRuntime", () => {beforeEach(() => {forceFlush.mockClear();+ internalTerminalState.mockClear();shutdown.mockClear();delete (globalThis as Record<symbol, unknown>)[RUNTIME_GLOBAL_KEY];});@@ -63,4 +77,44 @@ describe("installInstrumentationRuntime", () => {expect(shutdown).toHaveBeenCalledOnce();expect(providerShutdown).toHaveBeenCalledOnce();});++ it("isolates authored state from an internal provider with the same name", async () => {+ const authoredTerminalState = vi.fn();+ const runtime = installInstrumentationRuntime({+ collected: collectOtelPipeline([otelIntegration()]),+ frameworkVersion: "test",+ providers: [+ {+ events: {+ "turn.completed": (_event, ctx) => authoredTerminalState(ctx.state.get()),+ "turn.started": (_event, ctx) => ctx.state.set("authored"),+ },+ name: "eve.otel",+ stateNamespace: "authored:eve.otel",+ },+ ],+ serviceName: "weather",+ });+ const idempotencyKey = turnIdempotencyKey("session-1", "turn-1");++ await contextStorage.run(new ContextContainer(), async () => {+ await runtime.hooks.publish({+ idempotencyKey,+ rootSessionId: "session-1",+ sequence: 0,+ sessionId: "session-1",+ turnId: "turn-1",+ type: "turn.started",+ });+ await runtime.hooks.publish({+ idempotencyKey,+ sessionId: "session-1",+ turnId: "turn-1",+ type: "turn.completed",+ });+ });++ expect(internalTerminalState).toHaveBeenCalledExactlyOnceWith("framework");+ expect(authoredTerminalState).toHaveBeenCalledExactlyOnceWith("authored");+ });});diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.ts b/packages/eve/src/tracing/install-instrumentation-runtime.tsindex a0dd60bbe..a9acad5d3 100644--- a/packages/eve/src/tracing/install-instrumentation-runtime.ts+++ b/packages/eve/src/tracing/install-instrumentation-runtime.ts@@ -18,14 +18,24 @@ import { registerOtelPipeline, type RegisteredOtelPipeline } from "#tracing/otelconst log = createLogger("tracing.install-instrumentation-runtime");-/** Installs the bus and the one OpenTelemetry pipeline collected for this process. */+/**+ * Installs the process instrumentation runtime around a collected pipeline.+ *+ * Both layouts land here. `eve dev`'s zero-config default and an authored+ * `agent/instrumentation/` directory differ only in where the declared values+ * came from, so sharing the install keeps them on one runtime path.+ *+ * A directory that declared no OpenTelemetry still gets a bus: its providers+ * see every event, they just have no spans to hang them on.+ */export function installInstrumentationRuntime(input: {readonly collected: CollectedOtel;readonly frameworkVersion: string;readonly providers: readonly InstrumentationProviderDefinition[];readonly serviceName: string;}): InstrumentationRuntime {- const providers: InstrumentationProviderDefinition[] = [...input.providers];+ const serialBefore: InstrumentationProviderDefinition[] = [];+ const serialAfter: InstrumentationProviderDefinition[] = [];let otelRuntime: RegisteredOtelPipeline | undefined;let runInContext: InstrumentationRuntime["runInContext"] = (_operation, execute) => execute();@@ -42,29 +52,35 @@ export function installInstrumentationRuntime(input: {stateStore: new ContextAgentTraceStateStore(),tracer: trace.getTracer("eve.agent", input.frameworkVersion),});- providers.unshift(agentOtel.hook);+ // The span must exist before authored providers observe the lifecycle event.+ serialBefore.push({ ...agentOtel.hook, stateNamespace: "internal:otel" });runInContext = agentOtel.runInContext;const releasable = input.collected.pipeline.spanProcessors.filter(isSpanProcessor).filter(hasSessionRelease);- if (releasable.length > 0) providers.push(sessionReleaseProvider(releasable));+ if (releasable.length > 0) serialAfter.push(sessionReleaseProvider(releasable));}+ const allProviders = [...serialBefore, ...input.providers, ...serialAfter];let shutdown: Promise<void> | undefined;return registerInstrumentationRuntime({forceFlush: () =>settleAll([...(otelRuntime === undefined ? [] : [otelRuntime.forceFlush]),- ...providers.map((provider) => () => provider.flush?.()),+ ...allProviders.map((provider) => () => provider.flush?.()),]),- hooks: createInstrumentationHooks(providers),+ hooks: createInstrumentationHooks({+ parallel: input.providers,+ serialAfter,+ serialBefore,+ }),otelSettings: input.collected.declared ? input.collected.settings : undefined,runInContext,shutdown: () => {shutdown ??= settleAll([...(otelRuntime === undefined ? [] : [otelRuntime.shutdown]),- ...providers.map((provider) => () => provider.shutdown?.()),+ ...allProviders.map((provider) => () => provider.shutdown?.()),]);return shutdown;},@@ -84,6 +100,7 @@ function sessionReleaseProvider(return {events: { "session.completed": release, "session.failed": release },name: "eve.session-release",+ stateNamespace: "internal:session-release",};}diff --git a/packages/eve/src/tracing/local-traces.test.ts b/packages/eve/src/tracing/local-traces.test.tsindex 2128084f4..fb8e1b50a 100644--- a/packages/eve/src/tracing/local-traces.test.ts+++ b/packages/eve/src/tracing/local-traces.test.ts@@ -1,6 +1,7 @@import { afterEach, describe, expect, it, vi } from "vitest";-import { createLocalTracesProcessor, resolveLocalTracesContent } from "#tracing/local-traces.js";+import { createLocalTracesProcessor } from "#tracing/local-traces.js";+import { localTraces } from "#public/instrumentation/otel.js";vi.mock("#tracing/local-trace-span-processor.js", () => ({LocalTraceSpanProcessor: class {@@ -54,23 +55,11 @@ describe("createLocalTracesProcessor", () => {it("is inert outside a development worker", async () => {vi.stubEnv("EVE_DEV_WORKER_APP_ROOT", undefined);- const processor = createLocalTracesProcessor();+ const [processor] = localTraces().spanProcessors;+ if (processor === undefined || processor === "auto") throw new Error("Expected a processor.");expect(() => processor.onEnd(agentSpan("session-one", "a".repeat(32)))).not.toThrow();await expect(processor.forceFlush()).resolves.toBeUndefined();await expect(processor.shutdown()).resolves.toBeUndefined();});-- it("lets the environment override narrow only this destination", () => {- vi.stubEnv("EVE_TRACES_CONTENT", "off");- expect(resolveLocalTracesContent()).toEqual({ recordInputs: false, recordOutputs: false });- });-- it("preserves explicit destination narrowing when content is enabled", () => {- vi.stubEnv("EVE_TRACES_CONTENT", "on");- expect(resolveLocalTracesContent({ recordInputs: false })).toEqual({- recordInputs: false,- recordOutputs: true,- });- });});diff --git a/packages/eve/src/tracing/local-traces.ts b/packages/eve/src/tracing/local-traces.tsindex 8500f7d1e..83ca4591f 100644--- a/packages/eve/src/tracing/local-traces.ts+++ b/packages/eve/src/tracing/local-traces.ts@@ -23,7 +23,15 @@ export interface LocalTracesProcessor extends SpanProcessor {releaseSession(sessionId: string): Promise<boolean>;}-/** Whether a processor still exposes the local spool's session lifecycle. */+/**+ * Reports whether a processor tracks which session owns which trace, so eve can+ * tell it when that session is done.+ *+ * Anything standing between eve and the spool has to answer for the spool, so+ * this is the check a wrapper uses to decide whether it must forward the call.+ *+ * @internal+ */export function hasSessionRelease(processor: SpanProcessor): processor is LocalTracesProcessor {return typeof (processor as Partial<LocalTracesProcessor>).releaseSession === "function";}@@ -35,7 +43,8 @@ export function hasSessionRelease(processor: SpanProcessor): processor is LocalT* to observe spans to track which session owns which trace.** Internal because of `releaseSession`, which eve's runtime drives off session- * lifecycle.+ * lifecycle. The authored surface is `localTraces()`, which wraps this in an+ * `OtelIntegration`.*/export function createLocalTracesProcessor(input: { readonly appRoot?: string } = {},@@ -80,7 +89,16 @@ export function createLocalTracesProcessor(};}-/** Intersects the local destination policy with `EVE_TRACES_CONTENT`. */+/**+ * The local spool's content policy: its options, intersected with+ * `EVE_TRACES_CONTENT`.+ *+ * The variable used to be the process-wide switch. It now applies to this one+ * destination, and only ever narrows — `off` still wins where it applies, but+ * it no longer decides what a hosted backend beside it receives.+ *+ * @internal+ */export function resolveLocalTracesContent(options: {readonly recordInputs?: boolean;diff --git a/packages/eve/src/tracing/otel-declaration.test.ts b/packages/eve/src/tracing/otel-declaration.test.tsindex 0487cbcdf..09a8eed6d 100644--- a/packages/eve/src/tracing/otel-declaration.test.ts+++ b/packages/eve/src/tracing/otel-declaration.test.ts@@ -61,7 +61,9 @@ describe("otelIntegration", () => {expect(otelIntegration().content).toStrictEqual({ recordInputs: true, recordOutputs: true });});- it("puts a declined policy in front of every processor", () => {+ // An author's own processor is part of this destination, and the point of+ // declining is that nothing under it sees what was said.+ it("puts a declined policy in front of every processor, an author's included", () => {const first = processor();const integration = otelIntegration({ recordOutputs: false, spanProcessors: [first] });@@ -135,12 +137,16 @@ describe("collectOtelPipeline", () => {});expect(collected.settings).toStrictEqual({functionId: "weather",+ // Nothing declared a destination, so nothing asked for content.recordInputs: false,recordOutputs: false,traceChannelRequests: true,});});+ // Content governs what is written onto the span, which is upstream of every+ // destination — so one that wants it is enough, and the ones that declined+ // drop it on their own way out.it("takes content capture as the union across destinations", () => {const collected = collectOtelPipeline([otelIntegration({ recordInputs: false, recordOutputs: false }),diff --git a/packages/eve/src/tracing/otel-declaration.ts b/packages/eve/src/tracing/otel-declaration.tsindex b603e64cd..abfd25614 100644--- a/packages/eve/src/tracing/otel-declaration.ts+++ b/packages/eve/src/tracing/otel-declaration.ts@@ -6,6 +6,7 @@ import type {SpanProcessorOrName,} from "#compiled/@vercel/otel/index.js";+import { PROVIDER, type InstrumentationProvider } from "#public/instrumentation/provider.js";import { batchSpanProcessor } from "#tracing/batch-span-processor.js";import type { ResolvedContentOptions } from "#tracing/content-attributes.js";import { contentFilteringProcessor } from "#tracing/content-span-processor.js";@@ -50,7 +51,15 @@ export interface OtelOptions {readonly propagators?: readonly PropagatorOrName[];}-/** What one destination records of the conversation itself. */+/**+ * What one destination records of the conversation itself.+ *+ * Declining is per destination, not per process: content is written onto the+ * span if any destination wants it, and one that declined never exports it. So+ * an agent whose every destination declines still never materializes a prompt —+ * the union of nothing is nothing — but a local spool and a hosted backend no+ * longer have to agree.+ */export interface ContentOptions {/** Record model prompts and tool call inputs. Defaults to `true`. */readonly recordInputs?: boolean;@@ -73,14 +82,15 @@ const OTEL_INTEGRATION = Symbol.for("eve.instrumentation.otel-integration");* The declared OpenTelemetry pipeline settings. eve collects this before* building the tracer provider, so it is a value rather than a side effect.*/-export interface OtelDeclaration {+export interface OtelDeclaration extends InstrumentationProvider {readonly [OTEL_DECLARATION]: true;readonly options: OtelOptions;}/** One declared destination. A process may have as many as it has files. */-export interface OtelIntegration {+export interface OtelIntegration extends InstrumentationProvider {readonly [OTEL_INTEGRATION]: true;+ /** Resolved from `ContentOptions`, so the union does not re-apply defaults. */readonly content: ResolvedContentOptions;readonly spanProcessors: readonly SpanProcessorOrName[];}@@ -88,10 +98,12 @@ export interface OtelIntegration {/*** Declares the process-wide OpenTelemetry settings.*- * This remains internal until the provider authoring API exposes it.+ * Export it from `agent/instrumentation/otel.ts`. Omitting the file is the+ * common case: eve registers the pipeline for whatever destinations are+ * declared beside it, and this only names what those destinations share.*/export function otel(options: OtelOptions = {}): OtelDeclaration {- return { [OTEL_DECLARATION]: true, options };+ return { [OTEL_DECLARATION]: true, [PROVIDER]: true, options };}/**@@ -100,16 +112,25 @@ export function otel(options: OtelOptions = {}): OtelDeclaration {* A `traceExporter` is wrapped in eve's batching processor, which is what makes* the one-line form of a hosted backend enough. Pass `spanProcessors` instead* when the destination needs its own batching, sampling, or filtering.+ *+ * Declining content wraps every processor here, an author's included: they are+ * this destination, and the point of declining is that nothing under it sees+ * what was said.*/export function otelIntegration(options: OtelIntegrationOptions = {}): OtelIntegration {- const content = resolveContentOptions(options);+ const content: ResolvedContentOptions = {+ recordInputs: options.recordInputs !== false,+ recordOutputs: options.recordOutputs !== false,+ };const declared = options.spanProcessors ?? [];const spanProcessors =options.traceExporter === undefined? declared: [...declared, batchSpanProcessor(options.traceExporter)];+return {[OTEL_INTEGRATION]: true,+ [PROVIDER]: true,content,spanProcessors:content.recordInputs && content.recordOutputs@@ -123,6 +144,7 @@ export function agentRunsIntegration(options: ContentOptions = {}): OtelIntegratconst content = resolveContentOptions(options);return {[OTEL_INTEGRATION]: true,+ [PROVIDER]: true,content,spanProcessors:content.recordInputs && content.recordOutputs@@ -167,8 +189,9 @@ export interface OtelHarnessSettings {readonly functionId?: string;readonly traceChannelRequests: boolean;/**- * What to materialize on spans at all. Each destination independently strips- * anything it declined before export.+ * What to write onto a span at all, as opposed to what any one destination+ * exports. `agent/instrumentation.ts` sets this directly; a provider+ * directory arrives at it as the union of its destinations.*/readonly recordInputs?: boolean;readonly recordOutputs?: boolean;@@ -193,6 +216,10 @@ export interface CollectedOtel {* happened to visit first. With one declaration per file that collision needs* two files both exporting `otel()`, which is the only way to reach it.*+ * Content capture is the union of what the destinations asked for, because it+ * governs what is written rather than what is exported. Each destination's own+ * processors already drop what it declined.+ ** @internal*/export function collectOtelPipeline(values: readonly unknown[]): CollectedOtel {diff --git a/packages/eve/src/tracing/otel-registration.scenario.test.ts b/packages/eve/src/tracing/otel-registration.scenario.test.tsindex 85d5e3d6f..c9b70a089 100644--- a/packages/eve/src/tracing/otel-registration.scenario.test.ts+++ b/packages/eve/src/tracing/otel-registration.scenario.test.ts@@ -82,7 +82,8 @@ describe("registerOtelPipeline", () => {it("does not export the private registration span", async () => {const exporter = new InMemorySpanExporter();const processor = new SimpleSpanProcessor(exporter);- registerOtelPipeline({+ const shutdown = vi.spyOn(processor, "shutdown");+ const runtime = registerOtelPipeline({pipeline: { spanProcessors: [processor] },serviceName: "weather",});@@ -91,7 +92,8 @@ describe("registerOtelPipeline", () => {await processor.forceFlush();expect(exporter.getFinishedSpans().map((span) => span.name)).toEqual(["user.work"]);- await processor.shutdown();+ await runtime.shutdown();+ expect(shutdown).toHaveBeenCalledOnce();});});diff --git a/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts b/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.tsindex d4cc234f3..065930565 100644--- a/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts+++ b/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts@@ -20,6 +20,10 @@ const createAppRoot = useTemporaryAppRoots();describe("writeCompiledArtifactsFiles", () => {afterEach(() => {delete (globalThis as Record<string, unknown>).__eveInstrumentationLoaded;+ delete (globalThis as Record<string, unknown>).__eveProviderSetups;+ delete (globalThis as Record<symbol, unknown>)[+ Symbol.for("eve.harness-instrumentation-providers")+ ];});it("installs compile metadata into bundled compiled artifacts", async () => {@@ -112,6 +116,136 @@ describe("writeCompiledArtifactsFiles", () => {expect(instrumentationPluginModule.default()).toBeUndefined();});+ it("registers one provider per file when the instrumentationProviders flag is on", async () => {+ const { agentRoot, appRoot } = await createAppRoot("eve-compiled-artifacts-providers-", {+ packageName: "compiled-artifacts-providers-test-agent",+ });+ const outDir = join(appRoot, ".workflow-build");+ const definePath = resolvePackageSourceFilePath(+ "src/public/instrumentation/index.ts",+ ).replaceAll("\\", "/");++ await writeFile(+ join(agentRoot, "agent.ts"),+ [+ "export default {",+ ' model: "openai/gpt-5.4",',+ " experimental: { instrumentationProviders: true },",+ "};",+ "",+ ].join("\n"),+ );+ await writeFile(join(agentRoot, "instructions.md"), "You are a precise assistant.\n");+ await mkdir(join(agentRoot, "instrumentation"), { recursive: true });+ for (const slot of ["local", "otel"]) {+ await writeFile(+ join(agentRoot, "instrumentation", `${slot}.ts`),+ [+ `import { defineInstrumentation } from ${JSON.stringify(definePath)};`,+ "",+ "const container = globalThis as Record<string, unknown>;",+ "",+ "export default defineInstrumentation({",+ " setup(context) {",+ " container.__eveProviderSetups ??= [];",+ ` (container.__eveProviderSetups as string[]).push(\`${slot}:\${context.agentName}\`);`,+ " },",+ "});",+ "",+ ].join("\n"),+ );+ }++ const compileResult = await compileAgent({ startPath: appRoot });+ const generatedArtifacts = await writeCompiledArtifactsFiles({+ compileResult,+ defaultWorkflowWorld: "local",+ outDir,+ });+ const instrumentationPluginPath = generatedArtifacts.instrumentationPluginPath;++ if (instrumentationPluginPath === undefined) {+ throw new Error("Expected instrumentation plugin path to be generated.");+ }++ expect(generatedArtifacts.instrumentationSourcePaths).toEqual([+ join(agentRoot, "instrumentation", "local.ts"),+ join(agentRoot, "instrumentation", "otel.ts"),+ ]);++ const instrumentationPluginSource = await readFile(instrumentationPluginPath, "utf8");++ expect(instrumentationPluginSource).toContain('slot: "local"');+ expect(instrumentationPluginSource).toContain('slot: "otel"');+ expect(instrumentationPluginSource).toContain("seedInstrumentationProviders();");+ expect(instrumentationPluginSource).toContain("shutdownInstrumentationProviders");+ expect(instrumentationPluginSource).toContain("hooks?.hook('close'");+ expect(instrumentationPluginSource).not.toContain("registerInstrumentationConfig");++ const instrumentationPlugin = (await import(pathToFileURL(instrumentationPluginPath).href)) as {+ default: (nitroApp: {+ hooks: { hook(name: "close", handler: () => Promise<void>): void };+ }) => void;+ };+ const closeHandlers: Array<() => Promise<void>> = [];+ instrumentationPlugin.default({+ hooks: {+ hook: (_name, handler) => closeHandlers.push(handler),+ },+ });++ // The plugin resolves the registry by absolute path while the assertion+ // resolves it by package alias, so this also proves the globalThis rooting+ // survives two module instances.+ const { getInstrumentationProviders } =+ await import("../../src/harness/instrumentation-providers.js");++ expect((globalThis as Record<string, unknown>).__eveProviderSetups).toEqual([+ "local:compiled-artifacts-providers-test-agent",+ "otel:compiled-artifacts-providers-test-agent",+ ]);+ expect(getInstrumentationProviders().map((entry) => entry.slot)).toEqual(["local", "otel"]);+ expect(closeHandlers).toHaveLength(1);+ await closeHandlers[0]?.();+ });++ it("generates the provider plugin for built-in destinations without authored files", async () => {+ const { agentRoot, appRoot } = await createAppRoot(+ "eve-compiled-artifacts-default-providers-",+ {+ packageName: "compiled-artifacts-default-providers-test-agent",+ },+ );+ const outDir = join(appRoot, ".workflow-build");+ await writeFile(+ join(agentRoot, "agent.ts"),+ [+ "export default {",+ ' model: "openai/gpt-5.4",',+ " experimental: { instrumentationProviders: true },",+ "};",+ "",+ ].join("\n"),+ );+ await writeFile(join(agentRoot, "instructions.md"), "You are a precise assistant.\n");++ const compileResult = await compileAgent({ startPath: appRoot });+ const generatedArtifacts = await writeCompiledArtifactsFiles({+ compileResult,+ defaultWorkflowWorld: "local",+ outDir,+ });+ const instrumentationPluginPath = generatedArtifacts.instrumentationPluginPath;+ if (instrumentationPluginPath === undefined) {+ throw new Error("Expected instrumentation plugin path to be generated.");+ }++ expect(generatedArtifacts.instrumentationSourcePaths).toEqual([]);+ expect(await readFile(instrumentationPluginPath, "utf8")).toContain(+ "seedInstrumentationProviders();",+ );+ });+it("surfaces instrumentation import failures when the Nitro plugin module loads", async () => {const { agentRoot, appRoot } = await createAppRoot("eve-compiled-artifacts-instrumentation-error-",diff --git a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts b/packages/eve/test/scenarios/eval-command-environment.scenario.test.tsindex 875a3c419..d6dcd70c0 100644--- a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts+++ b/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts@@ -44,6 +44,8 @@ const DEVELOPMENT_ENV_KEYS = ["EVE_DEV_LOCAL_ONLY","EVE_DEV_SHARED","EVE_DEV_SHELL_ONLY",+ "EVE_EVALUATION",+ "EVE_EVALUATION_RUN_ID",] as const;async function createEnvironmentFixture(): Promise<string> {@@ -192,6 +194,8 @@ describe("eve eval environment loading", () => {expect(close).toHaveBeenCalledTimes(1);expect(handle.shutdown).toHaveBeenCalledTimes(1);+ expect(process.env.EVE_EVALUATION).toBe("1");+ expect(process.env.EVE_EVALUATION_RUN_ID).toMatch(/^[0-9a-f-]{36}$/u);expect(exit).toHaveBeenCalledWith(0);});