Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(eve): add the instrumentation provider layout (#1863)

Chad Hietalac90a459104e3
diff --git a/.changeset/instrumentation-provider-layout.md b/.changeset/instrumentation-provider-layout.md
new file mode 100644
index 000000000..1e40d0691
--- /dev/null
+++ b/.changeset/instrumentation-provider-layout.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Add the experimental `agent/instrumentation/` provider layout with durable lifecycle handlers, including user input boundaries and action settlement-time, outcome, error-code, and usage metadata, final setup context, reserved OpenTelemetry destinations, and coordinated flush and shutdown. OpenTelemetry singleton settings and destinations are exposed through `eve/instrumentation/otel`, and eve's AI SDK bridge composes with registered integrations.
diff --git a/packages/eve/package.json b/packages/eve/package.json
index 5346416a2..2cf228c5b 100644
--- a/packages/eve/package.json
+++ b/packages/eve/package.json
@@ -206,6 +206,11 @@
"import": "./dist/src/public/instrumentation/index.js",
"default": "./dist/src/public/instrumentation/index.js"
},
+ "./instrumentation/otel": {
+ "types": "./dist/src/public/instrumentation/otel.d.ts",
+ "import": "./dist/src/public/instrumentation/otel.js",
+ "default": "./dist/src/public/instrumentation/otel.js"
+ },
"./schedules": {
"types": "./dist/src/public/schedules/index.d.ts",
"import": "./dist/src/public/schedules/index.js",
diff --git a/packages/eve/src/compiler/manifest.ts b/packages/eve/src/compiler/manifest.ts
index fd5a9f432..21926310b 100644
--- a/packages/eve/src/compiler/manifest.ts
+++ b/packages/eve/src/compiler/manifest.ts
@@ -439,6 +439,7 @@ const compiledAgentConfigBaseFields = {
description: z.string().optional(),
experimental: z
.object({
+ instrumentationProviders: z.boolean().optional(),
subagentPersistentSessions: z.boolean().optional(),
workflow: compiledAgentWorkflowDefinitionSchema.optional(),
})
@@ -917,6 +918,7 @@ function cloneCompiledAgentDefinition(config: CompiledAgentDefinition): Compiled
config.experimental === undefined
? undefined
: {
+ instrumentationProviders: config.experimental.instrumentationProviders,
subagentPersistentSessions: config.experimental.subagentPersistentSessions,
workflow:
config.experimental.workflow === undefined
diff --git a/packages/eve/src/compiler/normalize-agent-config.ts b/packages/eve/src/compiler/normalize-agent-config.ts
index 8faf3dfb5..0c0bc81b7 100644
--- a/packages/eve/src/compiler/normalize-agent-config.ts
+++ b/packages/eve/src/compiler/normalize-agent-config.ts
@@ -185,6 +185,10 @@ function normalizeExperimentalDefinition(
const compiledExperimental: Mutable<NonNullable<CompiledAgentDefinition["experimental"]>> = {};
+ if (experimental.instrumentationProviders !== undefined) {
+ compiledExperimental.instrumentationProviders = experimental.instrumentationProviders;
+ }
+
if (experimental.subagentPersistentSessions !== undefined) {
compiledExperimental.subagentPersistentSessions = experimental.subagentPersistentSessions;
}
diff --git a/packages/eve/src/discover/agent.integration.test.ts b/packages/eve/src/discover/agent.integration.test.ts
index fb726983c..39b6daf10 100644
--- a/packages/eve/src/discover/agent.integration.test.ts
+++ b/packages/eve/src/discover/agent.integration.test.ts
@@ -511,6 +511,23 @@ describe("discoverAgent (memory)", () => {
]);
});
+ it("recognizes the instrumentation provider directory", async () => {
+ const project = buildMemoryAgentProject({
+ agentDirectories: ["instrumentation"],
+ agentFiles: {
+ "instructions.md": "You are a precise assistant.",
+ },
+ });
+
+ const result = await discoverAgent({
+ agentRoot: project.agentRoot,
+ appRoot: project.appRoot,
+ source: project.source,
+ });
+
+ expect(result.diagnostics).toEqual([]);
+ });
+
it("rejects authored tool filenames that violate the tool-name charset", async () => {
const project = buildMemoryAgentProject({
agentFiles: {
diff --git a/packages/eve/src/discover/filesystem.ts b/packages/eve/src/discover/filesystem.ts
index 814233d3c..2e96b3d04 100644
--- a/packages/eve/src/discover/filesystem.ts
+++ b/packages/eve/src/discover/filesystem.ts
@@ -44,6 +44,7 @@ export type AgentRootEntryKind =
| "extensions-directory"
| "hooks-directory"
| "ignored-directory"
+ | "instrumentation-directory"
| "instructions-directory"
| "instructions-markdown"
| "instructions-module"
@@ -179,6 +180,10 @@ export function classifyAgentRootEntry(
return "instructions-directory";
}
+ if (name === "instrumentation") {
+ return "instrumentation-directory";
+ }
+
if (name === "lib") {
return "lib-directory";
}
diff --git a/packages/eve/src/evals/cli/eval.ts b/packages/eve/src/evals/cli/eval.ts
index 26a709075..2ed4991ac 100644
--- a/packages/eve/src/evals/cli/eval.ts
+++ b/packages/eve/src/evals/cli/eval.ts
@@ -1,8 +1,13 @@
+import { randomUUID } from "node:crypto";
import { readFile } from "node:fs/promises";
import { basename, join } from "node:path";
import { loadDevelopmentEnvironmentFiles } from "#cli/dev/environment.js";
import { shutdownActiveSandboxHandles } from "#execution/sandbox/active-handles.js";
+import {
+ EVE_EVALUATION_ENV_FLAG,
+ EVE_EVALUATION_RUN_ID_ENV,
+} from "#internal/application/dev-environment.js";
import { resolveApplicationRoot } from "#internal/application/paths.js";
import { createDevelopmentServer, type DevelopmentServer } from "#internal/nitro/host.js";
import { createEvalClient } from "#evals/cli/eval-client.js";
@@ -141,6 +146,10 @@ export async function runEvalCommand(
url: options.url,
});
} else {
+ // Set before the server boots, because a provider's `setup` reads it
+ // once at startup and never again.
+ process.env[EVE_EVALUATION_ENV_FLAG] = "1";
+ process.env[EVE_EVALUATION_RUN_ID_ENV] = randomUUID();
devServer = createDevelopmentServer(appRoot, { host: "127.0.0.1", port: 0 });
const started = await devServer.start();
client = await createEvalClient({ kind: "local", url: started.url });
diff --git a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts b/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts
index 0a51032d6..f0fbea575 100644
--- a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts
+++ b/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts
@@ -94,7 +94,10 @@ describe("createAiSdkHookBridge", () => {
it("passes the identity captured at model-call start to the context runner", async () => {
const ids: string[] = [];
const hooks = createInstrumentationHooks([
- { events: { "model.call.started": (event) => void ids.push(event.idempotencyKey) } },
+ {
+ events: { "model.call.started": (event) => void ids.push(event.idempotencyKey) },
+ name: "identity",
+ },
]);
const bridge = createAiSdkHookBridge(scope, hooks, (operation, execute) => {
ids.push(operation.idempotencyKey);
@@ -132,7 +135,10 @@ describe("createAiSdkHookBridge", () => {
it("derives replay-stable model identity without the AI SDK call ID", async () => {
const keys: string[] = [];
const hooks = createInstrumentationHooks([
- { events: { "model.call.started": (event) => void keys.push(event.idempotencyKey) } },
+ {
+ events: { "model.call.started": (event) => void keys.push(event.idempotencyKey) },
+ name: "identity",
+ },
]);
for (const callId of ["sdk-random-1", "sdk-random-2"]) {
@@ -155,6 +161,7 @@ describe("createAiSdkHookBridge", () => {
events.push(event);
},
},
+ name: "metadata",
},
]);
const bridge = createAiSdkHookBridge(scope, hooks);
@@ -183,11 +190,13 @@ describe("createAiSdkHookBridge", () => {
throw new Error("provider failed");
},
},
+ name: "failing",
},
{
events: {
"model.call.completed": after,
},
+ name: "observer",
},
]);
const bridge = createAiSdkHookBridge(scope, hooks);
@@ -211,7 +220,9 @@ describe("createAiSdkHookBridge", () => {
it("terminalizes started operations when the attempt errors", async () => {
const after = vi.fn();
- const hooks = createInstrumentationHooks([{ events: { "model.call.failed": after } }]);
+ const hooks = createInstrumentationHooks([
+ { events: { "model.call.failed": after }, name: "terminal" },
+ ]);
const bridge = createAiSdkHookBridge(scope, hooks);
await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [
@@ -252,8 +263,8 @@ describe("createAiSdkHookBridge", () => {
});
const started = vi.fn();
const hooks = createInstrumentationHooks([
- { events: { "step.attempt.started": mutator } },
- { events: { "step.attempt.started": started } },
+ { events: { "step.attempt.started": mutator }, name: "mutator" },
+ { events: { "step.attempt.started": started }, name: "observer" },
]);
const bridge = createAiSdkHookBridge(scope, hooks);
@@ -288,7 +299,10 @@ describe("createAiSdkHookBridge", () => {
expect(Object.isFrozen(event.usage.inputTokenDetails)).toBe(true);
});
const hooks = createInstrumentationHooks([
- { events: { "model.call.completed": after, "model.call.started": before } },
+ {
+ events: { "model.call.completed": after, "model.call.started": before },
+ name: "model",
+ },
]);
const bridge = createAiSdkHookBridge(scope, hooks);
@@ -404,7 +418,10 @@ describe("createAiSdkHookBridge", () => {
expect(Object.isFrozen(event.output)).toBe(true);
});
const hooks = createInstrumentationHooks([
- { events: { "tool.call.completed": after, "tool.call.started": before } },
+ {
+ events: { "tool.call.completed": after, "tool.call.started": before },
+ name: "tool",
+ },
]);
const bridge = createAiSdkHookBridge(scope, hooks);
const toolCall = { input: { q: "eve" }, toolCallId: "tool-1", toolName: "search" };
@@ -478,7 +495,9 @@ describe("createAiSdkHookBridge", () => {
it("skips a terminal handler when the operation never started", async () => {
const completed = vi.fn();
- const hooks = createInstrumentationHooks([{ events: { "model.call.completed": completed } }]);
+ const hooks = createInstrumentationHooks([
+ { events: { "model.call.completed": completed }, name: "terminal" },
+ ]);
const bridge = createAiSdkHookBridge(scope, hooks);
// No onLanguageModelCallStart, so the bridge holds no id and publishes
@@ -516,6 +535,7 @@ describe("createAiSdkHookBridge", () => {
terminalStates.set(event.idempotencyKey, started.get(event.idempotencyKey));
},
},
+ name: "parallel",
},
]);
const bridge = createAiSdkHookBridge(scope, hooks);
diff --git a/packages/eve/src/harness/ai-sdk-telemetry.test.ts b/packages/eve/src/harness/ai-sdk-telemetry.test.ts
new file mode 100644
index 000000000..6049c90ca
--- /dev/null
+++ b/packages/eve/src/harness/ai-sdk-telemetry.test.ts
@@ -0,0 +1,25 @@
+import type { Telemetry } from "ai";
+import { afterEach, describe, expect, it } from "vitest";
+
+import { getRegisteredTelemetryIntegrations } from "#harness/ai-sdk-telemetry.js";
+
+describe("getRegisteredTelemetryIntegrations", () => {
+ const original = globalThis.AI_SDK_TELEMETRY_INTEGRATIONS;
+
+ afterEach(() => {
+ globalThis.AI_SDK_TELEMETRY_INTEGRATIONS = original;
+ });
+
+ it("is empty when nothing has registered", () => {
+ globalThis.AI_SDK_TELEMETRY_INTEGRATIONS = undefined;
+ expect(getRegisteredTelemetryIntegrations()).toEqual([]);
+ });
+
+ it("reports the integrations in registration order", () => {
+ const first: Telemetry = { onStart() {} };
+ const second: Telemetry = { onStart() {} };
+ globalThis.AI_SDK_TELEMETRY_INTEGRATIONS = [first, second];
+
+ expect(getRegisteredTelemetryIntegrations()).toEqual([first, second]);
+ });
+});
diff --git a/packages/eve/src/harness/otel-integration.ts b/packages/eve/src/harness/ai-sdk-telemetry.ts
similarity index 50%
rename from packages/eve/src/harness/otel-integration.ts
rename to packages/eve/src/harness/ai-sdk-telemetry.ts
index d34108a79..02954907f 100644
--- a/packages/eve/src/harness/otel-integration.ts
+++ b/packages/eve/src/harness/ai-sdk-telemetry.ts
@@ -16,10 +16,17 @@ export function ensureOtelIntegration(): void {
return;
}
registered = true;
- registerTelemetry(createOtelIntegration());
+ registerTelemetry(new OpenTelemetry({ runtimeContext: true }));
}
-/** Creates the existing OTel integration for explicit per-call composition. */
-export function createOtelIntegration(): Telemetry {
- return new OpenTelemetry({ runtimeContext: true });
+/**
+ * Every integration currently registered with the AI SDK — eve's own, plus any
+ * an authored instrumentation module added with `registerTelemetry`.
+ *
+ * A per-call `integrations` list replaces the registered ones rather than
+ * adding to them, so anything that passes integrations per call has to carry
+ * these forward or they stop receiving events.
+ */
+export function getRegisteredTelemetryIntegrations(): readonly Telemetry[] {
+ return globalThis.AI_SDK_TELEMETRY_INTEGRATIONS ?? [];
}
diff --git a/packages/eve/src/harness/instrumentation-config.test.ts b/packages/eve/src/harness/instrumentation-config.test.ts
index ef907d37a..fc027202c 100644
--- a/packages/eve/src/harness/instrumentation-config.test.ts
+++ b/packages/eve/src/harness/instrumentation-config.test.ts
@@ -1,5 +1,7 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { InstrumentationSetupContext } from "#public/instrumentation/index.js";
+
const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime");
beforeEach(() => {
@@ -31,7 +33,7 @@ describe("instrumentation-config chunk-isolation regression", () => {
vi.resetModules();
const moduleA = await import("#harness/instrumentation-config.js");
const config = { functionId: "test.instrumentation.cross-module.alice" };
- moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" });
+ await moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" });
vi.resetModules();
const moduleB = await import("#harness/instrumentation-config.js");
@@ -46,7 +48,7 @@ describe("instrumentation-config chunk-isolation regression", () => {
const { registerInstrumentationConfig } = await import("#harness/instrumentation-config.js");
const canary = { functionId: "test.instrumentation.global-mount.canary" };
- registerInstrumentationConfig(canary, { agentName: "test-agent" });
+ await registerInstrumentationConfig(canary, { agentName: "test-agent" });
expect((globalThis as Record<symbol, unknown>)[globalKey]).toBe(canary);
});
@@ -57,7 +59,7 @@ describe("instrumentation-config chunk-isolation regression", () => {
vi.resetModules();
const moduleA = await import("#harness/instrumentation-config.js");
const config = { functionId: "test.instrumentation.reimport.canary" };
- moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" });
+ await moduleA.registerInstrumentationConfig(config, { agentName: "test-agent" });
const firstRef = (globalThis as Record<symbol, unknown>)[globalKey];
vi.resetModules();
@@ -89,13 +91,23 @@ describe("instrumentation-config chunk-isolation regression", () => {
});
});
- it("invokes the setup callback with the supplied context", async () => {
+ it("awaits the setup callback with the resolved context", async () => {
vi.resetModules();
const { registerInstrumentationConfig } = await import("#harness/instrumentation-config.js");
- const setup = vi.fn();
- registerInstrumentationConfig({ setup }, { agentName: "weather-agent" });
+ const contexts: InstrumentationSetupContext[] = [];
+ await registerInstrumentationConfig(
+ {
+ setup: (context) => {
+ contexts.push(context);
+ },
+ },
+ { agentName: "weather-agent" },
+ );
- expect(setup).toHaveBeenCalledExactlyOnceWith({ agentName: "weather-agent" });
+ expect(contexts).toHaveLength(1);
+ expect(contexts[0]?.agentName).toBe("weather-agent");
+ expect(contexts[0]?.environment).toMatch(/^(development|production)$/);
+ expect(contexts[0]?.frameworkVersion).toEqual(expect.any(String));
});
});
diff --git a/packages/eve/src/harness/instrumentation-config.ts b/packages/eve/src/harness/instrumentation-config.ts
index fbe66be55..541ad1f15 100644
--- a/packages/eve/src/harness/instrumentation-config.ts
+++ b/packages/eve/src/harness/instrumentation-config.ts
@@ -1,9 +1,7 @@
-import type {
- InstrumentationDefinition,
- InstrumentationSetupContext,
-} from "#public/instrumentation/index.js";
import { createInstrumentationHooks } from "#harness/instrumentation-lifecycle.js";
import { registerInstrumentationRuntime } from "#harness/instrumentation-runtime.js";
+import { createInstrumentationSetupContext } from "#harness/instrumentation-setup-context.js";
+import type { InstrumentationDefinition } from "#public/instrumentation/index.js";
/**
* Process-global store for the authored instrumentation config.
@@ -30,19 +28,24 @@ interface InstrumentationConfigGlobal {
const globalContainer = globalThis as typeof globalThis & InstrumentationConfigGlobal;
/**
- * Registers the authored instrumentation config and invokes its `setup`
- * callback with the resolved agent name.
+ * Registers the authored instrumentation config and awaits its `setup`
+ * callback.
*
* Called once by the generated instrumentation Nitro plugin at server
* startup. Subsequent calls overwrite the previous value.
*
+ * The store write lands before `setup` runs so a synchronous caller sees the
+ * config without waiting on the returned promise.
+ *
* @internal — not part of the public API.
*/
export async function registerInstrumentationConfig(
config: InstrumentationDefinition,
- context: InstrumentationSetupContext,
+ input: { readonly agentName: string },
): Promise<void> {
globalContainer[INSTRUMENTATION_CONFIG_GLOBAL_KEY] = config;
+ // This legacy layout leaves `registerOTel` to `setup`, so install only the
+ // runtime projection consumed by the harness.
registerInstrumentationRuntime({
forceFlush: async () => undefined,
hooks: createInstrumentationHooks([]),
@@ -55,7 +58,7 @@ export async function registerInstrumentationConfig(
runInContext: (_operation, execute) => execute(),
shutdown: async () => undefined,
});
- await config.setup?.(context);
+ await config.setup?.(createInstrumentationSetupContext(input.agentName));
}
/**
diff --git a/packages/eve/src/harness/instrumentation-dispatch.ts b/packages/eve/src/harness/instrumentation-dispatch.ts
index 5125fd738..fffbf3c2b 100644
--- a/packages/eve/src/harness/instrumentation-dispatch.ts
+++ b/packages/eve/src/harness/instrumentation-dispatch.ts
@@ -155,17 +155,20 @@ async function dispatchToProvider(
const startedBoundary = event.type.endsWith(".started") || event.type === "input.requested";
const owner = stateOwner(event);
const providerName = provider.name;
- if (isInstrumentationStateAbandoned(providerName, event.idempotencyKey)) return;
+ const stateNamespace = provider.stateNamespace ?? providerName;
+ if (isInstrumentationStateAbandoned(stateNamespace, event.idempotencyKey)) return;
const handler = provider.events?.[event.type];
if (handler === undefined) return;
- const state = instrumentationStateSlot(providerName, event.idempotencyKey, owner);
+ const state = instrumentationStateSlot(stateNamespace, event.idempotencyKey, owner);
try {
const settled = await withTimeout(
() => (handler as InstrumentationEventHandler<InstrumentationEvent>)(event, { state }),
handlerTimeoutMs,
() => {
state.revoke();
- if (startedBoundary) abandonInstrumentationState(providerName, event.idempotencyKey, owner);
+ if (startedBoundary) {
+ abandonInstrumentationState(stateNamespace, event.idempotencyKey, owner);
+ }
},
);
if (!settled) {
diff --git a/packages/eve/src/harness/instrumentation-lifecycle.ts b/packages/eve/src/harness/instrumentation-lifecycle.ts
index c8a7bd022..f9cbce54a 100644
--- a/packages/eve/src/harness/instrumentation-lifecycle.ts
+++ b/packages/eve/src/harness/instrumentation-lifecycle.ts
@@ -445,6 +445,8 @@ export type InstrumentationEventHandler<TEvent> = (
/** Internal provider shape mirrored by the future public hook contract. */
export interface InstrumentationProviderDefinition {
readonly name: string;
+ /** Durable state identity, separate from the human-readable log name. */
+ readonly stateNamespace?: string;
readonly events?: {
readonly "step.attempt.started"?: InstrumentationEventHandler<InstrumentationStepAttemptStartedEvent>;
readonly "step.attempt.completed"?: InstrumentationEventHandler<InstrumentationStepAttemptCompletedEvent>;
diff --git a/packages/eve/src/harness/instrumentation-native-events.test.ts b/packages/eve/src/harness/instrumentation-native-events.test.ts
index afa99f2fd..16a5e19fc 100644
--- a/packages/eve/src/harness/instrumentation-native-events.test.ts
+++ b/packages/eve/src/harness/instrumentation-native-events.test.ts
@@ -160,6 +160,7 @@ describe("createInstrumentationHandleEvent", () => {
},
]);
});
+
it("publishes every runtime action and settles it in a replacement worker", async () => {
const events: unknown[] = [];
const scope = {
diff --git a/packages/eve/src/harness/instrumentation-native-events.ts b/packages/eve/src/harness/instrumentation-native-events.ts
index bfe369b6e..477d2469e 100644
--- a/packages/eve/src/harness/instrumentation-native-events.ts
+++ b/packages/eve/src/harness/instrumentation-native-events.ts
@@ -50,9 +50,9 @@ export function createInstrumentationHandleEvent(
const handleEvent = input.handleEvent;
const hooks = input.hooks;
- let activeTurnId = input.turnId;
const publishedActions = new Set<string>();
const publishedInputs = new Set<string>();
+ let activeTurnId = input.turnId;
return async (event, messages) => {
await handleEvent(event, messages);
const lifecycleEvent = toLifecycleEvent(event, input, activeTurnId);
diff --git a/packages/eve/src/harness/instrumentation-providers-local.scenario.test.ts b/packages/eve/src/harness/instrumentation-providers-local.scenario.test.ts
new file mode 100644
index 000000000..ca81922e4
--- /dev/null
+++ b/packages/eve/src/harness/instrumentation-providers-local.scenario.test.ts
@@ -0,0 +1,45 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import {
+ finalizeInstrumentationProviders,
+ getInstrumentationProviders,
+ registerInstrumentationProvider,
+ seedInstrumentationProviders,
+} from "#harness/instrumentation-providers.js";
+import { DEVELOPMENT_WORKER_APP_ROOT_ENV } from "#internal/workflow/development-world-protocol.js";
+import { otelIntegration } from "#public/instrumentation/otel.js";
+
+const temporaryDirectories: string[] = [];
+
+afterEach(async () => {
+ vi.unstubAllEnvs();
+ await Promise.all(
+ temporaryDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true })),
+ );
+});
+
+describe("instrumentation provider local default", () => {
+ it("registers default local traces and an authored destination in one pipeline", async () => {
+ const appRoot = await mkdtemp(join(tmpdir(), "eve-provider-local-"));
+ temporaryDirectories.push(appRoot);
+ vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, appRoot);
+ vi.stubEnv("EVE_TRACES", "off");
+
+ seedInstrumentationProviders();
+ await registerInstrumentationProvider({
+ agentName: "weather",
+ slot: "backend",
+ value: otelIntegration(),
+ });
+
+ const runtime = finalizeInstrumentationProviders({ serviceName: "weather" });
+ await runtime.forceFlush();
+ await runtime.shutdown();
+
+ expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["backend", "local"]);
+ });
+});
diff --git a/packages/eve/src/harness/instrumentation-providers-production.scenario.test.ts b/packages/eve/src/harness/instrumentation-providers-production.scenario.test.ts
new file mode 100644
index 000000000..1afb44245
--- /dev/null
+++ b/packages/eve/src/harness/instrumentation-providers-production.scenario.test.ts
@@ -0,0 +1,33 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import {
+ finalizeInstrumentationProviders,
+ getInstrumentationProviders,
+ registerInstrumentationProvider,
+ seedInstrumentationProviders,
+} from "#harness/instrumentation-providers.js";
+import { localTraces } from "#public/instrumentation/otel.js";
+
+afterEach(() => {
+ vi.unstubAllEnvs();
+});
+
+describe("instrumentation provider production defaults", () => {
+ it("keeps authored local traces inert beside Agent Runs", async () => {
+ vi.stubEnv("EVE_DEV_WORKER_APP_ROOT", undefined);
+ vi.stubEnv("VERCEL_ENV", "production");
+
+ seedInstrumentationProviders();
+ await registerInstrumentationProvider({
+ agentName: "weather",
+ slot: "local",
+ value: localTraces(),
+ });
+
+ const runtime = finalizeInstrumentationProviders({ serviceName: "weather" });
+ await runtime.forceFlush();
+ await runtime.shutdown();
+
+ expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["agent-runs", "local"]);
+ });
+});
diff --git a/packages/eve/src/harness/instrumentation-providers.integration.test.ts b/packages/eve/src/harness/instrumentation-providers.integration.test.ts
new file mode 100644
index 000000000..dfb6cd089
--- /dev/null
+++ b/packages/eve/src/harness/instrumentation-providers.integration.test.ts
@@ -0,0 +1,92 @@
+import { beforeEach, describe, expect, it } from "vitest";
+
+import { turnIdempotencyKey } from "#harness/instrumentation-lifecycle.js";
+import {
+ finalizeInstrumentationProviders,
+ registerInstrumentationProvider,
+} from "#harness/instrumentation-providers.js";
+import { defineInstrumentation } from "#public/instrumentation/index.js";
+
+const REGISTRY_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers");
+const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime");
+
+function deferred(): { readonly promise: Promise<void>; readonly resolve: () => void } {
+ let resolve!: () => void;
+ const promise = new Promise<void>((complete) => {
+ resolve = complete;
+ });
+ return { promise, resolve };
+}
+
+describe("authored instrumentation provider dispatch", () => {
+ beforeEach(() => {
+ delete (globalThis as Record<symbol, unknown>)[REGISTRY_GLOBAL_KEY];
+ delete (globalThis as Record<symbol, unknown>)[RUNTIME_GLOBAL_KEY];
+ });
+
+ it("awaits setup in slot order while event handlers overlap", async () => {
+ const firstSetup = deferred();
+ const firstHandler = deferred();
+ const secondHandler = deferred();
+ const order: string[] = [];
+
+ const firstRegistration = registerInstrumentationProvider({
+ agentName: "weather",
+ slot: "first",
+ value: defineInstrumentation({
+ events: {
+ "turn.started": async () => {
+ order.push("first:handler");
+ await firstHandler.promise;
+ },
+ },
+ setup: async () => {
+ order.push("first:setup");
+ await firstSetup.promise;
+ order.push("first:setup-complete");
+ },
+ }),
+ });
+
+ expect(order).toEqual(["first:setup"]);
+ firstSetup.resolve();
+ await firstRegistration;
+
+ await registerInstrumentationProvider({
+ agentName: "weather",
+ slot: "second",
+ value: defineInstrumentation({
+ events: {
+ "turn.started": async () => {
+ order.push("second:handler");
+ await secondHandler.promise;
+ },
+ },
+ setup: () => void order.push("second:setup"),
+ }),
+ });
+
+ expect(order).toEqual(["first:setup", "first:setup-complete", "second:setup"]);
+
+ const runtime = finalizeInstrumentationProviders({ serviceName: "weather" });
+ const publication = runtime.hooks.publish({
+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),
+ rootSessionId: "session-1",
+ sequence: 0,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "turn.started",
+ });
+
+ expect(order).toEqual([
+ "first:setup",
+ "first:setup-complete",
+ "second:setup",
+ "first:handler",
+ "second:handler",
+ ]);
+ firstHandler.resolve();
+ secondHandler.resolve();
+ await publication;
+ });
+});
diff --git a/packages/eve/src/harness/instrumentation-providers.test.ts b/packages/eve/src/harness/instrumentation-providers.test.ts
new file mode 100644
index 000000000..1b2842869
--- /dev/null
+++ b/packages/eve/src/harness/instrumentation-providers.test.ts
@@ -0,0 +1,255 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { turnIdempotencyKey } from "#harness/instrumentation-lifecycle.js";
+import {
+ EVE_EVALUATION_ENV_FLAG,
+ EVE_EVALUATION_RUN_ID_ENV,
+} from "#internal/application/dev-environment.js";
+import {
+ finalizeInstrumentationProviders,
+ getInstrumentationProviders,
+ registerInstrumentationProvider,
+ seedInstrumentationProviders,
+ shutdownInstrumentationProviders,
+} from "#harness/instrumentation-providers.js";
+import { DEVELOPMENT_WORKER_APP_ROOT_ENV } from "#internal/workflow/development-world-protocol.js";
+import { defineInstrumentation } from "#public/instrumentation/index.js";
+import { agentRuns, localTraces, otelIntegration } from "#public/instrumentation/otel.js";
+import {
+ disableInstrumentation,
+ type ProviderSetupContext,
+} from "#public/instrumentation/provider.js";
+
+const REGISTRY_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers");
+const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime");
+
+function register(slot: string, value: unknown): Promise<void> {
+ return registerInstrumentationProvider({ agentName: "weather-agent", slot, value });
+}
+
+describe("registerInstrumentationProvider", () => {
+ beforeEach(() => {
+ vi.unstubAllEnvs();
+ delete (globalThis as Record<symbol, unknown>)[REGISTRY_GLOBAL_KEY];
+ delete (globalThis as Record<symbol, unknown>)[RUNTIME_GLOBAL_KEY];
+ });
+
+ it("registers a provider under its slot", async () => {
+ const provider = defineInstrumentation({ events: {} });
+ await register("otel", provider);
+
+ expect(getInstrumentationProviders()).toEqual([{ provider, slot: "otel" }]);
+ });
+
+ it("preserves registration order across slots", async () => {
+ await register("agent-runs", defineInstrumentation({}));
+ await register("local", defineInstrumentation({}));
+
+ expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["agent-runs", "local"]);
+ });
+
+ it("awaits setup with the resolved provider context", async () => {
+ const contexts: ProviderSetupContext[] = [];
+ await register(
+ "otel",
+ defineInstrumentation({
+ setup: (context) => {
+ contexts.push(context);
+ },
+ shutdown: () => {},
+ }),
+ );
+
+ expect(contexts).toHaveLength(1);
+ expect(contexts[0]?.agentName).toBe("weather-agent");
+ expect(contexts[0]?.environment).toMatch(/^(development|preview|production)$/);
+ expect(contexts[0]?.evaluation).toBeUndefined();
+ expect(contexts[0]?.frameworkVersion).toEqual(expect.any(String));
+ });
+
+ it("reports Vercel preview separately from production", async () => {
+ vi.stubEnv("VERCEL_ENV", "preview");
+ const contexts: ProviderSetupContext[] = [];
+
+ await register(
+ "otel",
+ defineInstrumentation({ setup: (context) => void contexts.push(context) }),
+ );
+
+ expect(contexts[0]?.environment).toBe("preview");
+ });
+
+ it("reports an evaluation server to setup", async () => {
+ vi.stubEnv(EVE_EVALUATION_ENV_FLAG, "1");
+ vi.stubEnv(EVE_EVALUATION_RUN_ID_ENV, "eval-run-1");
+
+ const contexts: ProviderSetupContext[] = [];
+ await register(
+ "otel",
+ defineInstrumentation({
+ setup: (context) => {
+ contexts.push(context);
+ },
+ }),
+ );
+
+ expect(contexts[0]?.evaluation).toEqual({ runId: "eval-run-1" });
+ });
+
+ it("registers nothing for a disabled slot", async () => {
+ await register("local", disableInstrumentation());
+
+ expect(getInstrumentationProviders()).toEqual([]);
+ });
+
+ it("removes an already-registered provider when a later file disables the slot", async () => {
+ await register("local", defineInstrumentation({}));
+ await register("local", disableInstrumentation());
+
+ expect(getInstrumentationProviders()).toEqual([]);
+ });
+
+ // A slot that registers nothing is telemetry that silently does nothing —
+ // the failure this surface exists to prevent — so the shape check throws
+ // rather than skipping the file.
+ it.each([
+ ["a bare object", { events: {} }],
+ ["a function", () => {}],
+ ["undefined", undefined],
+ ["null", null],
+ ])("throws when the default export is %s", async (_label, value) => {
+ await expect(register("otel", value)).rejects.toThrow(
+ /The default export of "instrumentation\/otel" is not an instrumentation provider/,
+ );
+ });
+});
+
+describe("seedInstrumentationProviders", () => {
+ beforeEach(() => {
+ vi.unstubAllEnvs();
+ delete (globalThis as Record<symbol, unknown>)[REGISTRY_GLOBAL_KEY];
+ delete (globalThis as Record<symbol, unknown>)[RUNTIME_GLOBAL_KEY];
+ vi.stubEnv("EVE_TRACES", "off");
+ vi.stubEnv("VERCEL_ENV", undefined);
+ vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, "/tmp/eve-seed-test");
+ });
+
+ it("sorts default local traces with authored destinations", async () => {
+ seedInstrumentationProviders();
+ await register("backend", otelIntegration());
+
+ expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["backend", "local"]);
+ });
+
+ it("seeds Agent Runs only in Vercel production", () => {
+ vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, undefined);
+ vi.stubEnv("VERCEL_ENV", "production");
+
+ seedInstrumentationProviders();
+
+ expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual(["agent-runs"]);
+ });
+
+ it("lets an authored reserved slot reconfigure or disable its default", async () => {
+ seedInstrumentationProviders();
+ const authored = localTraces({ recordInputs: false });
+ await register("local", authored);
+ expect(getInstrumentationProviders()).toEqual([{ provider: authored, slot: "local" }]);
+
+ await register("local", disableInstrumentation());
+ expect(getInstrumentationProviders()).toEqual([]);
+ });
+
+ it("lets an authored Agent Runs slot narrow the production default", async () => {
+ vi.stubEnv(DEVELOPMENT_WORKER_APP_ROOT_ENV, undefined);
+ vi.stubEnv("VERCEL_ENV", "production");
+ seedInstrumentationProviders();
+ const authored = agentRuns({ recordOutputs: false });
+
+ await register("agent-runs", authored);
+
+ expect(getInstrumentationProviders()).toEqual([{ provider: authored, slot: "agent-runs" }]);
+ });
+
+ it("sorts built-ins, authored slots, and reserved-slot replacements together", async () => {
+ vi.stubEnv("VERCEL_ENV", "production");
+ seedInstrumentationProviders();
+ await register("zeta", defineInstrumentation({}));
+ await register("audit", defineInstrumentation({}));
+ await register("local", localTraces({ recordInputs: false }));
+
+ expect(getInstrumentationProviders().map(({ slot }) => slot)).toEqual([
+ "agent-runs",
+ "audit",
+ "local",
+ "zeta",
+ ]);
+ });
+});
+
+describe("finalizeInstrumentationProviders", () => {
+ const turnStarted = {
+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),
+ rootSessionId: "session-1",
+ sequence: 0,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "turn.started",
+ } as const;
+
+ beforeEach(() => {
+ vi.unstubAllEnvs();
+ delete (globalThis as Record<symbol, unknown>)[REGISTRY_GLOBAL_KEY];
+ delete (globalThis as Record<symbol, unknown>)[RUNTIME_GLOBAL_KEY];
+ });
+
+ it("publishes to an authored handler", async () => {
+ const started = vi.fn();
+ await register("rows", defineInstrumentation({ events: { "turn.started": started } }));
+
+ const runtime = finalizeInstrumentationProviders({ serviceName: "weather-agent" });
+ await runtime.hooks.publish(turnStarted);
+
+ expect(started).toHaveBeenCalledOnce();
+ expect(started.mock.calls[0]?.[0]).toMatchObject({ turnId: "turn-1" });
+ });
+
+ it("still runs execution when no destination was declared", async () => {
+ // A directory with no `otel()` has nothing to hang a span on, so
+ // `runInContext` degrades to running the work directly rather than
+ // going missing.
+ await register("rows", defineInstrumentation({}));
+
+ const runtime = finalizeInstrumentationProviders({ serviceName: "weather-agent" });
+ const result = await runtime.runInContext(
+ {
+ idempotencyKey: "tool:session-1:turn-1:0:0:call-1:0",
+ scope: {
+ attemptId: "session-1:turn-1:0:0",
+ attemptIndex: 0,
+ sessionId: "session-1",
+ stepIndex: 0,
+ turnId: "turn-1",
+ },
+ type: "tool.call",
+ },
+ () => Promise.resolve("ran"),
+ );
+
+ expect(result).toBe("ran");
+ });
+
+ it("drains and releases every provider", async () => {
+ const flush = vi.fn();
+ const shutdown = vi.fn();
+ await register("rows", defineInstrumentation({ flush, shutdown }));
+
+ const runtime = finalizeInstrumentationProviders({ serviceName: "weather-agent" });
+ await runtime.forceFlush();
+ await shutdownInstrumentationProviders();
+ await shutdownInstrumentationProviders();
+
+ expect(flush).toHaveBeenCalledOnce();
+ expect(shutdown).toHaveBeenCalledOnce();
+ });
+});
diff --git a/packages/eve/src/harness/instrumentation-providers.ts b/packages/eve/src/harness/instrumentation-providers.ts
new file mode 100644
index 000000000..13fb2a3eb
--- /dev/null
+++ b/packages/eve/src/harness/instrumentation-providers.ts
@@ -0,0 +1,153 @@
+import type { InstrumentationProviderDefinition } from "#harness/instrumentation-lifecycle.js";
+import {
+ getInstrumentationRuntime,
+ type InstrumentationRuntime,
+} from "#harness/instrumentation-runtime.js";
+import { createInstrumentationSetupContext } from "#harness/instrumentation-setup-context.js";
+import { resolveInstalledPackageInfo } from "#internal/application/package.js";
+import { DEVELOPMENT_WORKER_APP_ROOT_ENV } from "#internal/workflow/development-world-protocol.js";
+import { agentRuns, localTraces } from "#public/instrumentation/otel.js";
+import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js";
+import { collectOtelPipeline } from "#tracing/otel-declaration.js";
+import {
+ isInstrumentationDisabled,
+ isInstrumentationProvider,
+ type InstrumentationProvider,
+} from "#public/instrumentation/provider.js";
+
+/**
+ * Process-global registry of the providers authored under
+ * `agent/instrumentation/`.
+ *
+ * Rooted on `globalThis` for the same reason the single-config store is: the
+ * generated Nitro plugin stays external by `file://` URL while the harness
+ * chunk is inlined, so the two resolve to distinct ESM module instances and
+ * need one shared source of truth.
+ */
+const INSTRUMENTATION_PROVIDERS_GLOBAL_KEY = Symbol.for("eve.harness-instrumentation-providers");
+
+/** One provider and the `instrumentation/<slot>.ts` file it came from. */
+export interface RegisteredInstrumentationProvider {
+ readonly provider: InstrumentationProvider;
+ readonly slot: string;
+}
+
+interface InstrumentationProvidersGlobal {
+ [INSTRUMENTATION_PROVIDERS_GLOBAL_KEY]?: Map<string, InstrumentationProvider>;
+}
+
+const globalContainer = globalThis as typeof globalThis & InstrumentationProvidersGlobal;
+
+function providerRegistry(): Map<string, InstrumentationProvider> {
+ const existing = globalContainer[INSTRUMENTATION_PROVIDERS_GLOBAL_KEY];
+ if (existing !== undefined) {
+ return existing;
+ }
+
+ const created = new Map<string, InstrumentationProvider>();
+ globalContainer[INSTRUMENTATION_PROVIDERS_GLOBAL_KEY] = created;
+ return created;
+}
+
+/** Fills reserved slots before authored files may reconfigure or disable them. */
+export function seedInstrumentationProviders(): void {
+ const registry = providerRegistry();
+ if (process.env.VERCEL_ENV === "production") registry.set("agent-runs", agentRuns());
+ if (process.env[DEVELOPMENT_WORKER_APP_ROOT_ENV] !== undefined) {
+ registry.set("local", localTraces());
+ }
+}
+
+/**
+ * Registers one authored provider and awaits its `setup`.
+ *
+ * Called once per `instrumentation/<slot>.ts` by the generated Nitro plugin at
+ * server startup, before any event is published. A default export that is not
+ * a `defineInstrumentation` result throws rather than being skipped: a slot
+ * that registers nothing is telemetry that silently does nothing, which is the
+ * failure this surface exists to prevent.
+ *
+ * @internal — not part of the public API.
+ */
+export async function registerInstrumentationProvider(input: {
+ readonly agentName: string;
+ readonly slot: string;
+ readonly value: unknown;
+}): Promise<void> {
+ if (isInstrumentationDisabled(input.value)) {
+ providerRegistry().delete(input.slot);
+ return;
+ }
+
+ if (!isInstrumentationProvider(input.value)) {
+ throw new Error(
+ `The default export of "instrumentation/${input.slot}" is not an instrumentation provider. Return the result of \`defineInstrumentation\` or \`disableInstrumentation\` from it.`,
+ );
+ }
+
+ providerRegistry().set(input.slot, input.value);
+ await input.value.setup?.(createInstrumentationSetupContext(input.agentName));
+}
+
+/** Registered providers in slot order. @internal */
+export function getInstrumentationProviders(): readonly RegisteredInstrumentationProvider[] {
+ return [...providerRegistry()]
+ .sort(([left], [right]) => left.localeCompare(right))
+ .map(([slot, provider]) => ({ provider, slot }));
+}
+
+/**
+ * Installs the process instrumentation runtime from the registered providers.
+ *
+ * Called once by the generated Nitro plugin after every slot has registered,
+ * which is also why it cannot happen inside `setup`: the OpenTelemetry pipeline
+ * is the union of every destination declared in the directory, so no single
+ * file knows enough to build it. A `setup` that reaches for a tracer therefore
+ * gets the no-op one; declare destinations as values and let this assemble
+ * them.
+ *
+ * A directory that declared no OpenTelemetry at all still gets a bus. Its
+ * providers see every event; they just have no spans to hang them on.
+ *
+ * @internal — not part of the public API.
+ */
+export function finalizeInstrumentationProviders(input: {
+ readonly serviceName: string;
+}): InstrumentationRuntime {
+ const registered = getInstrumentationProviders();
+ return installInstrumentationRuntime({
+ collected: collectOtelPipeline(registered.map((entry) => entry.provider)),
+ frameworkVersion: resolveInstalledPackageInfo().version,
+ providers: registered.map(toProviderDefinition),
+ serviceName: input.serviceName,
+ });
+}
+
+/**
+ * Releases every registered provider and OTel processor from Nitro's close
+ * hook, the last point a buffered exporter can still reach the network.
+ */
+export async function shutdownInstrumentationProviders(): Promise<void> {
+ await getInstrumentationRuntime()?.shutdown();
+}
+
+/**
+ * Adapts an authored provider onto the internal bus contract.
+ *
+ * The event maps are the same shape — the public one is derived from the
+ * internal union and both handlers take `(event, ctx)` — so only the name has
+ * to be supplied.
+ */
+function toProviderDefinition(
+ entry: RegisteredInstrumentationProvider,
+): InstrumentationProviderDefinition {
+ return {
+ events: entry.provider.events as InstrumentationProviderDefinition["events"],
+ flush: entry.provider.flush,
+ // The file the provider came from, which is the only name an author can
+ // recognize in a log line about it.
+ name: entry.slot,
+ shutdown: entry.provider.shutdown,
+ stateNamespace: `authored:${entry.slot}`,
+ };
+}
diff --git a/packages/eve/src/harness/instrumentation-setup-context.ts b/packages/eve/src/harness/instrumentation-setup-context.ts
new file mode 100644
index 000000000..b23dfe575
--- /dev/null
+++ b/packages/eve/src/harness/instrumentation-setup-context.ts
@@ -0,0 +1,31 @@
+import {
+ isEveDevEnvironment,
+ resolveEveEvaluationRunId,
+} from "#internal/application/dev-environment.js";
+import { resolveInstalledPackageInfo } from "#internal/application/package.js";
+import type { ProviderSetupContext } from "#public/instrumentation/provider.js";
+
+/**
+ * Builds the context handed to an authored `setup` at server startup.
+ *
+ * Shared by both layouts so the two cannot drift: a divergent context type
+ * would leave `defineInstrumentation`'s union without a contextual signature
+ * for `setup`, silently making every authored `setup(context)` parameter an
+ * implicit `any`.
+ *
+ * @internal — not part of the public API.
+ */
+export function createInstrumentationSetupContext(agentName: string): ProviderSetupContext {
+ const evaluationRunId = resolveEveEvaluationRunId();
+ return {
+ agentName,
+ environment: resolveInstrumentationEnvironment(),
+ evaluation: evaluationRunId === undefined ? undefined : { runId: evaluationRunId },
+ frameworkVersion: resolveInstalledPackageInfo().version,
+ };
+}
+
+function resolveInstrumentationEnvironment(): ProviderSetupContext["environment"] {
+ if (isEveDevEnvironment() || process.env.VERCEL_ENV === "development") return "development";
+ return process.env.VERCEL_ENV === "preview" ? "preview" : "production";
+}
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index d3d84af5a..81fb1f608 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -77,18 +77,25 @@ vi.mock("ai", () => ({
tool: vi.fn((t: unknown) => t),
}));
-const { existingOtelIntegration, mockCreateAiSdkHookBridge } = vi.hoisted(() => ({
- existingOtelIntegration: { onStart: vi.fn() },
+const {
+ mockCreateAiSdkHookBridge,
+ mockGetRegisteredTelemetryIntegrations,
+ registeredAuthorIntegration,
+ registeredOtelIntegration,
+} = vi.hoisted(() => ({
mockCreateAiSdkHookBridge: vi.fn((..._args: unknown[]) => ({ onStart: vi.fn() })),
+ mockGetRegisteredTelemetryIntegrations: vi.fn((): unknown[] => []),
+ registeredAuthorIntegration: { onStart: vi.fn() },
+ registeredOtelIntegration: { onStart: vi.fn() },
}));
vi.mock("./ai-sdk-hook-bridge.js", () => ({
createAiSdkHookBridge: (...args: unknown[]) => mockCreateAiSdkHookBridge(...args),
}));
-vi.mock("./otel-integration.js", () => ({
- createOtelIntegration: vi.fn(() => existingOtelIntegration),
+vi.mock("./ai-sdk-telemetry.js", () => ({
ensureOtelIntegration: vi.fn(),
+ getRegisteredTelemetryIntegrations: () => mockGetRegisteredTelemetryIntegrations(),
}));
const mockGetInstrumentationConfig = vi.fn().mockReturnValue(undefined);
@@ -101,6 +108,10 @@ vi.mock("./instrumentation-runtime.js", () => ({
getInstrumentationRuntime: (...args: unknown[]) => mockGetInstrumentationRuntime(...args),
}));
+/**
+ * Registering an authored config writes both stores, so the tests toggle
+ * telemetry through one call rather than keeping two mocks in step by hand.
+ */
function declareTelemetry(config: Readonly<Record<string, unknown>> | undefined): void {
mockGetInstrumentationConfig.mockReturnValue(config);
mockGetInstrumentationRuntime.mockReturnValue(
@@ -135,6 +146,7 @@ afterEach(() => {
vi.clearAllMocks();
vi.unstubAllEnvs();
declareTelemetry(undefined);
+ mockGetRegisteredTelemetryIntegrations.mockReturnValue([]);
});
function createTestSession(overrides?: Partial<HarnessSession>): HarnessSession {
@@ -10204,7 +10216,7 @@ describe("createToolLoopHarness", () => {
});
const attemptCompleted = vi.fn();
const hooks = createInstrumentationHooks([
- { events: { "step.attempt.completed": attemptCompleted } },
+ { events: { "step.attempt.completed": attemptCompleted }, name: "attempt" },
]);
const runInContext: InstrumentationContextRunner = (_operation, execute) => execute();
const config = createTestConfig("conversation", undefined, {
@@ -10278,7 +10290,7 @@ describe("createToolLoopHarness", () => {
});
const started = vi.fn();
const hooks = createInstrumentationHooks([
- { events: { "action.started": started }, name: "test" },
+ { events: { "action.started": started }, name: "action" },
]);
const { emit } = createEventCollector();
const runStep = createToolLoopHarness(
@@ -10301,7 +10313,7 @@ describe("createToolLoopHarness", () => {
);
});
- it("composes lifecycle hooks with existing authored OTel", async () => {
+ it("composes the bridge with every registered integration", async () => {
setupMockAgent({
finishReason: "stop",
response: { messages: [{ content: "Hello!", role: "assistant" }] },
@@ -10310,6 +10322,11 @@ describe("createToolLoopHarness", () => {
toolResults: [],
});
declareTelemetry({ recordInputs: true, recordOutputs: false });
+ // An authored module can register its own integration alongside eve's.
+ mockGetRegisteredTelemetryIntegrations.mockReturnValue([
+ registeredOtelIntegration,
+ registeredAuthorIntegration,
+ ]);
const hooks = createInstrumentationHooks([]);
const runStep = createToolLoopHarness(
createTestConfig("conversation", undefined, {
@@ -10331,7 +10348,7 @@ describe("createToolLoopHarness", () => {
};
};
expect(agentCall.telemetry).toMatchObject({
- integrations: [bridge, existingOtelIntegration],
+ integrations: [bridge, registeredOtelIntegration, registeredAuthorIntegration],
recordInputs: true,
recordOutputs: false,
});
diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.ts
index 0409fedc2..5e99ea925 100644
--- a/packages/eve/src/harness/tool-loop.ts
+++ b/packages/eve/src/harness/tool-loop.ts
@@ -191,7 +191,10 @@ import {
hasEmptyDeliverySentinel,
} from "#shared/empty-delivery.js";
import { extractWorkflowStreamWriteErrorDetails } from "#harness/workflow-stream-error.js";
-import { createOtelIntegration, ensureOtelIntegration } from "#harness/otel-integration.js";
+import {
+ ensureOtelIntegration,
+ getRegisteredTelemetryIntegrations,
+} from "#harness/ai-sdk-telemetry.js";
import { getAdvertisedTools } from "#harness/advertised-tools.js";
import {
applyLastToolCacheBreakpoint,
@@ -246,7 +249,7 @@ import {
/**
* Builds the `telemetry` value for the AI SDK from authored settings.
*
- * Custom context (authored instrumentation events plus
+ * Custom context (authored `InstrumentationDefinition.events` plus
* eve-specific identifiers such as `eve.session.id`) is flowed through
* {@link buildTelemetryRuntimeContext} because AI SDK v7 surfaces
* per-call attributes via `runtimeContext`, not a dedicated metadata field on
@@ -311,12 +314,12 @@ function enrichTelemetry(
return {
functionId: settings?.functionId ?? agentName,
includeRuntimeContext,
+ // Passing integrations replaces the registered ones for this call, so the
+ // bridge has to be composed with them rather than handed over on its own.
integrations:
bridgeIntegration === undefined
? undefined
- : getInstrumentationConfig() === undefined
- ? [bridgeIntegration]
- : [bridgeIntegration, createOtelIntegration()],
+ : [bridgeIntegration, ...getRegisteredTelemetryIntegrations()],
isEnabled: true,
recordInputs: settings?.recordInputs ?? true,
recordOutputs: settings?.recordOutputs ?? true,
@@ -549,6 +552,9 @@ function buildHarnessToolsWithDynamicSubagents(
export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
const baseEmit = config.handleEvent;
const otelSettings = getInstrumentationRuntime()?.otelSettings;
+ // The custom-context enrichment below still reads the authored config object
+ // directly. Its replacement on the provider surface is unresolved, so a
+ // provider directory contributes no custom context yet.
const authoredConfig = getInstrumentationConfig();
if (otelSettings !== undefined) {
ensureOtelIntegration();
diff --git a/packages/eve/src/internal/application/compiled-artifacts.ts b/packages/eve/src/internal/application/compiled-artifacts.ts
index 561d60f45..a79a9ea0f 100644
--- a/packages/eve/src/internal/application/compiled-artifacts.ts
+++ b/packages/eve/src/internal/application/compiled-artifacts.ts
@@ -1,4 +1,3 @@
-import { existsSync } from "node:fs";
import { copyFile, mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
@@ -12,12 +11,23 @@ import {
} from "#internal/application/package.js";
import { buildPackageUserAgent } from "#internal/user-agent.js";
import type { AgentWorkflowWorldDefinition } from "#shared/agent-definition.js";
-import { readMaterializedAuthoredModuleIndex } from "#internal/materialized-authored-modules.js";
+import {
+ readMaterializedAuthoredModuleIndex,
+ type MaterializedInstrumentation,
+} from "#internal/materialized-authored-modules.js";
+import {
+ resolveInstrumentationLayout,
+ type InstrumentationLayout,
+} from "#internal/instrumentation-layout.js";
import { usesParentDevelopmentWorkflowWorld } from "#internal/workflow/development-world-protocol.js";
import { resolveWorkflowWorldImport } from "#internal/workflow/world-target.js";
export type BuiltInWorkflowWorldTarget = "local" | "vercel";
+export type GeneratedInstrumentationLayout =
+ | { readonly kind: "file" }
+ | { readonly kind: "directory"; readonly slots: readonly string[] };
+
/**
* Paths to the generated compiled-artifacts files shared by Nitro and the
* vendored workflow bundles for one application.
@@ -31,15 +41,18 @@ export interface GeneratedCompiledArtifactsFiles {
/** Nitro plugin that installs the selected vendored Workflow world. */
workflowWorldPluginPath: string;
/**
- * Optional Nitro plugin that imports the authored instrumentation module
+ * Optional Nitro plugin that imports the authored instrumentation modules
* from the application when present.
*/
instrumentationPluginPath?: string;
+ /** Layout identity used by dev plugin selection and structural fingerprinting. */
+ instrumentationLayout?: GeneratedInstrumentationLayout;
/**
- * Absolute path to the authored instrumentation module when present.
- * Nitro uses this to preserve the module's side effects during bundling.
+ * Absolute paths to the authored instrumentation modules when present — one
+ * for the file layout, one per provider in the directory layout. Nitro uses these
+ * to preserve each module's side effects during bundling.
*/
- instrumentationSourcePath?: string;
+ instrumentationSourcePaths?: readonly string[];
}
/**
@@ -57,7 +70,11 @@ export async function writeCompiledArtifactsFiles(input: {
const bootstrapPath = join(input.outDir, "compiled-artifacts-bootstrap.mjs");
const instrumentationPluginPath = join(input.outDir, "compiled-artifacts-instrumentation.mjs");
const workflowWorldPluginPath = join(input.outDir, "compiled-artifacts-workflow-world.mjs");
- const instrumentationPath = resolveInstrumentationModule(input.compileResult.manifest.agentRoot);
+ const layout = resolveInstrumentationLayout({
+ agentRoot: input.compileResult.manifest.agentRoot,
+ providersEnabled:
+ input.compileResult.manifest.config.experimental?.instrumentationProviders ?? false,
+ });
await mkdir(input.outDir, { recursive: true });
await writeFile(
@@ -78,30 +95,31 @@ export async function writeCompiledArtifactsFiles(input: {
}),
);
- if (instrumentationPath !== undefined) {
+ const generatedArtifacts: GeneratedCompiledArtifactsFiles = {
+ bootstrapPath,
+ workflowWorldPluginPath,
+ };
+
+ if (layout !== undefined) {
await writeFile(
instrumentationPluginPath,
createInstrumentationPluginSource({
agentName: input.compileResult.manifest.config.name,
- instrumentationPath,
- registerConfigPath: resolvePackageSourceFilePath("src/harness/instrumentation-config.ts"),
+ layout,
}),
);
- }
-
- const generatedArtifacts: GeneratedCompiledArtifactsFiles = {
- bootstrapPath,
- workflowWorldPluginPath,
- };
-
- if (instrumentationPath !== undefined) {
generatedArtifacts.instrumentationPluginPath = instrumentationPluginPath;
- generatedArtifacts.instrumentationSourcePath = instrumentationPath;
+ generatedArtifacts.instrumentationLayout = generatedInstrumentationLayout(layout);
+ generatedArtifacts.instrumentationSourcePaths = instrumentationSourcePathsOf(layout);
}
return generatedArtifacts;
}
+function instrumentationSourcePathsOf(layout: InstrumentationLayout): readonly string[] {
+ return layout.kind === "file" ? [layout.modulePath] : Object.values(layout.modulePathsBySlot);
+}
+
// The dev host's Nitro inputs outlive any single generation, so nothing
// written here may point into authored source or a prunable snapshot: the
// bootstrap references no authored module, and the instrumentation bundle is
@@ -113,10 +131,6 @@ export async function writeDevelopmentCompiledArtifactsFiles(input: {
}): Promise<GeneratedCompiledArtifactsFiles> {
const bootstrapPath = join(input.outDir, "compiled-artifacts-bootstrap.mjs");
const instrumentationPluginPath = join(input.outDir, "compiled-artifacts-instrumentation.mjs");
- const instrumentationSourcePath = join(
- input.outDir,
- "compiled-artifacts-instrumentation-source.mjs",
- );
const workflowWorldPluginPath = join(input.outDir, "compiled-artifacts-workflow-world.mjs");
const materializedIndex = await readMaterializedAuthoredModuleIndex(input.runtimeAppRoot);
@@ -143,25 +157,66 @@ export async function writeDevelopmentCompiledArtifactsFiles(input: {
};
if (materializedIndex.instrumentation !== undefined) {
- await copyFile(
- join(input.runtimeAppRoot, ".eve", "compile", materializedIndex.instrumentation),
- instrumentationSourcePath,
- );
+ const layout = await copyMaterializedInstrumentation({
+ instrumentation: materializedIndex.instrumentation,
+ outDir: input.outDir,
+ runtimeAppRoot: input.runtimeAppRoot,
+ });
+
await writeFile(
instrumentationPluginPath,
createInstrumentationPluginSource({
agentName: input.compileResult.manifest.config.name,
- instrumentationPath: instrumentationSourcePath,
- registerConfigPath: resolvePackageSourceFilePath("src/harness/instrumentation-config.ts"),
+ layout,
}),
);
generatedArtifacts.instrumentationPluginPath = instrumentationPluginPath;
- generatedArtifacts.instrumentationSourcePath = instrumentationSourcePath;
+ generatedArtifacts.instrumentationLayout = generatedInstrumentationLayout(layout);
+ generatedArtifacts.instrumentationSourcePaths = instrumentationSourcePathsOf(layout);
}
return generatedArtifacts;
}
+function generatedInstrumentationLayout(
+ layout: InstrumentationLayout,
+): GeneratedInstrumentationLayout {
+ return layout.kind === "file"
+ ? { kind: "file" }
+ : { kind: "directory", slots: Object.keys(layout.modulePathsBySlot) };
+}
+
+/**
+ * Copies each materialized instrumentation bundle out of the prunable
+ * generation and into the stable dev-host directory, returning the layout in
+ * terms of the copies.
+ */
+async function copyMaterializedInstrumentation(input: {
+ readonly instrumentation: MaterializedInstrumentation;
+ readonly outDir: string;
+ readonly runtimeAppRoot: string;
+}): Promise<InstrumentationLayout> {
+ const compileRoot = join(input.runtimeAppRoot, ".eve", "compile");
+ const copyOne = async (sourceId: string, modulePath: string): Promise<string> => {
+ const destination = join(input.outDir, `compiled-artifacts-instrumentation-${sourceId}.mjs`);
+ await copyFile(join(compileRoot, modulePath), destination);
+ return destination;
+ };
+
+ if (input.instrumentation.kind === "file") {
+ return {
+ kind: "file",
+ modulePath: await copyOne("source", input.instrumentation.modulePath),
+ };
+ }
+
+ const modulePathsBySlot: Record<string, string> = {};
+ for (const [slot, modulePath] of Object.entries(input.instrumentation.modulePathsBySlot)) {
+ modulePathsBySlot[slot] = await copyOne(slot, modulePath);
+ }
+ return { kind: "directory", modulePathsBySlot };
+}
+
function createDevelopmentCompiledArtifactsBootstrapSource(agentName: string): string {
return [
"// Generated by eve. Do not edit by hand.",
@@ -174,23 +229,6 @@ function createDevelopmentCompiledArtifactsBootstrapSource(agentName: string): s
].join("\n");
}
-const INSTRUMENTATION_EXTENSIONS = [".ts", ".mts", ".js", ".mjs"];
-
-/**
- * Resolves the optional `agent/instrumentation` module from the agent root
- * directory. Returns the absolute path if found, `undefined` otherwise.
- */
-function resolveInstrumentationModule(agentRoot: string): string | undefined {
- for (const ext of INSTRUMENTATION_EXTENSIONS) {
- const candidate = join(agentRoot, `instrumentation${ext}`);
- if (existsSync(candidate)) {
- return candidate;
- }
- }
-
- return undefined;
-}
-
function stripCompiledModuleMapExports(source: string): string {
return source
.replace(/^export const moduleMap = /m, "const moduleMap = ")
@@ -380,23 +418,73 @@ export function createDevelopmentWorkflowWorldPluginSource(input: {
].join("\n");
}
+/**
+ * Generates the Nitro plugin that registers the authored instrumentation.
+ *
+ * The file layout registers one default export; the directory layout
+ * registers one per file, in slot order, awaiting each `setup` so a provider
+ * cannot miss an event published while it is still starting up, then builds the
+ * one OpenTelemetry pipeline their declarations add up to.
+ */
function createInstrumentationPluginSource(input: {
agentName: string;
- instrumentationPath: string;
- registerConfigPath: string;
+ layout: InstrumentationLayout;
}): string {
+ const agentName = JSON.stringify(input.agentName);
+
+ if (input.layout.kind === "file") {
+ const registerConfigPath = resolvePackageSourceFilePath(
+ "src/harness/instrumentation-config.ts",
+ );
+ return [
+ "// Generated by eve. Do not edit by hand.",
+ `import * as instrumentationModule from ${stringifyEsmImportSpecifier(input.layout.modulePath)};`,
+ `import { registerInstrumentationConfig } from ${stringifyEsmImportSpecifier(registerConfigPath)};`,
+ "",
+ "if (instrumentationModule.default != null) {",
+ ` await registerInstrumentationConfig(instrumentationModule.default, { agentName: ${agentName} });`,
+ "}",
+ "",
+ "// Default export satisfies the Nitro plugin contract so this file",
+ "// can be used directly as a Nitro plugin without a separate wrapper.",
+ "export default function installInstrumentationPlugin() {}",
+ "",
+ ].join("\n");
+ }
+
+ const registerProviderPath = resolvePackageSourceFilePath(
+ "src/harness/instrumentation-providers.ts",
+ );
+ const slots = Object.entries(input.layout.modulePathsBySlot);
+
return [
"// Generated by eve. Do not edit by hand.",
- `import * as instrumentationModule from ${stringifyEsmImportSpecifier(input.instrumentationPath)};`,
- `import { registerInstrumentationConfig } from ${stringifyEsmImportSpecifier(input.registerConfigPath)};`,
+ ...slots.map(
+ ([slot, modulePath], index) =>
+ `import * as provider${index} from ${stringifyEsmImportSpecifier(modulePath)}; // ${slot}`,
+ ),
+ `import { finalizeInstrumentationProviders, registerInstrumentationProvider, seedInstrumentationProviders, shutdownInstrumentationProviders } from ${stringifyEsmImportSpecifier(registerProviderPath)};`,
"",
- "if (instrumentationModule.default != null) {",
- ` await registerInstrumentationConfig(instrumentationModule.default, { agentName: ${JSON.stringify(input.agentName)} });`,
- "}",
+ "seedInstrumentationProviders();",
+ "",
+ ...slots.flatMap(([slot], index) => [
+ "await registerInstrumentationProvider({",
+ ` agentName: ${agentName},`,
+ ` slot: ${JSON.stringify(slot)},`,
+ ` value: provider${index}.default,`,
+ "});",
+ ]),
+ "",
+ `finalizeInstrumentationProviders({ serviceName: ${agentName} });`,
"",
"// Default export satisfies the Nitro plugin contract so this file",
"// can be used directly as a Nitro plugin without a separate wrapper.",
- "export default function installInstrumentationPlugin() {}",
+ "export default function installInstrumentationPlugin(nitroApp) {",
+ " // The last point a buffered exporter can still reach the network.",
+ " nitroApp?.hooks?.hook('close', async () => {",
+ " await shutdownInstrumentationProviders();",
+ " });",
+ "}",
"",
].join("\n");
}
diff --git a/packages/eve/src/internal/application/dev-environment.ts b/packages/eve/src/internal/application/dev-environment.ts
index acbd09cfc..daa2ff244 100644
--- a/packages/eve/src/internal/application/dev-environment.ts
+++ b/packages/eve/src/internal/application/dev-environment.ts
@@ -5,3 +5,25 @@ export const EVE_DEV_ENV_FLAG = "EVE_DEV";
export function isEveDevEnvironment(): boolean {
return process.env[EVE_DEV_ENV_FLAG] === "1";
}
+
+/** Environment flag set for a server `eve eval` started to run against. */
+export const EVE_EVALUATION_ENV_FLAG = "EVE_EVALUATION";
+
+/** Stable identifier for the local eval run this server was started to serve. */
+export const EVE_EVALUATION_RUN_ID_ENV = "EVE_EVALUATION_RUN_ID";
+
+/**
+ * Reports whether this process exists to serve an eval run.
+ *
+ * False for a server that `eve eval --url` merely points at: that process was
+ * started to serve ordinary traffic and cannot know an eval is among it.
+ */
+export function isEveEvaluationEnvironment(): boolean {
+ return process.env[EVE_EVALUATION_ENV_FLAG] === "1";
+}
+
+export function resolveEveEvaluationRunId(): string | undefined {
+ if (!isEveEvaluationEnvironment()) return undefined;
+ const runId = process.env[EVE_EVALUATION_RUN_ID_ENV];
+ return runId === undefined || runId.length === 0 ? undefined : runId;
+}
diff --git a/packages/eve/src/internal/authored-definition/core.ts b/packages/eve/src/internal/authored-definition/core.ts
index 93ce950ae..633970fe1 100644
--- a/packages/eve/src/internal/authored-definition/core.ts
+++ b/packages/eve/src/internal/authored-definition/core.ts
@@ -259,9 +259,20 @@ function normalizeAgentExperimentalDefinition(
message: string,
): NonNullable<NormalizedAgentDefinition["experimental"]> {
const record = expectObjectRecord(value, message);
- expectOnlyKnownKeys(record, ["subagentPersistentSessions", "workflow"], message);
+ expectOnlyKnownKeys(
+ record,
+ ["instrumentationProviders", "subagentPersistentSessions", "workflow"],
+ message,
+ );
const normalizedDefinition: Mutable<NonNullable<NormalizedAgentDefinition["experimental"]>> = {};
+ if (record.instrumentationProviders !== undefined) {
+ if (typeof record.instrumentationProviders !== "boolean") {
+ throw new Error(`${message} "experimental.instrumentationProviders" must be a boolean.`);
+ }
+ normalizedDefinition.instrumentationProviders = record.instrumentationProviders;
+ }
+
if (record.subagentPersistentSessions !== undefined) {
if (typeof record.subagentPersistentSessions !== "boolean") {
throw new Error(`${message} "experimental.subagentPersistentSessions" must be a boolean.`);
diff --git a/packages/eve/src/internal/instrumentation-layout.integration.test.ts b/packages/eve/src/internal/instrumentation-layout.integration.test.ts
new file mode 100644
index 000000000..eca880b2b
--- /dev/null
+++ b/packages/eve/src/internal/instrumentation-layout.integration.test.ts
@@ -0,0 +1,126 @@
+import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { beforeEach, describe, expect, it } from "vitest";
+
+import { resolveInstrumentationLayout } from "#internal/instrumentation-layout.js";
+
+let agentRoot: string;
+
+beforeEach(() => {
+ agentRoot = mkdtempSync(join(tmpdir(), "eve-instrumentation-layout-"));
+});
+
+function writeInstrumentationFile(extension = ".ts"): string {
+ const path = join(agentRoot, `instrumentation${extension}`);
+ writeFileSync(path, "export default {};\n");
+ return path;
+}
+
+function writeInstrumentationProvider(fileName: string): string {
+ const directory = join(agentRoot, "instrumentation");
+ mkdirSync(directory, { recursive: true });
+ const path = join(directory, fileName);
+ writeFileSync(path, "export default {};\n");
+ return path;
+}
+
+describe("resolveInstrumentationLayout with providers off", () => {
+ it("returns nothing when the agent authored no instrumentation", () => {
+ expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: false })).toBeUndefined();
+ });
+
+ it("resolves the single instrumentation file", () => {
+ const modulePath = writeInstrumentationFile();
+
+ expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: false })).toEqual({
+ kind: "file",
+ modulePath,
+ });
+ });
+
+ it.each([[".mts"], [".js"], [".mjs"]])("resolves a %s instrumentation file", (extension) => {
+ const modulePath = writeInstrumentationFile(extension);
+
+ expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: false })).toEqual({
+ kind: "file",
+ modulePath,
+ });
+ });
+
+ it("ignores a providers directory", () => {
+ writeInstrumentationProvider("otel.ts");
+
+ expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: false })).toBeUndefined();
+ });
+});
+
+describe("resolveInstrumentationLayout with providers on", () => {
+ it("returns an empty directory layout for eve's built-in destinations", () => {
+ expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toEqual({
+ kind: "directory",
+ modulePathsBySlot: {},
+ });
+ });
+
+ it("keys each file by the slot its name derives", () => {
+ const otel = writeInstrumentationProvider("otel.ts");
+ const local = writeInstrumentationProvider("local.mts");
+
+ expect(resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toEqual({
+ kind: "directory",
+ modulePathsBySlot: { local, otel },
+ });
+ });
+
+ it("orders slots independently of directory enumeration", () => {
+ writeInstrumentationProvider("otel.ts");
+ writeInstrumentationProvider("agent-runs.ts");
+ writeInstrumentationProvider("local.ts");
+
+ const layout = resolveInstrumentationLayout({ agentRoot, providersEnabled: true });
+
+ expect(Object.keys(layout?.kind === "directory" ? layout.modulePathsBySlot : {})).toEqual([
+ "agent-runs",
+ "local",
+ "otel",
+ ]);
+ });
+
+ it("ignores files that are not instrumentation modules", () => {
+ writeInstrumentationProvider("otel.ts");
+ writeInstrumentationProvider("README.md");
+
+ const layout = resolveInstrumentationLayout({ agentRoot, providersEnabled: true });
+
+ expect(Object.keys(layout?.kind === "directory" ? layout.modulePathsBySlot : {})).toEqual([
+ "otel",
+ ]);
+ });
+
+ it("rejects two files claiming one slot", () => {
+ writeInstrumentationProvider("otel.ts");
+ writeInstrumentationProvider("otel.js");
+
+ expect(() => resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toThrow(
+ /Two files declare the "otel" instrumentation provider/,
+ );
+ });
+
+ it("rejects a single instrumentation file, naming the flag", () => {
+ writeInstrumentationFile();
+
+ expect(() => resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toThrow(
+ /experimental\.instrumentationProviders/,
+ );
+ });
+
+ it("prefers the file error when both layouts are present", () => {
+ writeInstrumentationFile();
+ writeInstrumentationProvider("otel.ts");
+
+ expect(() => resolveInstrumentationLayout({ agentRoot, providersEnabled: true })).toThrow(
+ /Move it into the "instrumentation\/" directory/,
+ );
+ });
+});
diff --git a/packages/eve/src/internal/instrumentation-layout.ts b/packages/eve/src/internal/instrumentation-layout.ts
new file mode 100644
index 000000000..6c46082d7
--- /dev/null
+++ b/packages/eve/src/internal/instrumentation-layout.ts
@@ -0,0 +1,119 @@
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+import { createLogger } from "#internal/logging.js";
+
+const INSTRUMENTATION_EXTENSIONS = [".ts", ".mts", ".js", ".mjs"] as const;
+
+const INSTRUMENTATION_DIRECTORY = "instrumentation";
+
+const PROVIDERS_FLAG = "experimental.instrumentationProviders";
+const log = createLogger("internal.instrumentation-layout");
+
+/**
+ * How instrumentation is authored for one agent.
+ *
+ * `file` is the single `agent/instrumentation.ts` default export. `directory`
+ * contains one provider per file, keyed by the slot name the
+ * file derives (`instrumentation/otel.ts` → `otel`). Which one an agent may use
+ * is decided by `experimental.instrumentationProviders`, never by what happens
+ * to be on disk.
+ */
+export type InstrumentationLayout =
+ | { readonly kind: "file"; readonly modulePath: string }
+ | { readonly kind: "directory"; readonly modulePathsBySlot: Readonly<Record<string, string>> };
+
+/**
+ * Resolves the instrumentation layout for one agent root.
+ *
+ * With providers on, an empty directory layout still installs eve's built-in
+ * destinations. Throws when the layout on disk is not the one the flag selects:
+ * the wrong layout would otherwise be skipped silently, and telemetry that
+ * quietly does nothing is the failure this whole surface exists to prevent.
+ */
+export function resolveInstrumentationLayout(input: {
+ readonly agentRoot: string;
+ readonly providersEnabled: boolean;
+}): InstrumentationLayout | undefined {
+ const filePath = resolveInstrumentationFile(input.agentRoot);
+ const directoryPath = join(input.agentRoot, INSTRUMENTATION_DIRECTORY);
+ const hasDirectory = existsSync(directoryPath) && statSync(directoryPath).isDirectory();
+
+ if (!input.providersEnabled) {
+ if (hasDirectory) {
+ log.warn("ignoring instrumentation provider directory because providers are off", {
+ agentRoot: input.agentRoot,
+ flag: PROVIDERS_FLAG,
+ });
+ }
+
+ return filePath === undefined ? undefined : { kind: "file", modulePath: filePath };
+ }
+
+ if (filePath !== undefined) {
+ throw new Error(
+ `Found "${filePath}", but \`${PROVIDERS_FLAG}\` is on. Move it into the "${INSTRUMENTATION_DIRECTORY}/" directory as one file per provider.`,
+ );
+ }
+
+ if (!hasDirectory) {
+ return { kind: "directory", modulePathsBySlot: {} };
+ }
+
+ return {
+ kind: "directory",
+ modulePathsBySlot: collectInstrumentationProviderModules(directoryPath),
+ };
+}
+
+/**
+ * Maps each `instrumentation/<slot>.<ext>` file to its absolute path.
+ *
+ * Slots are sorted so the registration order a provider sees does not depend on
+ * how the filesystem happens to enumerate the directory.
+ */
+function collectInstrumentationProviderModules(
+ directoryPath: string,
+): Readonly<Record<string, string>> {
+ const modulePathsBySlot = new Map<string, string>();
+
+ for (const entry of readdirSync(directoryPath, { withFileTypes: true })) {
+ if (!entry.isFile()) continue;
+
+ const extension = INSTRUMENTATION_EXTENSIONS.find((candidate) =>
+ entry.name.endsWith(candidate),
+ );
+ if (extension === undefined) continue;
+
+ const slot = entry.name.slice(0, -extension.length);
+ if (slot === "") continue;
+
+ const existing = modulePathsBySlot.get(slot);
+ if (existing !== undefined) {
+ throw new Error(
+ `Two files declare the "${slot}" instrumentation provider in "${directoryPath}". Keep one of them.`,
+ );
+ }
+
+ modulePathsBySlot.set(slot, join(directoryPath, entry.name));
+ }
+
+ return Object.fromEntries(
+ [...modulePathsBySlot].sort(([left], [right]) => left.localeCompare(right)),
+ );
+}
+
+/**
+ * Resolves the single `agent/instrumentation` module, ignoring any directory of
+ * the same name.
+ */
+function resolveInstrumentationFile(agentRoot: string): string | undefined {
+ for (const extension of INSTRUMENTATION_EXTENSIONS) {
+ const candidate = join(agentRoot, `${INSTRUMENTATION_DIRECTORY}${extension}`);
+ if (existsSync(candidate)) {
+ return candidate;
+ }
+ }
+
+ return undefined;
+}
diff --git a/packages/eve/src/internal/materialized-authored-modules.ts b/packages/eve/src/internal/materialized-authored-modules.ts
index 7c1376080..6967733cf 100644
--- a/packages/eve/src/internal/materialized-authored-modules.ts
+++ b/packages/eve/src/internal/materialized-authored-modules.ts
@@ -10,16 +10,24 @@ import {
bundleAuthoredModuleMapForGeneration,
} from "#internal/authored-module-loader.js";
import { serializeCompiledManifestForFingerprint } from "#internal/compiled-manifest-fingerprint.js";
+import { resolveInstrumentationLayout } from "#internal/instrumentation-layout.js";
const MATERIALIZED_MODULES_DIRECTORY = "authored-modules";
const MATERIALIZED_MODULES_INDEX = "authored-modules.json";
-const INSTRUMENTATION_EXTENSIONS = [".ts", ".mts", ".js", ".mjs"] as const;
+
+/**
+ * The materialized instrumentation modules, mirroring the layout they were
+ * authored in. Paths are relative to `.eve/compile`.
+ */
+export type MaterializedInstrumentation =
+ | { readonly kind: "file"; readonly modulePath: string }
+ | { readonly kind: "directory"; readonly modulePathsBySlot: Readonly<Record<string, string>> };
export interface MaterializedAuthoredModuleIndex {
readonly fingerprint: string;
- readonly instrumentation?: string;
+ readonly instrumentation?: MaterializedInstrumentation;
readonly moduleMap: string;
- readonly version: 2;
+ readonly version: 3;
}
export async function materializeAuthoredModules(input: {
@@ -54,22 +62,40 @@ export async function materializeAuthoredModules(input: {
await writeFile(join(modulesRoot, moduleMapFileName), moduleMapCode);
fingerprint.update("module-map\0").update(moduleMapCode).update("\0");
- const instrumentation = resolveInstrumentationModule(manifest.agentRoot);
- let instrumentationPath: string | undefined;
-
- if (instrumentation !== undefined) {
- const code = await bundleAuthoredModuleForGeneration(instrumentation, {
- externalDependencies: manifest.config.build?.externalDependencies ?? [],
- });
+ const layout = resolveInstrumentationLayout({
+ agentRoot: manifest.agentRoot,
+ providersEnabled: manifest.config.experimental?.instrumentationProviders ?? false,
+ });
+ const externalDependencies = manifest.config.build?.externalDependencies ?? [];
+ const materializeInstrumentationModule = async (
+ sourceId: string,
+ sourcePath: string,
+ ): Promise<string> => {
+ const code = await bundleAuthoredModuleForGeneration(sourcePath, { externalDependencies });
const fileName = createMaterializedModuleFileName(
ROOT_COMPILED_AGENT_NODE_ID,
- "instrumentation",
+ `instrumentation:${sourceId}`,
code,
);
await writeFile(join(modulesRoot, fileName), code);
- instrumentationPath = join(MATERIALIZED_MODULES_DIRECTORY, fileName);
- fingerprint.update("instrumentation\0").update(code).update("\0");
+ fingerprint.update(`instrumentation:${sourceId}\0`).update(code).update("\0");
+ return join(MATERIALIZED_MODULES_DIRECTORY, fileName);
+ };
+
+ let instrumentation: MaterializedInstrumentation | undefined;
+
+ if (layout?.kind === "file") {
+ instrumentation = {
+ kind: "file",
+ modulePath: await materializeInstrumentationModule("file", layout.modulePath),
+ };
+ } else if (layout?.kind === "directory") {
+ const modulePathsBySlot: Record<string, string> = {};
+ for (const [slot, sourcePath] of Object.entries(layout.modulePathsBySlot)) {
+ modulePathsBySlot[slot] = await materializeInstrumentationModule(slot, sourcePath);
+ }
+ instrumentation = { kind: "directory", modulePathsBySlot };
}
await hashDirectoryIfPresent({
@@ -79,16 +105,16 @@ export async function materializeAuthoredModules(input: {
});
const index: {
fingerprint: string;
- instrumentation?: string;
+ instrumentation?: MaterializedInstrumentation;
moduleMap: string;
- version: 2;
+ version: 3;
} = {
fingerprint: fingerprint.digest("hex"),
moduleMap: moduleMapPath,
- version: 2,
+ version: 3,
};
- if (instrumentationPath !== undefined) {
- index.instrumentation = instrumentationPath;
+ if (instrumentation !== undefined) {
+ index.instrumentation = instrumentation;
}
await writeFile(join(compileRoot, MATERIALIZED_MODULES_INDEX), `${JSON.stringify(index)}\n`);
return index;
@@ -106,12 +132,12 @@ export async function readMaterializedAuthoredModuleIndex(
await readFile(indexPath, "utf8"),
) as Partial<MaterializedAuthoredModuleIndex>;
if (
- parsed.version !== 2 ||
+ parsed.version !== 3 ||
typeof parsed.fingerprint !== "string" ||
parsed.fingerprint.length === 0 ||
typeof parsed.moduleMap !== "string" ||
parsed.moduleMap.length === 0 ||
- (parsed.instrumentation !== undefined && typeof parsed.instrumentation !== "string")
+ !isMaterializedInstrumentation(parsed.instrumentation)
) {
throw new Error(`Invalid materialized authored module index at "${indexPath}".`);
}
@@ -119,6 +145,26 @@ export async function readMaterializedAuthoredModuleIndex(
return parsed as MaterializedAuthoredModuleIndex;
}
+function isMaterializedInstrumentation(value: unknown): boolean {
+ if (value === undefined) return true;
+ if (typeof value !== "object" || value === null) return false;
+
+ const candidate = value as Partial<MaterializedInstrumentation>;
+ if (candidate.kind === "file") {
+ return typeof (candidate as { modulePath?: unknown }).modulePath === "string";
+ }
+ if (candidate.kind === "directory") {
+ const paths = (candidate as { modulePathsBySlot?: unknown }).modulePathsBySlot;
+ return (
+ typeof paths === "object" &&
+ paths !== null &&
+ Object.values(paths).every((path) => typeof path === "string")
+ );
+ }
+
+ return false;
+}
+
async function readCompiledManifest(path: string): Promise<CompiledAgentManifest> {
const manifest = JSON.parse(await readFile(path, "utf8")) as CompiledAgentManifest;
@@ -129,17 +175,6 @@ async function readCompiledManifest(path: string): Promise<CompiledAgentManifest
return manifest;
}
-function resolveInstrumentationModule(agentRoot: string): string | undefined {
- for (const extension of INSTRUMENTATION_EXTENSIONS) {
- const candidate = join(agentRoot, `instrumentation${extension}`);
- if (existsSync(candidate)) {
- return candidate;
- }
- }
-
- return undefined;
-}
-
function createMaterializedModuleFileName(nodeId: string, sourceId: string, code: string): string {
return `${createHash("sha256")
.update(nodeId)
diff --git a/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts b/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts
index 8887d0de8..f1add13b6 100644
--- a/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts
+++ b/packages/eve/src/internal/nitro/dev-generation-artifacts.scenario.test.ts
@@ -13,6 +13,7 @@ import {
stageDevelopmentGeneration,
} from "#internal/nitro/development-generation.js";
import { createAuthoredSourceRuntimeCompiledArtifactsSource } from "#internal/application/runtime-compiled-artifacts-source.js";
+import type { MaterializedInstrumentation } from "#internal/materialized-authored-modules.js";
import { useScenarioApp } from "#internal/testing/scenario-app.js";
describe("development generation artifacts", () => {
@@ -489,9 +490,12 @@ describe("development generation artifacts", () => {
join(first.runtimeAppRoot, ".eve", "compile", "authored-modules.json"),
"utf8",
),
- ) as { readonly instrumentation?: string };
+ ) as { readonly instrumentation?: MaterializedInstrumentation };
+ if (firstIndex.instrumentation?.kind !== "file") {
+ throw new Error("expected materialized file instrumentation");
+ }
const materializedInstrumentation = await readFile(
- join(first.runtimeAppRoot, ".eve", "compile", firstIndex.instrumentation!),
+ join(first.runtimeAppRoot, ".eve", "compile", firstIndex.instrumentation.modulePath),
"utf8",
);
diff --git a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts
index 3af429a9b..808b1da10 100644
--- a/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts
+++ b/packages/eve/src/internal/nitro/host/create-application-nitro.scenario.test.ts
@@ -216,6 +216,7 @@ describe("application Nitro creation", () => {
const { createDevelopmentApplicationNitro } =
await import("#internal/nitro/host/create-application-nitro.js");
const preparedHost = createPreparedHost();
+ preparedHost.compiledArtifacts.instrumentationLayout = { kind: "file" };
preparedHost.compiledArtifacts.instrumentationPluginPath = "/app/instrumentation.mjs";
await createDevelopmentApplicationNitro(preparedHost);
@@ -227,6 +228,29 @@ describe("application Nitro creation", () => {
);
});
+ it("lets the provider pipeline own default local tracing", async () => {
+ const { createDevelopmentApplicationNitro } =
+ await import("#internal/nitro/host/create-application-nitro.js");
+
+ for (const slots of ["rows", "local"] as const) {
+ const nitroStub = createNitroStub();
+ createNitroMock.mockResolvedValueOnce(nitroStub.nitro);
+ const preparedHost = createPreparedHost();
+ preparedHost.compiledArtifacts.instrumentationLayout = {
+ kind: "directory",
+ slots: [slots],
+ };
+ preparedHost.compiledArtifacts.instrumentationPluginPath = "/app/instrumentation.mjs";
+
+ await createDevelopmentApplicationNitro(preparedHost);
+
+ const plugins = createNitroMock.mock.calls.at(-1)?.[0].plugins as string[];
+ expect(plugins).not.toEqual(
+ expect.arrayContaining([expect.stringContaining("local-tracing-runtime-plugin.ts")]),
+ );
+ }
+ });
+
it("preserves workflow bundle side effects and skips workflow transform for cached bundles", async () => {
const nitroStub = createNitroStub();
createNitroMock.mockResolvedValueOnce(nitroStub.nitro);
diff --git a/packages/eve/src/internal/nitro/host/create-application-nitro.ts b/packages/eve/src/internal/nitro/host/create-application-nitro.ts
index 06bae42bf..a34a0b577 100644
--- a/packages/eve/src/internal/nitro/host/create-application-nitro.ts
+++ b/packages/eve/src/internal/nitro/host/create-application-nitro.ts
@@ -520,9 +520,11 @@ function addDynamicCapabilityTransformPlugin(nitro: Nitro): void {
*/
function addInstrumentationModuleSideEffectsPlugin(
nitro: Nitro,
- instrumentationModulePath: string,
+ instrumentationModulePaths: readonly string[],
): void {
- const normalizedInstrumentationModulePath = normalizePath(instrumentationModulePath);
+ const normalizedInstrumentationModulePaths = new Set(
+ instrumentationModulePaths.map(normalizePath),
+ );
nitro.hooks.hook("rollup:before", (_nitro, config) => {
if (!Array.isArray(config.plugins)) {
@@ -532,7 +534,7 @@ function addInstrumentationModuleSideEffectsPlugin(
config.plugins.unshift({
name: "eve:instrumentation-module-side-effects",
resolveId(source: string) {
- if (normalizePath(source) !== normalizedInstrumentationModulePath) {
+ if (!normalizedInstrumentationModulePaths.has(normalizePath(source))) {
return null;
}
@@ -682,10 +684,10 @@ function configureSharedApplicationNitro(
addDynamicCapabilityTransformPlugin(nitro);
- if (preparedHost.compiledArtifacts.instrumentationSourcePath !== undefined) {
+ if (preparedHost.compiledArtifacts.instrumentationSourcePaths !== undefined) {
addInstrumentationModuleSideEffectsPlugin(
nitro,
- preparedHost.compiledArtifacts.instrumentationSourcePath,
+ preparedHost.compiledArtifacts.instrumentationSourcePaths,
);
}
}
diff --git a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts
index 62e5d3621..b8d096c8b 100644
--- a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts
+++ b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.integration.test.ts
@@ -20,6 +20,7 @@ afterEach(async () => {
interface HostVariant {
readonly channels?: CompiledAgentManifest["channels"];
+ readonly instrumentationSlot?: string;
readonly instrumentationSource?: string;
readonly schedules?: CompiledAgentManifest["schedules"];
readonly workflowWorld?: "local" | "vercel";
@@ -55,7 +56,15 @@ async function createHost(variant: HostVariant = {}): Promise<PreparedDevelopmen
appRoot,
compiledArtifacts: {
bootstrapPath: join(appRoot, "bootstrap.mjs"),
- instrumentationSourcePath,
+ ...(instrumentationSourcePath === undefined
+ ? {}
+ : {
+ instrumentationLayout:
+ variant.instrumentationSlot === undefined
+ ? ({ kind: "file" } as const)
+ : ({ kind: "directory", slots: [variant.instrumentationSlot] } as const),
+ instrumentationSourcePaths: [instrumentationSourcePath],
+ }),
workflowWorldPluginPath: join(appRoot, "workflow-world.mjs"),
},
compileResult: { manifest } as PreparedDevelopmentApplicationHost["compileResult"],
@@ -81,6 +90,18 @@ describe("computeDevelopmentHostFingerprint", () => {
expect(changed).not.toBe(base);
});
+ it("treats provider slot names as structural", async () => {
+ const source = "export default {}\n";
+ const first = await computeDevelopmentHostFingerprint(
+ await createHost({ instrumentationSlot: "a", instrumentationSource: source }),
+ );
+ const renamed = await computeDevelopmentHostFingerprint(
+ await createHost({ instrumentationSlot: "b", instrumentationSource: source }),
+ );
+
+ expect(renamed).not.toBe(first);
+ });
+
it("treats channel route topology as structural", async () => {
const base = await computeDevelopmentHostFingerprint(await createHost());
const withRoute = await computeDevelopmentHostFingerprint(
diff --git a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts
index 5206c23f5..2b27a8a02 100644
--- a/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts
+++ b/packages/eve/src/internal/nitro/host/dev-host-fingerprint.ts
@@ -50,12 +50,21 @@ export async function computeDevelopmentHostFingerprint(
return createHash("sha256").update(JSON.stringify(payload)).digest("hex");
}
-async function readInstrumentationSource(
- host: PreparedDevelopmentApplicationHost,
-): Promise<string | null> {
- const path = host.compiledArtifacts.instrumentationSourcePath;
- if (path === undefined) {
+async function readInstrumentationSource(host: PreparedDevelopmentApplicationHost): Promise<{
+ readonly kind: "directory" | "file";
+ readonly modules: readonly { readonly slot: string | null; readonly source: string }[];
+} | null> {
+ const paths = host.compiledArtifacts.instrumentationSourcePaths;
+ const layout = host.compiledArtifacts.instrumentationLayout;
+ if (paths === undefined || layout === undefined) {
return null;
}
- return await readFile(path, "utf8");
+ const sources = await Promise.all(paths.map(async (path) => await readFile(path, "utf8")));
+ return {
+ kind: layout.kind,
+ modules: sources.map((source, index) => ({
+ slot: layout.kind === "directory" ? (layout.slots[index] ?? null) : null,
+ source,
+ })),
+ };
}
diff --git a/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts b/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts
index 4f0612f9e..e36d53b59 100644
--- a/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts
+++ b/packages/eve/src/internal/nitro/host/prepare-application-host.scenario.test.ts
@@ -108,9 +108,9 @@ describe("application host preparation", () => {
join(firstHostDirectory, "compiled-artifacts-workflow-world.mjs"),
);
expect(firstHost.compiledArtifacts.bootstrapPath).not.toContain("/.eve/dev-runtime/snapshots/");
- expect(firstHost.compiledArtifacts.instrumentationSourcePath).toBe(
+ expect(firstHost.compiledArtifacts.instrumentationSourcePaths).toEqual([
join(firstHostDirectory, "compiled-artifacts-instrumentation-source.mjs"),
- );
+ ]);
expect(await readFile(firstBootstrapPath, "utf8")).not.toContain(
normalizeEsmImportSpecifier(agentModulePath),
);
diff --git a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts
index 9617df9c6..48752f804 100644
--- a/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts
+++ b/packages/eve/src/internal/nitro/routes/channel-request-instrumentation.ts
@@ -58,9 +58,8 @@ export interface TraceChannelRequestInput {
* waiting for `event.waitUntil()` work or streamed response bodies.
*
* Emitting these spans is opt-in: unless authored instrumentation enables it
- * via `defineInstrumentation({ traceChannelRequests: true })`, the handler runs
- * with no span (`undefined`) and no context extraction — a true bypass, not a
- * non-recording span.
+ * via `traceChannelRequests: true`, the handler runs with no span (`undefined`)
+ * and no context extraction — a true bypass, not a non-recording span.
*
* This is observability-only: it never changes the response and performs no
* synchronous span export in the request path, adding only minimal in-process
diff --git a/packages/eve/src/public/definitions/exact.test.ts b/packages/eve/src/public/definitions/exact.test.ts
index d6aa982e6..dd1bf1d4e 100644
--- a/packages/eve/src/public/definitions/exact.test.ts
+++ b/packages/eve/src/public/definitions/exact.test.ts
@@ -221,11 +221,18 @@ function typeOnlyFixtures(): void {
// @ts-expect-error Instructions identity is path-derived.
defineInstructions(instructionsWithName);
+ defineInstrumentation({
+ isEnabled: true,
+ recordInputs: true,
+ });
+
+ // Unlike the helpers above, `defineInstrumentation` takes a generic union — a
+ // config and a provider overlap on `events` and `setup` — so it cannot use
+ // `ExactDefinition`. Excess keys reach `eve build` instead.
const instrumentationWithEnabled = {
isEnabled: true,
recordInputs: true,
};
- // @ts-expect-error Instrumentation has no separate enable toggle.
defineInstrumentation(instrumentationWithEnabled);
defineInstrumentation({
diff --git a/packages/eve/src/public/instrumentation/index.ts b/packages/eve/src/public/instrumentation/index.ts
index a5d38f09d..85be1b8da 100644
--- a/packages/eve/src/public/instrumentation/index.ts
+++ b/packages/eve/src/public/instrumentation/index.ts
@@ -1,15 +1,21 @@
-import type { ExactDefinition } from "#public/definitions/exact.js";
-
/**
- * Instrumentation authoring helpers for `agent/instrumentation.ts`.
+ * Instrumentation authoring helpers for `agent/instrumentation.ts` and, with
+ * `experimental.instrumentationProviders` on, `agent/instrumentation/`.
*/
import type { ModelMessage, SystemModelMessage } from "ai";
import type { SessionAuthContext, SessionParent } from "#channel/types.js";
import type { InstrumentationChannel } from "#public/channels/index.js";
+import {
+ PROVIDER,
+ type ProviderDefinition,
+ type ProviderSetupContext,
+} from "#public/instrumentation/provider.js";
import type { JsonObject } from "#shared/json.js";
+export * from "#public/instrumentation/provider.js";
+
// Re-export channel metadata types so existing `eve/instrumentation`
// imports continue to work. The canonical home is `eve/channels`.
export {
@@ -23,14 +29,13 @@ export {
/**
* Context passed to the {@link InstrumentationDefinition.setup} callback.
+ *
+ * The same context both layouts receive. Keeping one type is what gives
+ * {@link defineInstrumentation}'s union a contextual signature for `setup`;
+ * two divergent ones would leave every authored `setup(context)` parameter an
+ * implicit `any`.
*/
-export interface InstrumentationSetupContext {
- /**
- * The agent name declared by `defineAgent`. Use as the `serviceName` for
- * `registerOTel` instead of a hard-coded string.
- */
- readonly agentName: string;
-}
+export interface InstrumentationSetupContext extends ProviderSetupContext {}
/**
* User-authored runtime context values attached to AI SDK telemetry spans.
@@ -161,21 +166,39 @@ export interface InstrumentationDefinition {
*/
readonly traceChannelRequests?: boolean;
/**
- * Setup callback invoked at server startup with the resolved agent name.
- * Use it to call `registerOTel` or other OTel provider setup;
- * `context.agentName` comes from `defineAgent`.
+ * Setup callback invoked at server startup, before the first request. Use it
+ * to call `registerOTel` or other OTel provider setup; `context.agentName`
+ * comes from `defineAgent`. A returned promise is awaited.
*/
- readonly setup?: (context: InstrumentationSetupContext) => void;
+ readonly setup?: (context: InstrumentationSetupContext) => void | PromiseLike<void>;
}
/**
- * Export the result as the default export of `agent/instrumentation.ts`. eve
- * reads these settings at server startup and applies them to every AI SDK
- * model call. The `setup` callback runs later with the resolved agent name,
- * not during `defineInstrumentation` itself.
+ * Declares instrumentation, in either of eve's two layouts.
+ *
+ * Export the result as the default export of `agent/instrumentation.ts`, or —
+ * with `experimental.instrumentationProviders` on — of one file under
+ * `agent/instrumentation/`. The layout decides how eve reads the value; the two
+ * are mutually exclusive builds, so only one can apply. `setup` runs at server
+ * startup, not during this call.
+ *
+ * The parameter is a union because a provider and a legacy config overlap on
+ * `events` and `setup`, so no value-level check separates them. One consequence
+ * is that excess-property checking is weaker here than it was against the
+ * config shape alone, and a misspelled key can reach `eve build` rather than
+ * failing at `tsc`.
*/
-export function defineInstrumentation<T extends InstrumentationDefinition>(
- definition: ExactDefinition<T, InstrumentationDefinition>,
-): T {
- return definition;
+export function defineInstrumentation<
+ const TDefinition extends InstrumentationDefinition | ProviderDefinition,
+>(definition: TDefinition): InstrumentationDeclaration<TDefinition> {
+ return { ...definition, [PROVIDER]: true };
}
+
+/** The branded result of {@link defineInstrumentation}. */
+export type InstrumentationDeclaration<
+ TDefinition extends InstrumentationDefinition | ProviderDefinition =
+ | InstrumentationDefinition
+ | ProviderDefinition,
+> = TDefinition & {
+ readonly [PROVIDER]: true;
+};
diff --git a/packages/eve/src/public/instrumentation/otel.ts b/packages/eve/src/public/instrumentation/otel.ts
new file mode 100644
index 000000000..f5beb31ed
--- /dev/null
+++ b/packages/eve/src/public/instrumentation/otel.ts
@@ -0,0 +1,59 @@
+/**
+ * The OpenTelemetry authoring surface for `agent/instrumentation/`.
+ *
+ * Two halves, because OpenTelemetry has two: `otel()` is the settings a process
+ * can only hold one of, and an integration is a destination, of which there may
+ * be as many as there are files.
+ *
+ * Reachable only with `experimental.instrumentationProviders` on. With the flag
+ * off nothing discovers that directory, so these compile but never run.
+ */
+
+import { createLocalTracesProcessor, resolveLocalTracesContent } from "#tracing/local-traces.js";
+import {
+ agentRunsIntegration,
+ otelIntegration,
+ type ContentOptions,
+ type OtelIntegration,
+} from "#tracing/otel-declaration.js";
+
+export {
+ isOtelDeclaration,
+ isOtelIntegration,
+ otel,
+ otelIntegration,
+ type ContentOptions,
+ type OtelDeclaration,
+ type OtelIntegration,
+ type OtelIntegrationOptions,
+ type OtelOptions,
+} from "#tracing/otel-declaration.js";
+
+export type { SpanExporter, SpanProcessor } from "#compiled/@vercel/otel/index.js";
+
+/**
+ * Vercel Agent Runs, enabled by default in production.
+ *
+ * Export it from `agent/instrumentation/agent-runs.ts` to narrow content, or
+ * export `disableInstrumentation()` from that file to turn it off.
+ */
+export function agentRuns(options: ContentOptions = {}): OtelIntegration {
+ return agentRunsIntegration(options);
+}
+
+/**
+ * The local trace spool `eve dev` reads, as a destination.
+ *
+ * Export it from `agent/instrumentation/local.ts` to keep it alongside a hosted
+ * backend, or export `disableInstrumentation()` from that file to turn it off.
+ * Omitting the file leaves eve's default in place.
+ *
+ * `EVE_TRACES_CONTENT=off` narrows this destination and no other, so declining
+ * content locally leaves what a hosted backend receives alone.
+ */
+export function localTraces(options: ContentOptions = {}): OtelIntegration {
+ return otelIntegration({
+ ...resolveLocalTracesContent(options),
+ spanProcessors: [createLocalTracesProcessor()],
+ });
+}
diff --git a/packages/eve/src/public/instrumentation/provider.test.ts b/packages/eve/src/public/instrumentation/provider.test.ts
new file mode 100644
index 000000000..0fe3e5108
--- /dev/null
+++ b/packages/eve/src/public/instrumentation/provider.test.ts
@@ -0,0 +1,92 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ defineInstrumentation,
+ disableInstrumentation,
+ isInstrumentationDisabled,
+ isInstrumentationProvider,
+ type InstrumentationSetupContext,
+} from "#public/instrumentation/index.js";
+
+describe("defineInstrumentation", () => {
+ it("keeps the legacy setup context constructible with only the agent name", () => {
+ const context: InstrumentationSetupContext = { agentName: "weather" };
+
+ expect(context).toEqual({ agentName: "weather" });
+ });
+
+ it("brands a provider-shaped declaration", () => {
+ const provider = defineInstrumentation({
+ events: {
+ "session.started"(event) {
+ void event.sessionId;
+ },
+ },
+ });
+
+ expect(isInstrumentationProvider(provider)).toBe(true);
+ });
+
+ it("brands a legacy config-shaped declaration", () => {
+ const config = defineInstrumentation({ functionId: "support", recordInputs: false });
+
+ expect(isInstrumentationProvider(config)).toBe(true);
+ expect(config.functionId).toBe("support");
+ expect(config).toMatchObject({ functionId: "support", recordInputs: false });
+ });
+
+ it("infers terminal handler events from union-typed discriminants", () => {
+ const provider = defineInstrumentation({
+ events: {
+ "action.completed": (event) => void [event.acceptedAtMs, event.outcome, event.usage],
+ "action.failed": (event) => void [event.acceptedAtMs, event.errorCode, event.outcome],
+ "session.completed": (event) => void event.sessionId,
+ "step.attempt.completed": (event) => void event.scope,
+ "turn.failed": (event) => void event.error,
+ },
+ });
+
+ expect(isInstrumentationProvider(provider)).toBe(true);
+ });
+
+ it("exposes durable input request and resolution events", () => {
+ const provider = defineInstrumentation({
+ events: {
+ "input.requested": (event) => void event.action.callId,
+ "input.resolved": (event) => void event.outcome,
+ },
+ });
+
+ expect(isInstrumentationProvider(provider)).toBe(true);
+ });
+
+ it("preserves the authored fields", () => {
+ const setup = (): void => {};
+ const declaration = defineInstrumentation({ setup });
+
+ expect(declaration).toMatchObject({ setup });
+ });
+});
+
+describe("isInstrumentationProvider", () => {
+ it("rejects a value that never went through defineInstrumentation", () => {
+ expect(isInstrumentationProvider({ events: {} })).toBe(false);
+ });
+
+ it.each([[null], [undefined], ["provider"], [42]])("rejects %p", (value) => {
+ expect(isInstrumentationProvider(value)).toBe(false);
+ });
+});
+
+describe("disableInstrumentation", () => {
+ it("is recognizable as a disabled slot rather than a provider", () => {
+ const disabled = disableInstrumentation();
+
+ expect(isInstrumentationDisabled(disabled)).toBe(true);
+ expect(isInstrumentationProvider(disabled)).toBe(false);
+ });
+
+ it("does not treat a provider as a disabled slot", () => {
+ expect(isInstrumentationDisabled(defineInstrumentation({}))).toBe(false);
+ });
+});
diff --git a/packages/eve/src/public/instrumentation/provider.ts b/packages/eve/src/public/instrumentation/provider.ts
new file mode 100644
index 000000000..dac98d3a7
--- /dev/null
+++ b/packages/eve/src/public/instrumentation/provider.ts
@@ -0,0 +1,189 @@
+/**
+ * The provider contract authored under `agent/instrumentation/`.
+ *
+ * Reachable only with `experimental.instrumentationProviders` on. With the flag
+ * off nothing discovers that directory, so these types compile but never run.
+ */
+
+// Type-only, so nothing couples the public entrypoint to the harness at
+// runtime. The event shapes are eve's own vocabulary; deriving the handler map
+// from the union below is what keeps the public contract from drifting away
+// from the bus that feeds it.
+import type { InstrumentationEvent } from "#harness/instrumentation-lifecycle.js";
+import type { JsonValue } from "#public/types/json.js";
+
+export type { JsonValue } from "#public/types/json.js";
+
+export type {
+ InstrumentationActionCompletedEvent,
+ InstrumentationActionFailedEvent,
+ InstrumentationActionKind,
+ InstrumentationActionOutcome,
+ InstrumentationActionOutput,
+ InstrumentationActionStartedEvent,
+ InstrumentationAttemptScope,
+ InstrumentationContentPart,
+ InstrumentationEvent,
+ InstrumentationInputKind,
+ InstrumentationInputOption,
+ InstrumentationInputOutcome,
+ InstrumentationInputRequest,
+ InstrumentationInputRequestedEvent,
+ InstrumentationInputResolvedEvent,
+ InstrumentationInputResponse,
+ InstrumentationModelCallCompletedEvent,
+ InstrumentationModelCallFailedEvent,
+ InstrumentationModelCallStartedEvent,
+ InstrumentationModelRef,
+ InstrumentationOperationRef,
+ InstrumentationParentLineage,
+ InstrumentationSessionFailedEvent,
+ InstrumentationSessionSettledEvent,
+ InstrumentationSessionStartedEvent,
+ InstrumentationSessionTransitionEvent,
+ InstrumentationStepAttemptMetadataEvent,
+ InstrumentationStepAttemptCompletedEvent,
+ InstrumentationStepAttemptFailedEvent,
+ InstrumentationStepAttemptStartedEvent,
+ InstrumentationStepAttemptTerminalEvent,
+ InstrumentationToolCallCompletedEvent,
+ InstrumentationToolCallFailedEvent,
+ InstrumentationToolCallStartedEvent,
+ InstrumentationToolOutput,
+ InstrumentationTraceContext,
+ InstrumentationTurnFailedEvent,
+ InstrumentationTurnSettledEvent,
+ InstrumentationTurnStartedEvent,
+ InstrumentationTurnTerminalEvent,
+ InstrumentationUsage,
+} from "#harness/instrumentation-lifecycle.js";
+
+/**
+ * Marks a value as having come from `defineInstrumentation` or a built-in
+ * factory.
+ *
+ * It does not say which layout the value belongs to: a provider and a legacy
+ * config both carry `events` and `setup`, so no value-level check separates
+ * them. The layout decides — `agent/instrumentation.ts` is read as a config and
+ * `agent/instrumentation/*.ts` as providers, and the two are mutually exclusive
+ * builds. The brand's job is only to catch a default export that never went
+ * through eve at all.
+ */
+export const PROVIDER = Symbol.for("eve.instrumentation.provider");
+
+/** Marks a slot the author turned off rather than configured. */
+export const DISABLED = Symbol.for("eve.instrumentation.disabled");
+
+/** Where the agent is running when `setup` fires. */
+export type InstrumentationEnvironment = "development" | "preview" | "production";
+
+/** The local eval run this server was started to serve. */
+export interface EvaluationRef {
+ readonly runId: string;
+}
+
+/**
+ * Passed to {@link InstrumentationProvider.setup} once at server startup,
+ * before any event is published.
+ */
+export interface ProviderSetupContext {
+ /** The agent name declared by `defineAgent`. */
+ readonly agentName: string;
+ /** Always supplied at runtime; optional for legacy setup-context compatibility. */
+ readonly environment?: InstrumentationEnvironment;
+ /** Present only when this server was started for a local `eve eval` run. */
+ readonly evaluation?: EvaluationRef;
+ /** The eve version running the agent. */
+ /** Always supplied at runtime; optional for legacy setup-context compatibility. */
+ readonly frameworkVersion?: string;
+}
+
+export interface ProviderState {
+ get(): JsonValue | undefined;
+ /** Stages a JSON value; `undefined` releases this operation's slot. */
+ set(value: JsonValue | undefined): void;
+}
+
+export interface ProviderContext {
+ readonly state: ProviderState;
+}
+
+/**
+ * One event handler.
+ *
+ * A handler can carry durable JSON state from a start to its terminal through
+ * `ctx.state`. eve scopes and releases that state by provider and operation.
+ */
+export type Handler<TEvent> = (event: TEvent, ctx: ProviderContext) => void | PromiseLike<void>;
+
+type EventForType<TEvent, TType> = TEvent extends { readonly type: infer TEventType }
+ ? TType extends TEventType
+ ? TEvent & { readonly type: TType }
+ : never
+ : never;
+
+/**
+ * The events a provider may handle, one optional handler per event type.
+ *
+ * Derived from the event union rather than written out, so a new event reaches
+ * providers the moment the bus can publish it.
+ */
+export type ProviderEvents = {
+ readonly [TType in InstrumentationEvent["type"]]?: Handler<
+ EventForType<InstrumentationEvent, TType>
+ >;
+};
+
+/**
+ * What an author writes for one file under `agent/instrumentation/`.
+ *
+ * Setup runs in slot order, but event handlers across authored providers run
+ * concurrently and are failure-isolated. Do not coordinate providers through
+ * completion order.
+ */
+export interface ProviderDefinition {
+ readonly events?: ProviderEvents;
+ /** Runs once at server startup, before any event is published. */
+ readonly setup?: (context: ProviderSetupContext) => void | PromiseLike<void>;
+ /** Drains anything buffered. eve calls this before a session goes idle. */
+ readonly flush?: () => void | PromiseLike<void>;
+ /** Releases resources when the process is going away. */
+ readonly shutdown?: () => void | PromiseLike<void>;
+}
+
+/** A {@link ProviderDefinition} that has been through `defineInstrumentation`. */
+export type InstrumentationProvider = ProviderDefinition & {
+ readonly [PROVIDER]: true;
+};
+
+/** A slot the author turned off. eve registers nothing for it. */
+export interface InstrumentationDisabled {
+ readonly [DISABLED]: true;
+}
+
+/**
+ * Turns off the slot the file it is exported from names.
+ *
+ * Export it as the default of `agent/instrumentation/local.ts` to stop eve
+ * spooling local traces, for instance. Omitting the file entirely leaves eve's
+ * default in place, which is why turning one off takes a value.
+ */
+export function disableInstrumentation(): InstrumentationDisabled {
+ return { [DISABLED]: true };
+}
+
+export function isInstrumentationProvider(value: unknown): value is InstrumentationProvider {
+ return (
+ typeof value === "object" &&
+ value !== null &&
+ (value as Partial<InstrumentationProvider>)[PROVIDER] === true
+ );
+}
+
+export function isInstrumentationDisabled(value: unknown): value is InstrumentationDisabled {
+ return (
+ typeof value === "object" &&
+ value !== null &&
+ (value as Partial<InstrumentationDisabled>)[DISABLED] === true
+ );
+}
diff --git a/packages/eve/src/runtime/resolve-agent.ts b/packages/eve/src/runtime/resolve-agent.ts
index dae3eaf2e..d26449eb9 100644
--- a/packages/eve/src/runtime/resolve-agent.ts
+++ b/packages/eve/src/runtime/resolve-agent.ts
@@ -204,6 +204,7 @@ function createResolvedAgentConfig(
if (manifest.config.experimental !== undefined) {
config.experimental = {
+ instrumentationProviders: manifest.config.experimental.instrumentationProviders,
subagentPersistentSessions: manifest.config.experimental.subagentPersistentSessions,
workflow:
manifest.config.experimental.workflow === undefined
diff --git a/packages/eve/src/shared/agent-definition.ts b/packages/eve/src/shared/agent-definition.ts
index a42a5c941..a403b7b4b 100644
--- a/packages/eve/src/shared/agent-definition.ts
+++ b/packages/eve/src/shared/agent-definition.ts
@@ -191,6 +191,15 @@ export interface AgentLimitsDefinition {
* These options are unstable and may change or be removed in any release.
*/
export interface AgentExperimentalDefinition {
+ /**
+ * Reads instrumentation from an `instrumentation/` directory of providers
+ * rather than a single `agent/instrumentation.ts` config object.
+ *
+ * The two layouts are mutually exclusive: with this on, an
+ * `agent/instrumentation.ts` is a build error, and with it off, an
+ * `instrumentation/` directory is.
+ */
+ readonly instrumentationProviders?: boolean;
/**
* Keeps this agent's delegated subagent sessions alive after they answer.
* The model can pass `agentId` to a subagent tool to continue a previous
diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.ts
index e5611b1cc..2db1b4ad2 100644
--- a/packages/eve/src/tracing/agent-otel-provider.test.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts
@@ -938,10 +938,10 @@ describe("createAgentOtelInstrumentation", () => {
spanProcessors: [new SimpleSpanProcessor(exporter)],
});
const agentOtel = createAgentOtelInstrumentation({
- recordInputs: false,
- recordOutputs: false,
frameworkVersion: "test",
idGenerator,
+ recordInputs: false,
+ recordOutputs: false,
stateStore: new InMemoryAgentTraceStateStore(),
tracer: provider.getTracer("eve.agent"),
});
diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.ts
index 9ff4b7280..b7347a35d 100644
--- a/packages/eve/src/tracing/agent-otel-provider.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.ts
@@ -58,9 +58,13 @@ interface AttemptSpanState {
}
export interface AgentOtelInstrumentationInput {
- /** Whether any destination requested model prompts and tool inputs. */
+ /**
+ * Whether to write model prompts and tool call inputs onto spans at all.
+ * This is the union across destinations, not one destination's policy: a
+ * destination that declined drops these on its way out instead.
+ */
readonly recordInputs?: boolean;
- /** Whether any destination requested model responses and tool outputs. */
+ /** The same, for model responses and tool call outputs. */
readonly recordOutputs?: boolean;
readonly frameworkVersion: string;
readonly idGenerator: AgentSpanIdGenerator;
@@ -509,7 +513,6 @@ export function createAgentOtelInstrumentation(
return {
hook: {
- name: "eve.otel",
events: {
"action.completed": actions.events["action.completed"],
"action.failed": actions.events["action.failed"],
@@ -535,6 +538,7 @@ export function createAgentOtelInstrumentation(
"turn.failed": onTurnTerminal,
"turn.started": onTurnStarted,
},
+ name: "eve.otel",
},
runInContext(operation, execute) {
const scope = attemptScopes.get(operation.scope.attemptId) ?? operation.scope;
diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts
index 4d65b92a0..ea6824f8d 100644
--- a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts
+++ b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts
@@ -1,10 +1,13 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
+import { ContextContainer, contextStorage } from "#context/container.js";
+import { turnIdempotencyKey } from "#harness/instrumentation-lifecycle.js";
import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js";
import { otelIntegration, collectOtelPipeline } from "#tracing/otel-declaration.js";
-const { forceFlush, shutdown } = vi.hoisted(() => ({
+const { forceFlush, internalTerminalState, shutdown } = vi.hoisted(() => ({
forceFlush: vi.fn(async () => undefined),
+ internalTerminalState: vi.fn(),
shutdown: vi.fn(async () => undefined),
}));
@@ -25,7 +28,17 @@ vi.mock("#tracing/otel-registration.js", async (importOriginal) => {
vi.mock("#tracing/agent-otel-provider.js", () => ({
createAgentOtelInstrumentation: () => ({
- hook: {},
+ hook: {
+ events: {
+ "turn.completed": (_event: unknown, ctx: { state: { get(): unknown } }) => {
+ internalTerminalState(ctx.state.get());
+ },
+ "turn.started": (_event: unknown, ctx: { state: { set(value: string): void } }) => {
+ ctx.state.set("framework");
+ },
+ },
+ name: "eve.otel",
+ },
runInContext: (_operation: unknown, execute: () => PromiseLike<unknown>) => execute(),
}),
}));
@@ -35,6 +48,7 @@ const RUNTIME_GLOBAL_KEY = Symbol.for("eve.instrumentation-runtime");
describe("installInstrumentationRuntime", () => {
beforeEach(() => {
forceFlush.mockClear();
+ internalTerminalState.mockClear();
shutdown.mockClear();
delete (globalThis as Record<symbol, unknown>)[RUNTIME_GLOBAL_KEY];
});
@@ -63,4 +77,44 @@ describe("installInstrumentationRuntime", () => {
expect(shutdown).toHaveBeenCalledOnce();
expect(providerShutdown).toHaveBeenCalledOnce();
});
+
+ it("isolates authored state from an internal provider with the same name", async () => {
+ const authoredTerminalState = vi.fn();
+ const runtime = installInstrumentationRuntime({
+ collected: collectOtelPipeline([otelIntegration()]),
+ frameworkVersion: "test",
+ providers: [
+ {
+ events: {
+ "turn.completed": (_event, ctx) => authoredTerminalState(ctx.state.get()),
+ "turn.started": (_event, ctx) => ctx.state.set("authored"),
+ },
+ name: "eve.otel",
+ stateNamespace: "authored:eve.otel",
+ },
+ ],
+ serviceName: "weather",
+ });
+ const idempotencyKey = turnIdempotencyKey("session-1", "turn-1");
+
+ await contextStorage.run(new ContextContainer(), async () => {
+ await runtime.hooks.publish({
+ idempotencyKey,
+ rootSessionId: "session-1",
+ sequence: 0,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "turn.started",
+ });
+ await runtime.hooks.publish({
+ idempotencyKey,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "turn.completed",
+ });
+ });
+
+ expect(internalTerminalState).toHaveBeenCalledExactlyOnceWith("framework");
+ expect(authoredTerminalState).toHaveBeenCalledExactlyOnceWith("authored");
+ });
});
diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.ts b/packages/eve/src/tracing/install-instrumentation-runtime.ts
index a0dd60bbe..a9acad5d3 100644
--- a/packages/eve/src/tracing/install-instrumentation-runtime.ts
+++ b/packages/eve/src/tracing/install-instrumentation-runtime.ts
@@ -18,14 +18,24 @@ import { registerOtelPipeline, type RegisteredOtelPipeline } from "#tracing/otel
const log = createLogger("tracing.install-instrumentation-runtime");
-/** Installs the bus and the one OpenTelemetry pipeline collected for this process. */
+/**
+ * Installs the process instrumentation runtime around a collected pipeline.
+ *
+ * Both layouts land here. `eve dev`'s zero-config default and an authored
+ * `agent/instrumentation/` directory differ only in where the declared values
+ * came from, so sharing the install keeps them on one runtime path.
+ *
+ * A directory that declared no OpenTelemetry still gets a bus: its providers
+ * see every event, they just have no spans to hang them on.
+ */
export function installInstrumentationRuntime(input: {
readonly collected: CollectedOtel;
readonly frameworkVersion: string;
readonly providers: readonly InstrumentationProviderDefinition[];
readonly serviceName: string;
}): InstrumentationRuntime {
- const providers: InstrumentationProviderDefinition[] = [...input.providers];
+ const serialBefore: InstrumentationProviderDefinition[] = [];
+ const serialAfter: InstrumentationProviderDefinition[] = [];
let otelRuntime: RegisteredOtelPipeline | undefined;
let runInContext: InstrumentationRuntime["runInContext"] = (_operation, execute) => execute();
@@ -42,29 +52,35 @@ export function installInstrumentationRuntime(input: {
stateStore: new ContextAgentTraceStateStore(),
tracer: trace.getTracer("eve.agent", input.frameworkVersion),
});
- providers.unshift(agentOtel.hook);
+ // The span must exist before authored providers observe the lifecycle event.
+ serialBefore.push({ ...agentOtel.hook, stateNamespace: "internal:otel" });
runInContext = agentOtel.runInContext;
const releasable = input.collected.pipeline.spanProcessors
.filter(isSpanProcessor)
.filter(hasSessionRelease);
- if (releasable.length > 0) providers.push(sessionReleaseProvider(releasable));
+ if (releasable.length > 0) serialAfter.push(sessionReleaseProvider(releasable));
}
+ const allProviders = [...serialBefore, ...input.providers, ...serialAfter];
let shutdown: Promise<void> | undefined;
return registerInstrumentationRuntime({
forceFlush: () =>
settleAll([
...(otelRuntime === undefined ? [] : [otelRuntime.forceFlush]),
- ...providers.map((provider) => () => provider.flush?.()),
+ ...allProviders.map((provider) => () => provider.flush?.()),
]),
- hooks: createInstrumentationHooks(providers),
+ hooks: createInstrumentationHooks({
+ parallel: input.providers,
+ serialAfter,
+ serialBefore,
+ }),
otelSettings: input.collected.declared ? input.collected.settings : undefined,
runInContext,
shutdown: () => {
shutdown ??= settleAll([
...(otelRuntime === undefined ? [] : [otelRuntime.shutdown]),
- ...providers.map((provider) => () => provider.shutdown?.()),
+ ...allProviders.map((provider) => () => provider.shutdown?.()),
]);
return shutdown;
},
@@ -84,6 +100,7 @@ function sessionReleaseProvider(
return {
events: { "session.completed": release, "session.failed": release },
name: "eve.session-release",
+ stateNamespace: "internal:session-release",
};
}
diff --git a/packages/eve/src/tracing/local-traces.test.ts b/packages/eve/src/tracing/local-traces.test.ts
index 2128084f4..fb8e1b50a 100644
--- a/packages/eve/src/tracing/local-traces.test.ts
+++ b/packages/eve/src/tracing/local-traces.test.ts
@@ -1,6 +1,7 @@
import { afterEach, describe, expect, it, vi } from "vitest";
-import { createLocalTracesProcessor, resolveLocalTracesContent } from "#tracing/local-traces.js";
+import { createLocalTracesProcessor } from "#tracing/local-traces.js";
+import { localTraces } from "#public/instrumentation/otel.js";
vi.mock("#tracing/local-trace-span-processor.js", () => ({
LocalTraceSpanProcessor: class {
@@ -54,23 +55,11 @@ describe("createLocalTracesProcessor", () => {
it("is inert outside a development worker", async () => {
vi.stubEnv("EVE_DEV_WORKER_APP_ROOT", undefined);
- const processor = createLocalTracesProcessor();
+ const [processor] = localTraces().spanProcessors;
+ if (processor === undefined || processor === "auto") throw new Error("Expected a processor.");
expect(() => processor.onEnd(agentSpan("session-one", "a".repeat(32)))).not.toThrow();
await expect(processor.forceFlush()).resolves.toBeUndefined();
await expect(processor.shutdown()).resolves.toBeUndefined();
});
-
- it("lets the environment override narrow only this destination", () => {
- vi.stubEnv("EVE_TRACES_CONTENT", "off");
- expect(resolveLocalTracesContent()).toEqual({ recordInputs: false, recordOutputs: false });
- });
-
- it("preserves explicit destination narrowing when content is enabled", () => {
- vi.stubEnv("EVE_TRACES_CONTENT", "on");
- expect(resolveLocalTracesContent({ recordInputs: false })).toEqual({
- recordInputs: false,
- recordOutputs: true,
- });
- });
});
diff --git a/packages/eve/src/tracing/local-traces.ts b/packages/eve/src/tracing/local-traces.ts
index 8500f7d1e..83ca4591f 100644
--- a/packages/eve/src/tracing/local-traces.ts
+++ b/packages/eve/src/tracing/local-traces.ts
@@ -23,7 +23,15 @@ export interface LocalTracesProcessor extends SpanProcessor {
releaseSession(sessionId: string): Promise<boolean>;
}
-/** Whether a processor still exposes the local spool's session lifecycle. */
+/**
+ * Reports whether a processor tracks which session owns which trace, so eve can
+ * tell it when that session is done.
+ *
+ * Anything standing between eve and the spool has to answer for the spool, so
+ * this is the check a wrapper uses to decide whether it must forward the call.
+ *
+ * @internal
+ */
export function hasSessionRelease(processor: SpanProcessor): processor is LocalTracesProcessor {
return typeof (processor as Partial<LocalTracesProcessor>).releaseSession === "function";
}
@@ -35,7 +43,8 @@ export function hasSessionRelease(processor: SpanProcessor): processor is LocalT
* to observe spans to track which session owns which trace.
*
* Internal because of `releaseSession`, which eve's runtime drives off session
- * lifecycle.
+ * lifecycle. The authored surface is `localTraces()`, which wraps this in an
+ * `OtelIntegration`.
*/
export function createLocalTracesProcessor(
input: { readonly appRoot?: string } = {},
@@ -80,7 +89,16 @@ export function createLocalTracesProcessor(
};
}
-/** Intersects the local destination policy with `EVE_TRACES_CONTENT`. */
+/**
+ * The local spool's content policy: its options, intersected with
+ * `EVE_TRACES_CONTENT`.
+ *
+ * The variable used to be the process-wide switch. It now applies to this one
+ * destination, and only ever narrows — `off` still wins where it applies, but
+ * it no longer decides what a hosted backend beside it receives.
+ *
+ * @internal
+ */
export function resolveLocalTracesContent(
options: {
readonly recordInputs?: boolean;
diff --git a/packages/eve/src/tracing/otel-declaration.test.ts b/packages/eve/src/tracing/otel-declaration.test.ts
index 0487cbcdf..09a8eed6d 100644
--- a/packages/eve/src/tracing/otel-declaration.test.ts
+++ b/packages/eve/src/tracing/otel-declaration.test.ts
@@ -61,7 +61,9 @@ describe("otelIntegration", () => {
expect(otelIntegration().content).toStrictEqual({ recordInputs: true, recordOutputs: true });
});
- it("puts a declined policy in front of every processor", () => {
+ // An author's own processor is part of this destination, and the point of
+ // declining is that nothing under it sees what was said.
+ it("puts a declined policy in front of every processor, an author's included", () => {
const first = processor();
const integration = otelIntegration({ recordOutputs: false, spanProcessors: [first] });
@@ -135,12 +137,16 @@ describe("collectOtelPipeline", () => {
});
expect(collected.settings).toStrictEqual({
functionId: "weather",
+ // Nothing declared a destination, so nothing asked for content.
recordInputs: false,
recordOutputs: false,
traceChannelRequests: true,
});
});
+ // Content governs what is written onto the span, which is upstream of every
+ // destination — so one that wants it is enough, and the ones that declined
+ // drop it on their own way out.
it("takes content capture as the union across destinations", () => {
const collected = collectOtelPipeline([
otelIntegration({ recordInputs: false, recordOutputs: false }),
diff --git a/packages/eve/src/tracing/otel-declaration.ts b/packages/eve/src/tracing/otel-declaration.ts
index b603e64cd..abfd25614 100644
--- a/packages/eve/src/tracing/otel-declaration.ts
+++ b/packages/eve/src/tracing/otel-declaration.ts
@@ -6,6 +6,7 @@ import type {
SpanProcessorOrName,
} from "#compiled/@vercel/otel/index.js";
+import { PROVIDER, type InstrumentationProvider } from "#public/instrumentation/provider.js";
import { batchSpanProcessor } from "#tracing/batch-span-processor.js";
import type { ResolvedContentOptions } from "#tracing/content-attributes.js";
import { contentFilteringProcessor } from "#tracing/content-span-processor.js";
@@ -50,7 +51,15 @@ export interface OtelOptions {
readonly propagators?: readonly PropagatorOrName[];
}
-/** What one destination records of the conversation itself. */
+/**
+ * What one destination records of the conversation itself.
+ *
+ * Declining is per destination, not per process: content is written onto the
+ * span if any destination wants it, and one that declined never exports it. So
+ * an agent whose every destination declines still never materializes a prompt —
+ * the union of nothing is nothing — but a local spool and a hosted backend no
+ * longer have to agree.
+ */
export interface ContentOptions {
/** Record model prompts and tool call inputs. Defaults to `true`. */
readonly recordInputs?: boolean;
@@ -73,14 +82,15 @@ const OTEL_INTEGRATION = Symbol.for("eve.instrumentation.otel-integration");
* The declared OpenTelemetry pipeline settings. eve collects this before
* building the tracer provider, so it is a value rather than a side effect.
*/
-export interface OtelDeclaration {
+export interface OtelDeclaration extends InstrumentationProvider {
readonly [OTEL_DECLARATION]: true;
readonly options: OtelOptions;
}
/** One declared destination. A process may have as many as it has files. */
-export interface OtelIntegration {
+export interface OtelIntegration extends InstrumentationProvider {
readonly [OTEL_INTEGRATION]: true;
+ /** Resolved from `ContentOptions`, so the union does not re-apply defaults. */
readonly content: ResolvedContentOptions;
readonly spanProcessors: readonly SpanProcessorOrName[];
}
@@ -88,10 +98,12 @@ export interface OtelIntegration {
/**
* Declares the process-wide OpenTelemetry settings.
*
- * This remains internal until the provider authoring API exposes it.
+ * Export it from `agent/instrumentation/otel.ts`. Omitting the file is the
+ * common case: eve registers the pipeline for whatever destinations are
+ * declared beside it, and this only names what those destinations share.
*/
export function otel(options: OtelOptions = {}): OtelDeclaration {
- return { [OTEL_DECLARATION]: true, options };
+ return { [OTEL_DECLARATION]: true, [PROVIDER]: true, options };
}
/**
@@ -100,16 +112,25 @@ export function otel(options: OtelOptions = {}): OtelDeclaration {
* A `traceExporter` is wrapped in eve's batching processor, which is what makes
* the one-line form of a hosted backend enough. Pass `spanProcessors` instead
* when the destination needs its own batching, sampling, or filtering.
+ *
+ * Declining content wraps every processor here, an author's included: they are
+ * this destination, and the point of declining is that nothing under it sees
+ * what was said.
*/
export function otelIntegration(options: OtelIntegrationOptions = {}): OtelIntegration {
- const content = resolveContentOptions(options);
+ const content: ResolvedContentOptions = {
+ recordInputs: options.recordInputs !== false,
+ recordOutputs: options.recordOutputs !== false,
+ };
const declared = options.spanProcessors ?? [];
const spanProcessors =
options.traceExporter === undefined
? declared
: [...declared, batchSpanProcessor(options.traceExporter)];
+
return {
[OTEL_INTEGRATION]: true,
+ [PROVIDER]: true,
content,
spanProcessors:
content.recordInputs && content.recordOutputs
@@ -123,6 +144,7 @@ export function agentRunsIntegration(options: ContentOptions = {}): OtelIntegrat
const content = resolveContentOptions(options);
return {
[OTEL_INTEGRATION]: true,
+ [PROVIDER]: true,
content,
spanProcessors:
content.recordInputs && content.recordOutputs
@@ -167,8 +189,9 @@ export interface OtelHarnessSettings {
readonly functionId?: string;
readonly traceChannelRequests: boolean;
/**
- * What to materialize on spans at all. Each destination independently strips
- * anything it declined before export.
+ * What to write onto a span at all, as opposed to what any one destination
+ * exports. `agent/instrumentation.ts` sets this directly; a provider
+ * directory arrives at it as the union of its destinations.
*/
readonly recordInputs?: boolean;
readonly recordOutputs?: boolean;
@@ -193,6 +216,10 @@ export interface CollectedOtel {
* happened to visit first. With one declaration per file that collision needs
* two files both exporting `otel()`, which is the only way to reach it.
*
+ * Content capture is the union of what the destinations asked for, because it
+ * governs what is written rather than what is exported. Each destination's own
+ * processors already drop what it declined.
+ *
* @internal
*/
export function collectOtelPipeline(values: readonly unknown[]): CollectedOtel {
diff --git a/packages/eve/src/tracing/otel-registration.scenario.test.ts b/packages/eve/src/tracing/otel-registration.scenario.test.ts
index 85d5e3d6f..c9b70a089 100644
--- a/packages/eve/src/tracing/otel-registration.scenario.test.ts
+++ b/packages/eve/src/tracing/otel-registration.scenario.test.ts
@@ -82,7 +82,8 @@ describe("registerOtelPipeline", () => {
it("does not export the private registration span", async () => {
const exporter = new InMemorySpanExporter();
const processor = new SimpleSpanProcessor(exporter);
- registerOtelPipeline({
+ const shutdown = vi.spyOn(processor, "shutdown");
+ const runtime = registerOtelPipeline({
pipeline: { spanProcessors: [processor] },
serviceName: "weather",
});
@@ -91,7 +92,8 @@ describe("registerOtelPipeline", () => {
await processor.forceFlush();
expect(exporter.getFinishedSpans().map((span) => span.name)).toEqual(["user.work"]);
- await processor.shutdown();
+ await runtime.shutdown();
+ expect(shutdown).toHaveBeenCalledOnce();
});
});
diff --git a/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts b/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts
index d4cc234f3..065930565 100644
--- a/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts
+++ b/packages/eve/test/scenarios/compiled-artifacts-bootstrap.scenario.test.ts
@@ -20,6 +20,10 @@ const createAppRoot = useTemporaryAppRoots();
describe("writeCompiledArtifactsFiles", () => {
afterEach(() => {
delete (globalThis as Record<string, unknown>).__eveInstrumentationLoaded;
+ delete (globalThis as Record<string, unknown>).__eveProviderSetups;
+ delete (globalThis as Record<symbol, unknown>)[
+ Symbol.for("eve.harness-instrumentation-providers")
+ ];
});
it("installs compile metadata into bundled compiled artifacts", async () => {
@@ -112,6 +116,136 @@ describe("writeCompiledArtifactsFiles", () => {
expect(instrumentationPluginModule.default()).toBeUndefined();
});
+ it("registers one provider per file when the instrumentationProviders flag is on", async () => {
+ const { agentRoot, appRoot } = await createAppRoot("eve-compiled-artifacts-providers-", {
+ packageName: "compiled-artifacts-providers-test-agent",
+ });
+ const outDir = join(appRoot, ".workflow-build");
+ const definePath = resolvePackageSourceFilePath(
+ "src/public/instrumentation/index.ts",
+ ).replaceAll("\\", "/");
+
+ await writeFile(
+ join(agentRoot, "agent.ts"),
+ [
+ "export default {",
+ ' model: "openai/gpt-5.4",',
+ " experimental: { instrumentationProviders: true },",
+ "};",
+ "",
+ ].join("\n"),
+ );
+ await writeFile(join(agentRoot, "instructions.md"), "You are a precise assistant.\n");
+ await mkdir(join(agentRoot, "instrumentation"), { recursive: true });
+ for (const slot of ["local", "otel"]) {
+ await writeFile(
+ join(agentRoot, "instrumentation", `${slot}.ts`),
+ [
+ `import { defineInstrumentation } from ${JSON.stringify(definePath)};`,
+ "",
+ "const container = globalThis as Record<string, unknown>;",
+ "",
+ "export default defineInstrumentation({",
+ " setup(context) {",
+ " container.__eveProviderSetups ??= [];",
+ ` (container.__eveProviderSetups as string[]).push(\`${slot}:\${context.agentName}\`);`,
+ " },",
+ "});",
+ "",
+ ].join("\n"),
+ );
+ }
+
+ const compileResult = await compileAgent({ startPath: appRoot });
+ const generatedArtifacts = await writeCompiledArtifactsFiles({
+ compileResult,
+ defaultWorkflowWorld: "local",
+ outDir,
+ });
+ const instrumentationPluginPath = generatedArtifacts.instrumentationPluginPath;
+
+ if (instrumentationPluginPath === undefined) {
+ throw new Error("Expected instrumentation plugin path to be generated.");
+ }
+
+ expect(generatedArtifacts.instrumentationSourcePaths).toEqual([
+ join(agentRoot, "instrumentation", "local.ts"),
+ join(agentRoot, "instrumentation", "otel.ts"),
+ ]);
+
+ const instrumentationPluginSource = await readFile(instrumentationPluginPath, "utf8");
+
+ expect(instrumentationPluginSource).toContain('slot: "local"');
+ expect(instrumentationPluginSource).toContain('slot: "otel"');
+ expect(instrumentationPluginSource).toContain("seedInstrumentationProviders();");
+ expect(instrumentationPluginSource).toContain("shutdownInstrumentationProviders");
+ expect(instrumentationPluginSource).toContain("hooks?.hook('close'");
+ expect(instrumentationPluginSource).not.toContain("registerInstrumentationConfig");
+
+ const instrumentationPlugin = (await import(pathToFileURL(instrumentationPluginPath).href)) as {
+ default: (nitroApp: {
+ hooks: { hook(name: "close", handler: () => Promise<void>): void };
+ }) => void;
+ };
+ const closeHandlers: Array<() => Promise<void>> = [];
+ instrumentationPlugin.default({
+ hooks: {
+ hook: (_name, handler) => closeHandlers.push(handler),
+ },
+ });
+
+ // The plugin resolves the registry by absolute path while the assertion
+ // resolves it by package alias, so this also proves the globalThis rooting
+ // survives two module instances.
+ const { getInstrumentationProviders } =
+ await import("../../src/harness/instrumentation-providers.js");
+
+ expect((globalThis as Record<string, unknown>).__eveProviderSetups).toEqual([
+ "local:compiled-artifacts-providers-test-agent",
+ "otel:compiled-artifacts-providers-test-agent",
+ ]);
+ expect(getInstrumentationProviders().map((entry) => entry.slot)).toEqual(["local", "otel"]);
+ expect(closeHandlers).toHaveLength(1);
+ await closeHandlers[0]?.();
+ });
+
+ it("generates the provider plugin for built-in destinations without authored files", async () => {
+ const { agentRoot, appRoot } = await createAppRoot(
+ "eve-compiled-artifacts-default-providers-",
+ {
+ packageName: "compiled-artifacts-default-providers-test-agent",
+ },
+ );
+ const outDir = join(appRoot, ".workflow-build");
+ await writeFile(
+ join(agentRoot, "agent.ts"),
+ [
+ "export default {",
+ ' model: "openai/gpt-5.4",',
+ " experimental: { instrumentationProviders: true },",
+ "};",
+ "",
+ ].join("\n"),
+ );
+ await writeFile(join(agentRoot, "instructions.md"), "You are a precise assistant.\n");
+
+ const compileResult = await compileAgent({ startPath: appRoot });
+ const generatedArtifacts = await writeCompiledArtifactsFiles({
+ compileResult,
+ defaultWorkflowWorld: "local",
+ outDir,
+ });
+ const instrumentationPluginPath = generatedArtifacts.instrumentationPluginPath;
+ if (instrumentationPluginPath === undefined) {
+ throw new Error("Expected instrumentation plugin path to be generated.");
+ }
+
+ expect(generatedArtifacts.instrumentationSourcePaths).toEqual([]);
+ expect(await readFile(instrumentationPluginPath, "utf8")).toContain(
+ "seedInstrumentationProviders();",
+ );
+ });
+
it("surfaces instrumentation import failures when the Nitro plugin module loads", async () => {
const { agentRoot, appRoot } = await createAppRoot(
"eve-compiled-artifacts-instrumentation-error-",
diff --git a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts b/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts
index 875a3c419..d6dcd70c0 100644
--- a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts
+++ b/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts
@@ -44,6 +44,8 @@ const DEVELOPMENT_ENV_KEYS = [
"EVE_DEV_LOCAL_ONLY",
"EVE_DEV_SHARED",
"EVE_DEV_SHELL_ONLY",
+ "EVE_EVALUATION",
+ "EVE_EVALUATION_RUN_ID",
] as const;
async function createEnvironmentFixture(): Promise<string> {
@@ -192,6 +194,8 @@ describe("eve eval environment loading", () => {
expect(close).toHaveBeenCalledTimes(1);
expect(handle.shutdown).toHaveBeenCalledTimes(1);
+ expect(process.env.EVE_EVALUATION).toBe("1");
+ expect(process.env.EVE_EVALUATION_RUN_ID).toMatch(/^[0-9a-f-]{36}$/u);
expect(exit).toHaveBeenCalledWith(0);
});