The Open Framework for Building Agents
feat(eve): add provider event capture policy (#1864)
Chad Hietala3b43b3deeb07
diff --git a/.changeset/provider-event-capture.md b/.changeset/provider-event-capture.mdnew file mode 100644index 000000000..3cfd1421f--- /dev/null+++ b/.changeset/provider-event-capture.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Instrumentation providers can choose metadata-only or content-bearing events. eve builds sensitive projections only when requested and keeps prompts, responses, tool payloads, exceptions, and opaque provider metadata away from metadata-only providers.diff --git a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts b/packages/eve/src/harness/ai-sdk-hook-bridge.test.tsindex f0fbea575..675effd99 100644--- a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts+++ b/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts@@ -2,7 +2,9 @@ import { describe, expect, it, vi } from "vitest";import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";import {+ attemptIdempotencyKey,createInstrumentationHooks,+ modelCallIdempotencyKey,type InstrumentationAttemptScope,type InstrumentationModelCallStartedEvent,type InstrumentationModelCallTerminalEvent,@@ -58,7 +60,7 @@ describe("createAiSdkHookBridge", () => {},]);- const id = `model:${scope.attemptId}:0`;+ const id = modelCallIdempotencyKey(scope, 0);expect(calls).toEqual([`a:started:${id}`,`b:started:${id}`,@@ -96,7 +98,7 @@ describe("createAiSdkHookBridge", () => {const hooks = createInstrumentationHooks([{events: { "model.call.started": (event) => void ids.push(event.idempotencyKey) },- name: "identity",+ name: "recorder",},]);const bridge = createAiSdkHookBridge(scope, hooks, (operation, execute) => {@@ -114,7 +116,7 @@ describe("createAiSdkHookBridge", () => {await bridge.executeLanguageModelCall!({ callId: "call-1", execute: async () => "result" });- const expected = `model:${scope.attemptId}:0`;+ const expected = modelCallIdempotencyKey(scope, 0);expect(ids).toEqual([expected, expected]);});@@ -137,7 +139,7 @@ describe("createAiSdkHookBridge", () => {const hooks = createInstrumentationHooks([{events: { "model.call.started": (event) => void keys.push(event.idempotencyKey) },- name: "identity",+ name: "keys",},]);@@ -149,7 +151,7 @@ describe("createAiSdkHookBridge", () => {]);}- expect(keys).toEqual([`model:${scope.attemptId}:2`, `model:${scope.attemptId}:2`]);+ expect(keys).toEqual([modelCallIdempotencyKey(scope, 2), modelCallIdempotencyKey(scope, 2)]);});it("publishes step provider metadata as step.metadata, skipping steps without any", async () => {@@ -167,14 +169,23 @@ describe("createAiSdkHookBridge", () => {const bridge = createAiSdkHookBridge(scope, hooks);await Reflect.apply(bridge.onStepEnd!, bridge, [- { providerMetadata: { gateway: { cost: "0.000082" } } },+ {+ providerMetadata: {+ gateway: {+ cost: "0.000082",+ generationId: "generation-1",+ groundingSegments: ["private result"],+ },+ google: { searchQueries: ["private query"] },+ },+ },]);await Reflect.apply(bridge.onStepEnd!, bridge, [{ providerMetadata: undefined }]);expect(events).toEqual([{- idempotencyKey: `step:${scope.attemptId}`,- providerMetadata: { gateway: { cost: "0.000082" } },+ idempotencyKey: attemptIdempotencyKey(scope),+ providerMetadata: { gateway: { cost: "0.000082", generationId: "generation-1" } },scope,type: "step.attempt.metadata",},@@ -190,13 +201,13 @@ describe("createAiSdkHookBridge", () => {throw new Error("provider failed");},},- name: "failing",+ name: "thrower",},{events: {"model.call.completed": after,},- name: "observer",+ name: "after",},]);const bridge = createAiSdkHookBridge(scope, hooks);@@ -221,7 +232,7 @@ describe("createAiSdkHookBridge", () => {it("terminalizes started operations when the attempt errors", async () => {const after = vi.fn();const hooks = createInstrumentationHooks([- { events: { "model.call.failed": after }, name: "terminal" },+ { capture: "content", events: { "model.call.failed": after }, name: "after" },]);const bridge = createAiSdkHookBridge(scope, hooks);@@ -239,7 +250,9 @@ describe("createAiSdkHookBridge", () => {it("terminalizes started operations with the abort reason", async () => {const after = vi.fn();- const hooks = createInstrumentationHooks([{ events: { "tool.call.failed": after } }]);+ const hooks = createInstrumentationHooks([+ { capture: "content", events: { "tool.call.failed": after }, name: "after" },+ ]);const bridge = createAiSdkHookBridge(scope, hooks);const toolCall = { input: {}, toolCallId: "tool-1", toolName: "search" };@@ -264,7 +277,7 @@ describe("createAiSdkHookBridge", () => {const started = vi.fn();const hooks = createInstrumentationHooks([{ events: { "step.attempt.started": mutator }, name: "mutator" },- { events: { "step.attempt.started": started }, name: "observer" },+ { events: { "step.attempt.started": started }, name: "started" },]);const bridge = createAiSdkHookBridge(scope, hooks);@@ -274,7 +287,7 @@ describe("createAiSdkHookBridge", () => {await Reflect.apply(bridge.onStepStart!, bridge, [{ callId: "call-1", stepNumber: 0 }]);const expected = {- idempotencyKey: `step:${scope.attemptId}`,+ idempotencyKey: attemptIdempotencyKey(scope),operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },scope,type: "step.attempt.started",@@ -285,6 +298,7 @@ describe("createAiSdkHookBridge", () => {it("projects the model call callbacks onto eve fields only", async () => {const before = vi.fn((event: InstrumentationModelCallStartedEvent) => {+ if (event.input === undefined) throw new Error("expected model input");expect(Object.isFrozen(event)).toBe(true);expect(Object.isFrozen(event.input)).toBe(true);expect(Object.isFrozen(event.input.messages)).toBe(true);@@ -292,6 +306,7 @@ describe("createAiSdkHookBridge", () => {});const after = vi.fn((event: InstrumentationModelCallTerminalEvent) => {if (event.type !== "model.call.completed") throw new Error("expected completed model call");+ if (event.content === undefined) throw new Error("expected model content");expect(Object.isFrozen(event)).toBe(true);expect(Object.isFrozen(event.content)).toBe(true);expect(event.content.every((part) => Object.isFrozen(part))).toBe(true);@@ -300,8 +315,9 @@ describe("createAiSdkHookBridge", () => {});const hooks = createInstrumentationHooks([{+ capture: "content",events: { "model.call.completed": after, "model.call.started": before },- name: "model",+ name: "spy",},]);const bridge = createAiSdkHookBridge(scope, hooks);@@ -352,7 +368,7 @@ describe("createAiSdkHookBridge", () => {expect(before).toHaveBeenCalledExactlyOnceWith({- idempotencyKey: `model:${scope.attemptId}:0`,+ idempotencyKey: modelCallIdempotencyKey(scope, 0),input: { instructions: "be brief", messages: [{ content: "hi", role: "user" }] },model: { modelId: "model", provider: "test" },scope,@@ -384,7 +400,7 @@ describe("createAiSdkHookBridge", () => {},],finishReason: "tool-calls",- idempotencyKey: `model:${scope.attemptId}:0`,+ idempotencyKey: modelCallIdempotencyKey(scope, 0),scope,type: "model.call.completed",usage: {@@ -417,10 +433,16 @@ describe("createAiSdkHookBridge", () => {expect(Object.isFrozen(event)).toBe(true);expect(Object.isFrozen(event.output)).toBe(true);});+ const actionStarted = vi.fn();const hooks = createInstrumentationHooks([{- events: { "tool.call.completed": after, "tool.call.started": before },- name: "tool",+ capture: "content",+ events: {+ "action.started": actionStarted,+ "tool.call.completed": after,+ "tool.call.started": before,+ },+ name: "spy",},]);const bridge = createAiSdkHookBridge(scope, hooks);@@ -451,9 +473,90 @@ describe("createAiSdkHookBridge", () => {},expect.anything(),);+ expect(actionStarted).not.toHaveBeenCalled();},);+ it("omits content from the projection when no provider asked for it", async () => {+ const modelStarted = vi.fn();+ const modelCompleted = vi.fn();+ const toolStarted = vi.fn();+ const toolCompleted = vi.fn();+ const hooks = createInstrumentationHooks([+ {+ events: {+ "model.call.completed": modelCompleted,+ "model.call.started": modelStarted,+ "tool.call.completed": toolCompleted,+ "tool.call.started": toolStarted,+ },+ name: "metadata-only",+ },+ ]);+ const bridge = createAiSdkHookBridge(scope, hooks);+ const toolCall = { input: { q: "eve" }, toolCallId: "tool-1", toolName: "search" };++ await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [+ {+ callId: "call-1",+ instructions: "be brief",+ messages: [{ content: "hi", role: "user" }],+ modelId: "model",+ provider: "test",+ tools: undefined,+ },+ ]);+ await Reflect.apply(bridge.onLanguageModelCallEnd!, bridge, [+ {+ callId: "call-1",+ content: [{ text: "hello", type: "text" }],+ finishReason: "stop",+ performance: { responseTimeMs: 1 },+ responseId: "response-1",+ usage: { inputTokens: 1, outputTokens: 2 },+ },+ ]);+ await Reflect.apply(bridge.onToolExecutionStart!, bridge, [{ callId: "call-1", toolCall }]);+ await Reflect.apply(bridge.onToolExecutionEnd!, bridge, [+ {+ callId: "call-1",+ toolCall,+ toolExecutionMs: 1,+ toolOutput: { output: "ok", type: "tool-result" },+ },+ ]);++ expect(modelStarted.mock.calls[0]?.[0].input).toBeUndefined();+ expect(modelCompleted.mock.calls[0]?.[0].content).toBeUndefined();+ // Structure survives: usage, the finish reason, and the tool's identity are+ // not what was said.+ expect(modelCompleted.mock.calls[0]?.[0].finishReason).toBe("stop");+ expect(toolStarted.mock.calls[0]?.[0].input).toBeUndefined();+ expect(toolStarted.mock.calls[0]?.[0].toolName).toBe("search");+ expect(toolCompleted.mock.calls[0]?.[0].output).toEqual({ type: "result" });+ });++ it("withholds content from a metadata provider sharing a bus with a content one", async () => {+ const metadataOnly = vi.fn();+ const wantsContent = vi.fn();+ const hooks = createInstrumentationHooks([+ { events: { "tool.call.started": metadataOnly }, name: "metadata-only" },+ {+ capture: "content",+ events: { "tool.call.started": wantsContent },+ name: "wants-content",+ },+ ]);+ const bridge = createAiSdkHookBridge(scope, hooks);++ await Reflect.apply(bridge.onToolExecutionStart!, bridge, [+ { callId: "call-1", toolCall: { input: { q: "eve" }, toolCallId: "t", toolName: "search" } },+ ]);++ expect(wantsContent.mock.calls[0]?.[0].input).toEqual({ q: "eve" });+ expect(metadataOnly.mock.calls[0]?.[0].input).toBeUndefined();+ expect(metadataOnly.mock.calls[0]?.[0].toolName).toBe("search");+ });it("keeps each provider's state to itself", async () => {const observed = new Map<string, unknown>();const provider = (name: string): InstrumentationProviderDefinition => {@@ -496,7 +599,7 @@ describe("createAiSdkHookBridge", () => {it("skips a terminal handler when the operation never started", async () => {const completed = vi.fn();const hooks = createInstrumentationHooks([- { events: { "model.call.completed": completed }, name: "terminal" },+ { events: { "model.call.completed": completed }, name: "completed" },]);const bridge = createAiSdkHookBridge(scope, hooks);diff --git a/packages/eve/src/harness/ai-sdk-hook-bridge.ts b/packages/eve/src/harness/ai-sdk-hook-bridge.tsindex 0807e7ea1..b258de24c 100644--- a/packages/eve/src/harness/ai-sdk-hook-bridge.ts+++ b/packages/eve/src/harness/ai-sdk-hook-bridge.ts@@ -19,10 +19,13 @@ import {modelCallIdempotencyKey,toolCallIdempotencyKey,} from "#harness/instrumentation-lifecycle.js";+import { structuralProviderMetadata } from "#harness/instrumentation-content.js";type TelemetryEvent<TKey extends keyof Telemetry> = Parameters<NonNullable<Telemetry[TKey]>>[0];interface AttemptState {+ /** False when no provider asked for content, so none is projected at all. */+ readonly capturesContent: boolean;readonly modelKeys: Map<string, string>;readonly scope: InstrumentationAttemptScope;readonly toolKeys: Map<string, string>;@@ -38,6 +41,7 @@ export function createAiSdkHookBridge(runInContext: InstrumentationContextRunner = directRunInContext,): Telemetry {const state: AttemptState = {+ capturesContent: hooks.capturesContent,modelKeys: new Map(),scope,toolKeys: new Map(),@@ -80,10 +84,13 @@ export function createAiSdkHookBridge(// that the per-call telemetry events don't. Publish it for providers// that know what to do with it; skip when there is none.if (event.providerMetadata === undefined) return;+ const providerMetadata = state.capturesContent+ ? event.providerMetadata+ : structuralProviderMetadata(event.providerMetadata);await hooks.publish(Object.freeze({idempotencyKey: attemptIdempotencyKey(state.scope),- providerMetadata: event.providerMetadata,+ providerMetadata,scope: state.scope,type: "step.attempt.metadata",}),@@ -160,10 +167,12 @@ function toModelCallStarted(): InstrumentationModelCallStartedEvent {return Object.freeze({idempotencyKey,- input: Object.freeze({- instructions: source.instructions,- messages: Object.freeze([...source.messages]),- }),+ input: state.capturesContent+ ? Object.freeze({+ instructions: source.instructions,+ messages: Object.freeze([...source.messages]),+ })+ : undefined,model: Object.freeze({ modelId: source.modelId, provider: source.provider }),scope: state.scope,type: "model.call.started",@@ -176,7 +185,7 @@ function toModelCallCompleted(source: TelemetryEvent<"onLanguageModelCallEnd">,): InstrumentationModelCallCompletedEvent {return Object.freeze({- content: toContentParts(source.content),+ content: state.capturesContent ? toContentParts(source.content) : undefined,finishReason: source.finishReason,idempotencyKey,scope: state.scope,@@ -254,7 +263,7 @@ function toToolCallStarted(return Object.freeze({callId: source.toolCall.toolCallId,idempotencyKey,- input: source.toolCall.input,+ input: state.capturesContent ? source.toolCall.input : undefined,scope: state.scope,toolName: source.toolCall.toolName,type: "tool.call.started",@@ -268,7 +277,7 @@ function toToolCallCompleted(): InstrumentationToolCallCompletedEvent {return Object.freeze({idempotencyKey,- output: toToolOutput(source.toolOutput),+ output: toToolOutput(source.toolOutput, state.capturesContent),scope: state.scope,type: "tool.call.completed",});@@ -276,8 +285,14 @@ function toToolCallCompleted(function toToolOutput(toolOutput: TelemetryEvent<"onToolExecutionEnd">["toolOutput"],+ capturesContent: boolean,): InstrumentationToolOutput {- return toolOutput.type === "tool-result"- ? Object.freeze({ output: toolOutput.output, type: "result" })- : Object.freeze({ error: toolOutput.error, type: "error" });+ if (toolOutput.type === "tool-result") {+ return Object.freeze(+ capturesContent ? { output: toolOutput.output, type: "result" } : { type: "result" },+ );+ }+ return Object.freeze(+ capturesContent ? { error: toolOutput.error, type: "error" } : { type: "error" },+ );}diff --git a/packages/eve/src/harness/instrumentation-content.ts b/packages/eve/src/harness/instrumentation-content.tsnew file mode 100644index 000000000..264311808--- /dev/null+++ b/packages/eve/src/harness/instrumentation-content.ts@@ -0,0 +1,56 @@+import type { InstrumentationEvent } from "#harness/instrumentation-lifecycle.js";++/** Returns an immutable event projection with conversation content removed. */+export function withoutInstrumentationContent(event: InstrumentationEvent): InstrumentationEvent {+ switch (event.type) {+ case "action.started":+ return Object.freeze({ ...event, input: undefined });+ case "action.completed":+ return Object.freeze({ ...event, output: Object.freeze({ type: event.output.type }) });+ case "input.requested":+ return Object.freeze({ ...event, request: undefined });+ case "input.resolved":+ return Object.freeze({ ...event, error: undefined, response: undefined });+ case "tool.call.started":+ return Object.freeze({ ...event, input: undefined });+ case "tool.call.completed":+ return Object.freeze({ ...event, output: Object.freeze({ type: event.output.type }) });+ case "model.call.started":+ return Object.freeze({ ...event, input: undefined });+ case "model.call.completed":+ return Object.freeze({ ...event, content: undefined });+ case "step.attempt.metadata":+ return Object.freeze({+ ...event,+ providerMetadata: structuralProviderMetadata(event.providerMetadata),+ });+ case "action.failed":+ case "model.call.failed":+ case "session.failed":+ case "step.attempt.failed":+ case "tool.call.failed":+ case "turn.failed":+ return Object.freeze({ ...event, error: undefined });+ default:+ return event;+ }+}++/** Provider metadata fields that describe cost/identity rather than content. */+export function structuralProviderMetadata(+ metadata: Readonly<Record<string, unknown>>,+): Readonly<Record<string, unknown>> {+ const gateway = metadata["gateway"];+ if (typeof gateway !== "object" || gateway === null || Array.isArray(gateway)) {+ return Object.freeze({});+ }+ const source = gateway as Readonly<Record<string, unknown>>;+ const structural: Record<string, string | number> = {};+ for (const key of ["cost", "generationId"] as const) {+ const value = source[key];+ if (typeof value === "string" || typeof value === "number") structural[key] = value;+ }+ return Object.freeze(+ Object.keys(structural).length === 0 ? {} : { gateway: Object.freeze(structural) },+ );+}diff --git a/packages/eve/src/harness/instrumentation-dispatch.ts b/packages/eve/src/harness/instrumentation-dispatch.tsindex fffbf3c2b..8c0ba153f 100644--- a/packages/eve/src/harness/instrumentation-dispatch.ts+++ b/packages/eve/src/harness/instrumentation-dispatch.ts@@ -8,6 +8,7 @@ import {takeInstrumentationActionScopes,type InstrumentationStateOwner,} from "#harness/instrumentation-state.js";+import { withoutInstrumentationContent } from "#harness/instrumentation-content.js";import { createLogger, formatError } from "#internal/logging.js";import type {@@ -35,6 +36,8 @@ export function createInstrumentationDispatcher(const handlerTimeoutMs = options.handlerTimeoutMs ?? DEFAULT_HANDLER_TIMEOUT_MS;const groups = normalizeDispatchGroups(input);const snapshots = new WeakMap<object, unknown>();+ const providers = [...groups.serialBefore, ...groups.parallel, ...groups.serialAfter];+ const capturesContent = providers.some((provider) => provider.capture === "content");const publish = async (event: InstrumentationEvent): Promise<void> => {const snapshot = snapshotInstrumentationEvent(event, snapshots);@@ -57,18 +60,32 @@ export function createInstrumentationDispatcher(}}+ let stripped: InstrumentationEvent | undefined;+ const visibleEvent = (provider: InstrumentationProviderDefinition): InstrumentationEvent => {+ if (provider.capture === "content") return snapshot;+ stripped ??= withoutInstrumentationContent(snapshot);+ return stripped;+ };+try {try {for (const provider of groups.serialBefore) {- await dispatchToProvider(provider, snapshot, handlerTimeoutMs);+ await dispatchToProvider(provider, snapshot, handlerTimeoutMs, () =>+ visibleEvent(provider),+ );}if (groups.parallel.length === 1) {- await dispatchToProvider(groups.parallel[0]!, snapshot, handlerTimeoutMs);+ const provider = groups.parallel[0]!;+ await dispatchToProvider(provider, snapshot, handlerTimeoutMs, () =>+ visibleEvent(provider),+ );} else if (groups.parallel.length > 1) {const results = await Promise.allSettled(groups.parallel.map((provider) =>- dispatchToProvider(provider, snapshot, handlerTimeoutMs),+ dispatchToProvider(provider, snapshot, handlerTimeoutMs, () =>+ visibleEvent(provider),+ ),),);const rejected = results.find(@@ -78,7 +95,9 @@ export function createInstrumentationDispatcher(}} finally {for (const provider of groups.serialAfter) {- await dispatchToProvider(provider, snapshot, handlerTimeoutMs);+ await dispatchToProvider(provider, snapshot, handlerTimeoutMs, () =>+ visibleEvent(provider),+ );}}} finally {@@ -86,7 +105,7 @@ export function createInstrumentationDispatcher(}};- return { publish };+ return { capturesContent, publish };}function snapshotInstrumentationEvent(@@ -151,6 +170,7 @@ async function dispatchToProvider(provider: InstrumentationProviderDefinition,event: InstrumentationEvent,handlerTimeoutMs: number,+ visibleEvent: () => InstrumentationEvent,): Promise<void> {const startedBoundary = event.type.endsWith(".started") || event.type === "input.requested";const owner = stateOwner(event);@@ -162,7 +182,8 @@ async function dispatchToProvider(const state = instrumentationStateSlot(stateNamespace, event.idempotencyKey, owner);try {const settled = await withTimeout(- () => (handler as InstrumentationEventHandler<InstrumentationEvent>)(event, { state }),+ () =>+ (handler as InstrumentationEventHandler<InstrumentationEvent>)(visibleEvent(), { state }),handlerTimeoutMs,() => {state.revoke();diff --git a/packages/eve/src/harness/instrumentation-lifecycle.test.ts b/packages/eve/src/harness/instrumentation-lifecycle.test.tsindex 237873627..e30593d99 100644--- a/packages/eve/src/harness/instrumentation-lifecycle.test.ts+++ b/packages/eve/src/harness/instrumentation-lifecycle.test.ts@@ -14,6 +14,7 @@ import {type InstrumentationAttemptScope,type InstrumentationModelCallStartedEvent,type InstrumentationProviderDefinition,+ type InstrumentationToolCallCompletedEvent,} from "#harness/instrumentation-lifecycle.js";import {findInstrumentationActionScopeForCall,@@ -819,3 +820,210 @@ describe("provider dispatch groups", () => {expect(order).toEqual(["first:start", "first:end", "second"]);});});++describe("capture", () => {+ it("reports content capture only when some provider asked for it", () => {+ expect(createInstrumentationHooks([{ name: "quiet" }]).capturesContent).toBe(false);+ expect(+ createInstrumentationHooks([{ capture: "metadata", name: "quiet" }, { name: "also-quiet" }])+ .capturesContent,+ ).toBe(false);+ expect(+ createInstrumentationHooks([{ name: "quiet" }, { capture: "content", name: "loud" }])+ .capturesContent,+ ).toBe(true);+ expect(+ createInstrumentationHooks({+ parallel: [{ capture: "content", name: "loud" }],+ }).capturesContent,+ ).toBe(true);+ });++ it("allows only structural provider metadata by default", async () => {+ const metadataOnly = vi.fn();+ const wantsContent = vi.fn();+ const hooks = createInstrumentationHooks([+ { events: { "step.attempt.metadata": metadataOnly }, name: "metadata" },+ {+ capture: "content",+ events: { "step.attempt.metadata": wantsContent },+ name: "content",+ },+ ]);+ const providerMetadata = {+ gateway: {+ cost: "0.01",+ generationId: "generation-1",+ groundingSegments: ["private result"],+ },+ google: { searchQueries: ["private query"], thoughtSignature: "private signature" },+ };++ await hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ providerMetadata,+ scope,+ type: "step.attempt.metadata",+ });++ const visible = metadataOnly.mock.calls[0]?.[0];+ expect(visible.providerMetadata).toEqual({+ gateway: { cost: "0.01", generationId: "generation-1" },+ });+ expect(Object.isFrozen(visible)).toBe(true);+ expect(Object.isFrozen(visible.providerMetadata)).toBe(true);+ expect(Object.isFrozen(visible.providerMetadata.gateway)).toBe(true);+ expect(wantsContent.mock.calls[0]?.[0].providerMetadata).toEqual(providerMetadata);+ expect(wantsContent.mock.calls[0]?.[0].providerMetadata).not.toBe(providerMetadata);+ expect(Object.isFrozen(wantsContent.mock.calls[0]?.[0].providerMetadata)).toBe(true);+ });++ it("withholds failure details from metadata providers", async () => {+ const metadataOnly = vi.fn();+ const wantsContent = vi.fn();+ const hooks = createInstrumentationHooks([+ { events: { "action.failed": metadataOnly }, name: "metadata" },+ { capture: "content", events: { "action.failed": wantsContent }, name: "content" },+ ]);+ const error = { output: "private tool output", requestBody: "private request" };+ const actionScope = { ...scope };++ await hooks.publish({+ error,+ errorCode: "SUBAGENT_EXECUTION_FAILED",+ idempotencyKey: actionIdempotencyKey(scope.sessionId, scope.turnId, "call-1"),+ outcome: "failed",+ scope: actionScope,+ type: "action.failed",+ });++ expect(metadataOnly.mock.calls[0]?.[0]).toMatchObject({+ error: undefined,+ errorCode: "SUBAGENT_EXECUTION_FAILED",+ outcome: "failed",+ type: "action.failed",+ });+ expect(Object.isFrozen(metadataOnly.mock.calls[0]?.[0])).toBe(true);+ expect(wantsContent.mock.calls[0]?.[0].error).toEqual(error);+ expect(wantsContent.mock.calls[0]?.[0].error).not.toBe(error);+ expect(Object.isFrozen(wantsContent.mock.calls[0]?.[0].error)).toBe(true);+ });++ it("keeps action outcome and usage when output content is withheld", async () => {+ const metadataOnly = vi.fn();+ const hooks = createInstrumentationHooks([+ { events: { "action.completed": metadataOnly }, name: "metadata" },+ ]);+ await hooks.publish({+ acceptedAtMs: 1_234,+ idempotencyKey: actionIdempotencyKey(scope.sessionId, scope.turnId, "call-1"),+ outcome: "completed",+ output: { output: "private result", type: "result" },+ scope,+ type: "action.completed",+ usage: { inputTokens: 10, outputTokens: 5 },+ });++ expect(metadataOnly.mock.calls[0]?.[0]).toMatchObject({+ acceptedAtMs: 1_234,+ outcome: "completed",+ output: { type: "result" },+ usage: { inputTokens: 10, outputTokens: 5 },+ });+ });++ it("withholds input request and response content from metadata providers", async () => {+ const metadataEvents: unknown[] = [];+ const contentEvents: unknown[] = [];+ const hooks = createInstrumentationHooks([+ {+ events: {+ "input.requested": (event) => void metadataEvents.push(event),+ "input.resolved": (event) => void metadataEvents.push(event),+ },+ name: "metadata",+ },+ {+ capture: "content",+ events: {+ "input.requested": (event) => void contentEvents.push(event),+ "input.resolved": (event) => void contentEvents.push(event),+ },+ name: "content",+ },+ ]);+ const idempotencyKey = inputIdempotencyKey(scope.sessionId, scope.turnId, "request-1");++ await hooks.publish({+ action: { callId: "call-1", name: "weather" },+ idempotencyKey,+ kind: "tool-approval",+ request: { prompt: "Approve private input?" },+ requestId: "request-1",+ scope,+ type: "input.requested",+ });+ await hooks.publish({+ idempotencyKey,+ kind: "tool-approval",+ outcome: "denied",+ requestId: "request-1",+ response: { optionId: "deny", text: "private reason" },+ scope,+ type: "input.resolved",+ });++ expect(metadataEvents).toMatchObject([+ { request: undefined, type: "input.requested" },+ { response: undefined, type: "input.resolved" },+ ]);+ expect(contentEvents).toMatchObject([+ { request: { prompt: "Approve private input?" }, type: "input.requested" },+ {+ response: { optionId: "deny", text: "private reason" },+ type: "input.resolved",+ },+ ]);+ });++ it("freezes one stripped projection shared by metadata providers", async () => {+ const observed = vi.fn();+ const content = vi.fn();+ const mutator = vi.fn((event: InstrumentationToolCallCompletedEvent) => {+ expect(Reflect.set(event, "finishReason", "corrupted")).toBe(false);+ expect(Reflect.set(event.output, "type", "error")).toBe(false);+ });+ const hooks = createInstrumentationHooks({+ parallel: [+ { events: { "tool.call.completed": mutator }, name: "first" },+ { events: { "tool.call.completed": observed }, name: "second" },+ {+ capture: "content",+ events: { "tool.call.completed": content },+ name: "content",+ },+ ],+ });++ const event = {+ idempotencyKey: toolCallIdempotencyKey(scope, "call-1", 0),+ output: { output: "private", type: "result" },+ scope,+ type: "tool.call.completed",+ } as const;+ await hooks.publish(event);++ expect(observed.mock.calls[0]?.[0].output).toEqual({ type: "result" });+ expect(observed.mock.calls[0]?.[0]).toBe(mutator.mock.calls[0]?.[0]);+ expect(Object.isFrozen(observed.mock.calls[0]?.[0])).toBe(true);+ expect(Object.isFrozen(observed.mock.calls[0]?.[0].output)).toBe(true);+ const contentEvent = content.mock.calls[0]?.[0];+ expect(contentEvent).not.toBe(event);+ expect(contentEvent).toEqual(event);+ expect(contentEvent.output).toEqual({ output: "private", type: "result" });+ expect(Object.isFrozen(contentEvent)).toBe(true);+ expect(Object.isFrozen(contentEvent.output)).toBe(true);+ expect(Object.isFrozen(contentEvent.scope)).toBe(true);+ expect(Object.isFrozen(event)).toBe(false);+ });+});diff --git a/packages/eve/src/harness/instrumentation-lifecycle.ts b/packages/eve/src/harness/instrumentation-lifecycle.tsindex f9cbce54a..b99f0a38b 100644--- a/packages/eve/src/harness/instrumentation-lifecycle.ts+++ b/packages/eve/src/harness/instrumentation-lifecycle.ts@@ -88,10 +88,30 @@ export type InstrumentationActionKind =| "subagent-call"| "tool-call";-/** How one action ended. */+/**+ * How one action ended.+ *+ * `type` survives a provider that declined content, so whether the tool errored+ * is answerable without seeing what it returned.+ */export type InstrumentationActionOutput =- | { readonly type: "result"; readonly output: unknown }- | { readonly type: "error"; readonly error: unknown };+ | { readonly type: "result"; readonly output?: unknown }+ | { readonly type: "error"; readonly error?: unknown };++/**+ * How much of an event a provider is handed.+ *+ * `"metadata"` — the default — is structure, identity, usage, and timing: every+ * field except what the conversation actually said. `"content"` adds the+ * prompt, the response, tool arguments, and tool results.+ *+ * Declared per provider rather than per process, because two consumers of one+ * bus rarely have the same retention path. Content is built at all only when+ * some provider asked for it, and a provider that did not ask never receives+ * it — which is the same guarantee a destination that declines content gets,+ * one layer lower and without an OpenTelemetry pipeline to route it through.+ */+export type InstrumentationCapture = "content" | "metadata";/*** Every event carries an `idempotencyKey` naming the operation it is about: a@@ -175,7 +195,8 @@ export interface InstrumentationInputRequestedEvent {};readonly idempotencyKey: string;readonly kind: InstrumentationInputKind;- readonly request: InstrumentationInputRequest;+ /** Content. Absent unless this provider declared `capture: "content"`. */+ readonly request?: InstrumentationInputRequest;readonly requestId: string;readonly scope: InstrumentationAttemptScope;}@@ -187,6 +208,7 @@ export interface InstrumentationInputResolvedEvent {readonly kind: InstrumentationInputKind;readonly outcome: InstrumentationInputOutcome;readonly requestId: string;+ /** Content. Absent unless this provider declared `capture: "content"`. */readonly response?: InstrumentationInputResponse;readonly scope: InstrumentationAttemptScope;}@@ -246,7 +268,8 @@ export interface InstrumentationSessionSettledEvent {export interface InstrumentationSessionFailedEvent {readonly type: "session.failed";- readonly error: unknown;+ /** Content. Absent unless this provider declared `capture: "content"`. */+ readonly error?: unknown;readonly idempotencyKey: string;readonly sessionId: string;readonly turnId?: string;@@ -283,7 +306,8 @@ export interface InstrumentationTurnSettledEvent {export interface InstrumentationTurnFailedEvent {readonly type: "turn.failed";- readonly error: unknown;+ /** Content. Absent unless this provider declared `capture: "content"`. */+ readonly error?: unknown;readonly idempotencyKey: string;readonly sessionId: string;readonly turnId: string;@@ -301,7 +325,8 @@ export interface InstrumentationStepAttemptCompletedEvent {export interface InstrumentationStepAttemptFailedEvent {readonly type: "step.attempt.failed";- readonly error: unknown;+ /** Content. Absent unless this provider declared `capture: "content"`. */+ readonly error?: unknown;readonly idempotencyKey: string;readonly scope: InstrumentationAttemptScope;}@@ -325,14 +350,16 @@ export interface InstrumentationStepAttemptMetadataEvent {export interface InstrumentationModelCallStartedEvent {readonly type: "model.call.started";readonly idempotencyKey: string;- readonly input: InstrumentationModelInput;+ /** Content. Absent unless this provider declared `capture: "content"`. */+ readonly input?: InstrumentationModelInput;readonly model: InstrumentationModelRef;readonly scope: InstrumentationAttemptScope;}export interface InstrumentationModelCallCompletedEvent {readonly type: "model.call.completed";- readonly content: readonly InstrumentationContentPart[];+ /** Content. Absent unless this provider declared `capture: "content"`. */+ readonly content?: readonly InstrumentationContentPart[];readonly finishReason: string;readonly idempotencyKey: string;readonly scope: InstrumentationAttemptScope;@@ -341,7 +368,8 @@ export interface InstrumentationModelCallCompletedEvent {export interface InstrumentationModelCallFailedEvent {readonly type: "model.call.failed";- readonly error: unknown;+ /** Content. Absent unless this provider declared `capture: "content"`. */+ readonly error?: unknown;readonly idempotencyKey: string;readonly scope: InstrumentationAttemptScope;}@@ -370,7 +398,8 @@ export interface InstrumentationToolCallCompletedEvent {export interface InstrumentationToolCallFailedEvent {readonly type: "tool.call.failed";- readonly error: unknown;+ /** Content. Absent unless this provider declared `capture: "content"`. */+ readonly error?: unknown;readonly idempotencyKey: string;readonly scope: InstrumentationAttemptScope;}@@ -388,7 +417,8 @@ export interface InstrumentationActionStartedEvent {readonly type: "action.started";readonly callId: string;readonly idempotencyKey: string;- readonly input: unknown;+ /** Content. Absent unless this provider declared `capture: "content"`. */+ readonly input?: unknown;readonly kind: InstrumentationActionKind;readonly name: string;readonly scope: InstrumentationAttemptScope;@@ -414,7 +444,8 @@ export interface InstrumentationActionCompletedEvent {export interface InstrumentationActionFailedEvent {readonly type: "action.failed";readonly acceptedAtMs?: number;- readonly error: unknown;+ /** Content. Absent unless this provider declared `capture: "content"`. */+ readonly error?: unknown;readonly errorCode?: string;readonly idempotencyKey: string;readonly outcome: Exclude<InstrumentationActionOutcome, "completed">;@@ -447,6 +478,8 @@ export interface InstrumentationProviderDefinition {readonly name: string;/** Durable state identity, separate from the human-readable log name. */readonly stateNamespace?: string;+ /** Defaults to `"metadata"`. See {@link InstrumentationCapture}. */+ readonly capture?: InstrumentationCapture;readonly events?: {readonly "step.attempt.started"?: InstrumentationEventHandler<InstrumentationStepAttemptStartedEvent>;readonly "step.attempt.completed"?: InstrumentationEventHandler<InstrumentationStepAttemptCompletedEvent>;@@ -535,6 +568,14 @@ export type InstrumentationExecutionOperation =/** Provider-neutral hook operations consumed by the AI SDK bridge. */export interface InstrumentationHooks {+ /**+ * Whether any registered provider declared `capture: "content"`.+ *+ * False means nothing downstream can read what was said, so the publisher+ * should not serialize it in the first place. This is the only way the+ * projection is skipped rather than merely withheld.+ */+ readonly capturesContent: boolean;publish(event: InstrumentationEvent): Promise<void>;}diff --git a/packages/eve/src/harness/instrumentation-native-events.test.ts b/packages/eve/src/harness/instrumentation-native-events.test.tsindex 16a5e19fc..82869bcc5 100644--- a/packages/eve/src/harness/instrumentation-native-events.test.ts+++ b/packages/eve/src/harness/instrumentation-native-events.test.ts@@ -30,6 +30,7 @@ describe("createInstrumentationHandleEvent", () => {it("publishes native lifecycle transitions after durable handling", async () => {const order: string[] = [];const hooks: InstrumentationHooks = {+ capturesContent: false,publish: async (event) => {order.push(`lifecycle:${event.type}`);},@@ -82,6 +83,7 @@ describe("createInstrumentationHandleEvent", () => {expect(createInstrumentationHandleEvent({hooks: {+ capturesContent: false,publish: async () => {},},sessionId: "session-1",@@ -94,6 +96,7 @@ describe("createInstrumentationHandleEvent", () => {const handleEvent = createInstrumentationHandleEvent({handleEvent: async () => {},hooks: {+ capturesContent: false,publish: async (event) => {events.push(event);},@@ -125,6 +128,7 @@ describe("createInstrumentationHandleEvent", () => {const handleEvent = createInstrumentationHandleEvent({handleEvent: async () => {},hooks: {+ capturesContent: false,publish: async (event) => {events.push(event);},@@ -203,7 +207,7 @@ describe("createInstrumentationHandleEvent", () => {const handleEvent = createInstrumentationHandleEvent({getAttemptScope: () => scope,handleEvent: async () => {},- hooks: { publish: async (event) => void events.push(event) },+ hooks: { capturesContent: true, publish: async (event) => void events.push(event) },sessionId: "session-1",})!;await handleEvent(requested);@@ -215,7 +219,7 @@ describe("createInstrumentationHandleEvent", () => {restored.set(RuntimeActionSettlementTimesKey, { "remote-1": 1_234 });const handleEvent = createInstrumentationHandleEvent({handleEvent: async () => {},- hooks: { publish: async (event) => void events.push(event) },+ hooks: { capturesContent: true, publish: async (event) => void events.push(event) },sessionId: "session-1",})!;await handleEvent(@@ -385,7 +389,7 @@ describe("createInstrumentationHandleEvent", () => {const handleEvent = createInstrumentationHandleEvent({getAttemptScope: () => scope,handleEvent: async () => {},- hooks: { publish: async (event) => void events.push(event) },+ hooks: { capturesContent: true, publish: async (event) => void events.push(event) },sessionId: "session-1",})!;await handleEvent(requested);@@ -405,7 +409,7 @@ describe("createInstrumentationHandleEvent", () => {},],},- hooks: { publish: async (event) => void events.push(event) },+ hooks: { capturesContent: true, publish: async (event) => void events.push(event) },sessionId: "session-1",}),);diff --git a/packages/eve/src/harness/instrumentation-native-events.ts b/packages/eve/src/harness/instrumentation-native-events.tsindex 477d2469e..4b4db594b 100644--- a/packages/eve/src/harness/instrumentation-native-events.ts+++ b/packages/eve/src/harness/instrumentation-native-events.ts@@ -94,12 +94,14 @@ async function publishInputStarts(}),idempotencyKey,kind: request.kind,- request: Object.freeze({- allowFreeform: request.allowFreeform,- display: request.display,- options: request.options,- prompt: request.prompt,- }),+ request: hooks.capturesContent+ ? Object.freeze({+ allowFreeform: request.allowFreeform,+ display: request.display,+ options: request.options,+ prompt: request.prompt,+ })+ : undefined,requestId: request.requestId,scope,type: "input.requested",@@ -129,7 +131,7 @@ export async function publishInputResolutions(input: {outcome: resolved.outcome,requestId: resolved.request.requestId,response:- resolved.response === undefined+ !input.hooks.capturesContent || resolved.response === undefined? undefined: Object.freeze({optionId: resolved.response.optionId,@@ -160,7 +162,7 @@ async function publishActionStarts(Object.freeze({callId: action.callId,idempotencyKey,- input: action.input,+ input: hooks.capturesContent ? action.input : undefined,kind: action.kind,name: actionName(action),scope,@@ -190,7 +192,11 @@ async function publishActionTerminal(],idempotencyKey,outcome: "completed",- output: Object.freeze({ output: event.data.result.output, type: "result" }),+ output: Object.freeze(+ hooks.capturesContent+ ? { output: event.data.result.output, type: "result" }+ : { type: "result" },+ ),scope,type: "action.completed",usage: actionUsage(event.data.result),diff --git a/packages/eve/src/harness/instrumentation-providers.ts b/packages/eve/src/harness/instrumentation-providers.tsindex 13fb2a3eb..2bb4c946f 100644--- a/packages/eve/src/harness/instrumentation-providers.ts+++ b/packages/eve/src/harness/instrumentation-providers.ts@@ -142,6 +142,7 @@ function toProviderDefinition(entry: RegisteredInstrumentationProvider,): InstrumentationProviderDefinition {return {+ capture: entry.provider.capture,events: entry.provider.events as InstrumentationProviderDefinition["events"],flush: entry.provider.flush,// The file the provider came from, which is the only name an author candiff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex 81fb1f608..0670db76d 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -10290,7 +10290,7 @@ describe("createToolLoopHarness", () => {});const started = vi.fn();const hooks = createInstrumentationHooks([- { events: { "action.started": started }, name: "action" },+ { events: { "action.started": started }, name: "actions" },]);const { emit } = createEventCollector();const runStep = createToolLoopHarness(diff --git a/packages/eve/src/public/instrumentation/provider.ts b/packages/eve/src/public/instrumentation/provider.tsindex dac98d3a7..89a92ad1e 100644--- a/packages/eve/src/public/instrumentation/provider.ts+++ b/packages/eve/src/public/instrumentation/provider.ts@@ -9,7 +9,10 @@// runtime. The event shapes are eve's own vocabulary; deriving the handler map// from the union below is what keeps the public contract from drifting away// from the bus that feeds it.-import type { InstrumentationEvent } from "#harness/instrumentation-lifecycle.js";+import type {+ InstrumentationCapture,+ InstrumentationEvent,+} from "#harness/instrumentation-lifecycle.js";import type { JsonValue } from "#public/types/json.js";export type { JsonValue } from "#public/types/json.js";@@ -22,6 +25,7 @@ export type {InstrumentationActionOutput,InstrumentationActionStartedEvent,InstrumentationAttemptScope,+ InstrumentationCapture,InstrumentationContentPart,InstrumentationEvent,InstrumentationInputKind,@@ -142,6 +146,15 @@ export type ProviderEvents = {* completion order.*/export interface ProviderDefinition {+ /**+ * How much of each event this provider is handed. Defaults to `"metadata"`:+ * structure, identity, usage, and timing, but not what was said.+ *+ * `"content"` adds the prompt, the response, tool arguments, and tool+ * results. Asking is what makes eve build the projection at all, so a+ * directory in which nobody asks never serializes a prompt.+ */+ readonly capture?: InstrumentationCapture;readonly events?: ProviderEvents;/** Runs once at server startup, before any event is published. */readonly setup?: (context: ProviderSetupContext) => void | PromiseLike<void>;diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex b7347a35d..bbbbe2663 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -336,7 +336,7 @@ export function createAgentOtelInstrumentation(},attempt.operation.context,);- if (recordInputs) {+ if (recordInputs && event.input !== undefined) {const messages = messagesContentAttribute(event.input.messages);if (messages !== undefined) span.setAttribute("ai.prompt.messages", messages);const genAiMessages = genAiInputMessagesAttribute(event.input.messages);@@ -366,13 +366,13 @@ export function createAgentOtelInstrumentation(if (attempt !== undefined) setAgentUsage(attempt.step.span, event.usage);if (recordOutputs) {state.span.setAttribute("ai.response.finish_reason", event.finishReason);- const content = event.content;+ const content = event.content ?? [];const outputMessages = genAiOutputMessagesAttribute(content, event.finishReason);if (outputMessages !== undefined) {state.span.setAttribute("gen_ai.output.messages", outputMessages);}const reasoning = textContentAttribute(- event.content+ content.filter((part) => part.type === "reasoning").map((part) => part.text).filter((part) => part.trim().length > 0)@@ -380,13 +380,13 @@ export function createAgentOtelInstrumentation();if (reasoning !== undefined) state.span.setAttribute("ai.response.reasoning", reasoning);const text = textContentAttribute(- event.content+ content.filter((part) => part.type === "text").map((part) => part.text).join(""),);if (text !== undefined) state.span.setAttribute("ai.response.text", text);- const toolCalls = event.content+ const toolCalls = content.filter((part) => part.type === "tool-call").map((part) => ({ callId: part.callId, input: part.input, toolName: part.toolName }));if (toolCalls.length > 0) {@@ -396,7 +396,7 @@ export function createAgentOtelInstrumentation(// Provider-executed tools (e.g. web_search) run inside the model call,// never reach eve's tool loop, and so never get an ai.toolCall span.// Their results only exist as content parts on the model response.- const toolResults = event.content+ const toolResults = content.filter((part) => part.type === "tool-result" || part.type === "tool-error").map((part) =>part.type === "tool-result"@@ -513,6 +513,10 @@ export function createAgentOtelInstrumentation(return {hook: {+ // The destinations behind this pipeline filter content per exporter, but+ // that filter only runs on a span that has it. Declining both here is+ // what stops the projection from being built upstream.+ capture: recordInputs || recordOutputs ? "content" : "metadata",events: {"action.completed": actions.events["action.completed"],"action.failed": actions.events["action.failed"],