The Open Framework for Building Agents
fix: honor explicit trace content capture for private audiences (#4765)
Chad Hietala0fd4240d31c0
diff --git a/.changeset/honest-private-traces.md b/.changeset/honest-private-traces.mdnew file mode 100644index 000000000..4ba3d1202--- /dev/null+++ b/.changeset/honest-private-traces.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Honor explicit OpenTelemetry trace policies that enable inputs or outputs for DMs, private channels, and unknown audiences. Default capture remains audience-aware, and inherited capture limits and destination redaction still apply.diff --git a/e2e/fixtures/agent-basic-runtime/agent/instrumentation/otel.ts b/e2e/fixtures/agent-basic-runtime/agent/instrumentation/otel.tsindex 58f2209af..f540bdf8a 100644--- a/e2e/fixtures/agent-basic-runtime/agent/instrumentation/otel.ts+++ b/e2e/fixtures/agent-basic-runtime/agent/instrumentation/otel.ts@@ -1,3 +1,6 @@import { otel } from "eve/instrumentation/otel";-export default otel({ instrumentations: ["fetch"] });+export default otel({+ instrumentations: ["fetch"],+ tracePolicy: () => ({ emit: true, recordInputs: true, recordOutputs: true }),+});diff --git a/e2e/fixtures/agent-basic-runtime/agent/lib/trace-audit.ts b/e2e/fixtures/agent-basic-runtime/agent/lib/trace-audit.tsindex e777d6d6c..e56df988d 100644--- a/e2e/fixtures/agent-basic-runtime/agent/lib/trace-audit.ts+++ b/e2e/fixtures/agent-basic-runtime/agent/lib/trace-audit.ts@@ -6,6 +6,8 @@ interface AuditSpan {spanId: string;parentSpanId?: string;attributes: Readonly<Record<string, unknown>>;+ finished: Promise<void>;+ finish: () => void;}// Instrumentation and tools load in separate bundles, so they share the buffer@@ -32,20 +34,38 @@ export const traceAuditProcessor: SpanProcessor = {runs.set(runId, spans);if (runs.size > 64) runs.delete(runs.keys().next().value!);}+ let finish!: () => void;+ const finished = new Promise<void>((resolve) => {+ finish = resolve;+ });spans.push({name: span.name,...reference,parentSpanId: span.parentSpanContext?.spanId,attributes: { ...span.attributes },+ finished,+ finish,});while (spans.length > 128) spans.shift();},- onEnd() {},+ onEnd(value) {+ const span = value as {+ attributes: Readonly<Record<string, unknown>>;+ spanContext: () => { spanId: string };+ };+ const runId = span.attributes["agent.run.id"];+ if (typeof runId !== "string") return;+ const recorded = runs.get(runId)?.find((entry) => entry.spanId === span.spanContext().spanId);+ if (recorded !== undefined) {+ recorded.attributes = { ...span.attributes };+ recorded.finish();+ }+ },async forceFlush() {},async shutdown() {},};-export function inspectTrace(runId: string) {+export async function inspectTrace(runId: string) {const owned = runs.get(runId) ?? [];// The tool's own span is durable and starts only when the call completes, so// the audit reads the step that requested it and that step's model call.@@ -60,8 +80,25 @@ export function inspectTrace(runId: string) {.reverse().find((span) => span.name === "agent.step" && modelOf(span) !== undefined);const model = step === undefined ? undefined : modelOf(step);+ // Streaming tool calls can execute before the model's terminal callback.+ if (model !== undefined) {+ let timeout: ReturnType<typeof setTimeout> | undefined;+ try {+ await Promise.race([+ model.finished,+ new Promise<void>((resolve) => {+ timeout = setTimeout(resolve, 5_000);+ }),+ ]);+ } finally {+ clearTimeout(timeout);+ }+ }return {modelStep: step !== undefined && model !== undefined,+ content:+ typeof model?.attributes["gen_ai.input.messages"] === "string" &&+ typeof model?.attributes["gen_ai.output.messages"] === "string",identity:step !== undefined &&model !== undefined &&diff --git a/e2e/fixtures/agent-basic-runtime/evals/runtime/trace-topology.eval.ts b/e2e/fixtures/agent-basic-runtime/evals/runtime/trace-topology.eval.tsindex c9d578151..bb2adc286 100644--- a/e2e/fixtures/agent-basic-runtime/evals/runtime/trace-topology.eval.ts+++ b/e2e/fixtures/agent-basic-runtime/evals/runtime/trace-topology.eval.ts@@ -3,7 +3,7 @@ import { satisfies } from "eve/evals/expect";export default defineEval({description:- "The installed tracing pipeline retains eve identity and model-step parent relationships.",+ "The installed tracing pipeline retains eve identity, model-step parents, and explicitly enabled content for the private HTTP channel.",async test(t) {const turn = await t.send("Alice is checking the support agent's diagnostics. Call inspect_trace once and report its result.",@@ -18,8 +18,9 @@ export default defineEval({typeof value === "object" &&value !== null &&Reflect.get(value, "modelStep") === true &&- Reflect.get(value, "identity") === true,- "the model has its step parent and eve schema identity is preserved",+ Reflect.get(value, "identity") === true &&+ Reflect.get(value, "content") === true,+ "the model has its step parent, eve schema identity, and captured input and output content",),);},diff --git a/packages/eve/src/instrumentation/runtime.test.ts b/packages/eve/src/instrumentation/runtime.test.tsindex d28134475..510bde974 100644--- a/packages/eve/src/instrumentation/runtime.test.ts+++ b/packages/eve/src/instrumentation/runtime.test.ts@@ -216,6 +216,7 @@ describe("initializeSessionInstrumentation", () => {action: "record",recordInputs: true,recordOutputs: false,+ explicitContentAudience: "private",});await expect(readTelemetry(diff --git a/packages/eve/src/shared/instrumentation-content.ts b/packages/eve/src/shared/instrumentation-content.tsindex 5a0631910..65da175d5 100644--- a/packages/eve/src/shared/instrumentation-content.ts+++ b/packages/eve/src/shared/instrumentation-content.ts@@ -14,12 +14,16 @@ export function shouldCaptureInstrumentationContent(return context.audience === "public" || context.environment === "development";}-/** Per-delivery audience is a hard ceiling over the session-level trace decision. */+/** An explicit policy overrides defaults only for the audience it evaluated. */export function applyAudienceCeiling(decision: InstrumentationDecision,context: InstrumentationContentContext,): InstrumentationDecision {- if (decision.action === "drop" || shouldCaptureInstrumentationContent(context)) {+ if (+ decision.action === "drop" ||+ decision.explicitContentAudience === context.audience ||+ shouldCaptureInstrumentationContent(context)+ ) {return decision;}return { action: "record", recordInputs: false, recordOutputs: false };diff --git a/packages/eve/src/shared/instrumentation-decision.ts b/packages/eve/src/shared/instrumentation-decision.tsindex 0d876da6d..8526fb578 100644--- a/packages/eve/src/shared/instrumentation-decision.ts+++ b/packages/eve/src/shared/instrumentation-decision.ts@@ -1,9 +1,13 @@+import { CHANNEL_AUDIENCES, type ChannelAudience } from "#shared/channel-audience.js";+export type InstrumentationDecision =| { readonly action: "drop" }| {readonly action: "record";readonly recordInputs: boolean;readonly recordOutputs: boolean;+ /** The audience for which an explicit policy authorized content. */+ readonly explicitContentAudience?: ChannelAudience;};export const DROP_INSTRUMENTATION: InstrumentationDecision = { action: "drop" };@@ -23,6 +27,10 @@ export function readInstrumentationDecision(value: unknown): InstrumentationDeciaction: "record",recordInputs: decision.recordInputs,recordOutputs: decision.recordOutputs,+ ...(decision.explicitContentAudience !== undefined &&+ CHANNEL_AUDIENCES.includes(decision.explicitContentAudience)+ ? { explicitContentAudience: decision.explicitContentAudience }+ : undefined),}: DROP_INSTRUMENTATION;}@@ -32,9 +40,17 @@ export function intersectInstrumentationDecisions(right: InstrumentationDecision,): InstrumentationDecision {if (left.action === "drop" || right.action === "drop") return DROP_INSTRUMENTATION;+ const explicitContentAudience =+ left.explicitContentAudience === undefined+ ? right.explicitContentAudience+ : right.explicitContentAudience === undefined ||+ left.explicitContentAudience === right.explicitContentAudience+ ? left.explicitContentAudience+ : undefined;return {action: "record",recordInputs: left.recordInputs && right.recordInputs,recordOutputs: left.recordOutputs && right.recordOutputs,+ ...(explicitContentAudience !== undefined ? { explicitContentAudience } : undefined),};}diff --git a/packages/eve/src/shared/trace-policy.test.ts b/packages/eve/src/shared/trace-policy.test.tsindex d5642c503..0854e3f04 100644--- a/packages/eve/src/shared/trace-policy.test.ts+++ b/packages/eve/src/shared/trace-policy.test.ts@@ -28,15 +28,30 @@ describe("resolveTracePolicyDecision", () => {],[{ emit: true, recordInputs: true, recordOutputs: false },- { action: "record", recordInputs: true, recordOutputs: false },+ {+ action: "record",+ recordInputs: true,+ recordOutputs: false,+ explicitContentAudience: "private",+ },],[{ emit: true, recordInputs: false, recordOutputs: true },- { action: "record", recordInputs: false, recordOutputs: true },+ {+ action: "record",+ recordInputs: false,+ recordOutputs: true,+ explicitContentAudience: "private",+ },],[{ emit: true, recordInputs: true, recordOutputs: true },- { action: "record", recordInputs: true, recordOutputs: true },+ {+ action: "record",+ recordInputs: true,+ recordOutputs: true,+ explicitContentAudience: "private",+ },],] as const)("normalizes the explicit $decision.emit decision", (decision, expected) => {expect(resolveTracePolicyDecision(decision, contentContext("private"))).toEqual(expected);diff --git a/packages/eve/src/shared/trace-policy.ts b/packages/eve/src/shared/trace-policy.tsindex 3610cd440..fe0a8adf8 100644--- a/packages/eve/src/shared/trace-policy.ts+++ b/packages/eve/src/shared/trace-policy.ts@@ -29,14 +29,7 @@ export function resolveTracePolicy(): InstrumentationDecision {try {const decision = policy?.(trace);- return resolveTracePolicyDecision(- decision ?? {- emit: true,- recordInputs: trace.audience === "public" || trace.environment === "development",- recordOutputs: trace.audience === "public" || trace.environment === "development",- },- trace,- );+ return resolveTracePolicyDecision(decision ?? true, trace);} catch (error) {try {onError?.(error);@@ -62,5 +55,8 @@ export function resolveTracePolicyDecision(action: "record",recordInputs: decision.recordInputs,recordOutputs: decision.recordOutputs,+ ...(decision.recordInputs || decision.recordOutputs+ ? { explicitContentAudience: context.audience }+ : undefined),};}diff --git a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts b/packages/eve/src/tracing/eve/agent-otel-provider.test.tsindex 8923c0b87..8dcb824a8 100644--- a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/eve/agent-otel-provider.test.ts@@ -58,6 +58,7 @@ import { parseLocalTraceSegment } from "#tracing/local/trace-reader.js";import { CONTENT_ATTRIBUTE_LIMIT } from "#compiled/@eve/tracing/otel.js";import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";import type { TraceCaptureContext } from "#shared/trace-policy.js";+import { resolveTracePolicyDecision } from "#shared/trace-policy.js";import {actionIdempotencyKey,attemptIdempotencyKey,@@ -1709,38 +1710,46 @@ describe("createAgentOtelInstrumentation", () => {);});- it("caps forwarded content for an explicitly private delivery", async () => {- const runtime = createRuntime(new InMemoryAgentTraceStateStore(), null);- const ctx = new ContextContainer();- const traceSeed = {- ...spanContext("1", "2"),- decision: { action: "record", recordInputs: true, recordOutputs: true },- forwardedTracePolicy: {- ceiling: { recordInputs: true, recordOutputs: true },- originAudience: "public",- },- } as const;- ctx.set(SessionTraceSeedKey, traceSeed);-- await expect(- contextStorage.run(ctx, () =>- runtime.projectEvent({- delivery: {- channelAudience: "private",- channelKind: "channel:slack",- channelName: "slack",- deliveryId: "delivery-private",- },- idempotencyKey: "channel-delivery:session-1:delivery-private",- input: { message: "private" },- rootSessionId: "session-1",- sessionId: "session-1",- traceSeed,- type: "channel.delivery.started",- }),- ),- ).resolves.toMatchObject({ input: undefined });- });+ it.each([false, true])(+ "caps public-origin content for a private delivery (explicit=%s)",+ async (explicit) => {+ const runtime = createRuntime(new InMemoryAgentTraceStateStore(), null);+ const ctx = new ContextContainer();+ const traceSeed = {+ ...spanContext("1", "2"),+ decision: explicit+ ? resolveTracePolicyDecision(+ { emit: true, recordInputs: true, recordOutputs: true },+ { audience: "public", environment: "production" },+ )+ : { action: "record" as const, recordInputs: true, recordOutputs: true },+ forwardedTracePolicy: {+ ceiling: { recordInputs: true, recordOutputs: true },+ originAudience: "public",+ },+ } as const;+ ctx.set(SessionTraceSeedKey, traceSeed);++ await expect(+ contextStorage.run(ctx, () =>+ runtime.projectEvent({+ delivery: {+ channelAudience: "private",+ channelKind: "channel:slack",+ channelName: "slack",+ deliveryId: "delivery-private",+ },+ idempotencyKey: "channel-delivery:session-1:delivery-private",+ input: { message: "private" },+ rootSessionId: "session-1",+ sessionId: "session-1",+ traceSeed,+ type: "channel.delivery.started",+ }),+ ),+ ).resolves.toMatchObject({ input: undefined });+ },+ );it("normalizes an event-carried drop before preparing its activation trace", async () => {const runtime = createRuntime(new InMemoryAgentTraceStateStore(), null);diff --git a/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts b/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.tsindex 0295d7fd7..5b615157e 100644--- a/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts+++ b/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts@@ -57,8 +57,10 @@ import { createPresentedRuntimeActionRequestFromToolCall } from "#harness/actionimport { taskWaitTool } from "#tools/provided/task-wait.js";import { taskCancelTool } from "#tools/provided/task-cancel.js";import { jsonSchema } from "ai";+import type { ChannelAudience } from "#shared/channel-audience.js";+import type { TraceCapturePolicy } from "#shared/trace-policy.js";-const traceContext = (agentName: string, audience: "public" | "private") => ({+const traceContext = (agentName: string, audience: ChannelAudience) => ({agentName,audience,channel: { kind: "http" as const },@@ -66,7 +68,7 @@ const traceContext = (agentName: string, audience: "public" | "private") => ({principalType: "anonymous",});-function createRuntime() {+function createRuntime(tracePolicy?: TraceCapturePolicy) {const exporter = new InMemorySpanExporter();const metadata = new InMemorySpanExporter();const idGenerator = new AgentSpanIdGenerator();@@ -80,6 +82,7 @@ function createRuntime() {],});const agent = createAgentOtelInstrumentation({+ tracePolicy,frameworkVersion: "test",recordInputs: true,recordOutputs: true,@@ -101,13 +104,18 @@ function createRuntime() {provider,idGenerator,ownsAgentSpans: true,- otelSettings: { recordInputs: true, recordOutputs: true, traceChannelRequests: false },+ otelSettings: {+ tracePolicy,+ recordInputs: true,+ recordOutputs: true,+ traceChannelRequests: false,+ },forceFlush: () => provider.forceFlush(),shutdown: () => provider.shutdown(),};}-function contextFor(audience: "public" | "private") {+function contextFor(audience: ChannelAudience) {const ctx = new ContextContainer();ctx.set(ChannelInstrumentationKey, { kind: "http", metadata: {} });ctx.set(ConversationContextKey, traceContext("parent", audience));@@ -126,7 +134,7 @@ function contextFor(audience: "public" | "private") {return ctx;}-function scopeFor(sessionId: string, audience: "public" | "private"): InstrumentationAttemptScope {+function scopeFor(sessionId: string, audience: ChannelAudience): InstrumentationAttemptScope {return {attemptId: `${sessionId}:turn_0:0:0`,attemptIndex: 0,@@ -660,10 +668,19 @@ describe("exported agent telemetry contract", () => {}});- it.each(["public", "private"] as const)(- "round-trips the normalized %s v4 trace forest through OTLP",- async (audience) => {- const runtime = createRuntime();+ it.each([+ { audience: "public", explicit: false, recordInputs: true, recordOutputs: true },+ { audience: "private", explicit: false, recordInputs: false, recordOutputs: false },+ { audience: "private", explicit: true, recordInputs: true, recordOutputs: true },+ { audience: "unknown", explicit: true, recordInputs: true, recordOutputs: true },+ { audience: "private", explicit: true, recordInputs: true, recordOutputs: false },+ { audience: "private", explicit: true, recordInputs: false, recordOutputs: true },+ ] as const)(+ "round-trips the normalized $audience v4 trace forest through OTLP (explicit=$explicit, inputs=$recordInputs, outputs=$recordOutputs)",+ async ({ audience, explicit, recordInputs, recordOutputs }) => {+ const runtime = createRuntime(+ explicit ? () => ({ emit: true, recordInputs, recordOutputs }) : undefined,+ );let parent = contextFor(audience);parent.set(ConversationIdKey, "original-conversation");const scope = scopeFor("parent", audience);@@ -786,6 +803,7 @@ describe("exported agent telemetry contract", () => {const modelKey = modelCallIdempotencyKey(childScope, 0, 0);await childHooks.publish({idempotencyKey: modelKey,+ input: { instructions: "private instructions", messages: [] },model: { modelId: "test", provider: "test" },scope: childScope,type: "model.call.started",@@ -1023,8 +1041,15 @@ describe("exported agent telemetry contract", () => {parsed.map((span) => [span.name, span.name]),);expect(metadata).not.toContain("private ");- if (audience === "private") expect(new TextDecoder().decode(bytes)).not.toContain("private ");- else expect(new TextDecoder().decode(bytes)).toContain("private reply");+ const serialized = new TextDecoder().decode(bytes);+ for (const text of ["private instructions", "private input"]) {+ if (recordInputs) expect(serialized).toContain(text);+ else expect(serialized).not.toContain(text);+ }+ for (const text of ["private output", "private reply"]) {+ if (recordOutputs) expect(serialized).toContain(text);+ else expect(serialized).not.toContain(text);+ }await runtime.shutdown();},);diff --git a/packages/eve/src/tracing/eve/invoke-tool-span.ts b/packages/eve/src/tracing/eve/invoke-tool-span.tsindex 5ddd23e55..01694d9d6 100644--- a/packages/eve/src/tracing/eve/invoke-tool-span.ts+++ b/packages/eve/src/tracing/eve/invoke-tool-span.ts@@ -143,7 +143,7 @@ async function traced(}const content = applyLiveDeliveryAudienceCeiling({- action: "record",+ ...decision,recordInputs: settings.recordInputs && decision.recordInputs,recordOutputs: settings.recordOutputs && decision.recordOutputs,},diff --git a/packages/eve/src/tracing/eve/otel-declaration.ts b/packages/eve/src/tracing/eve/otel-declaration.tsindex 5f82409b6..7efb6a21b 100644--- a/packages/eve/src/tracing/eve/otel-declaration.ts+++ b/packages/eve/src/tracing/eve/otel-declaration.ts@@ -58,14 +58,15 @@ export interface OtelOptions {/*** Process-wide trace and content decision. Boolean returns preserve the* existing audience-aware behavior; explicit decisions can disable emission- * or select input and output capture independently. Defaults to emitting- * every audience, with content only for public conversations. The result is- * an OpenTelemetry capture ceiling: local delivery audience and destination- * settings can only narrow it. For a trusted remote trace, eve evaluates this- * policy against the immutable origin audience and intersects it with the- * parent's effective ceiling, so every hop can only narrow capture. It never- * changes what lifecycle instrumentation receives. A thrown error rejects trace production without- * silencing lifecycle providers. The policy may be invoked more than once+ * or select input and output capture independently for any audience.+ * Defaults to emitting every audience, with content only for public+ * conversations or development. Explicit content decisions override defaults+ * for the evaluated audience, but destination settings can only narrow them.+ * For a trusted remote trace, eve evaluates this policy against the immutable+ * origin audience and intersects it with the parent's effective ceiling, so every+ * hop can only narrow capture. It never changes what lifecycle instrumentation+ * receives. A thrown error rejects trace production without silencing+ * lifecycle providers. The policy may be invoked more than once* while a new session is being prepared, so it must be deterministic for* consistent results.*/diff --git a/packages/eve/src/tracing/eve/policy.ts b/packages/eve/src/tracing/eve/policy.tsindex 88164cf82..00573a46c 100644--- a/packages/eve/src/tracing/eve/policy.ts+++ b/packages/eve/src/tracing/eve/policy.ts@@ -68,7 +68,7 @@ export function createEveCapturePolicy(input: {decision.action === "drop"? decision: {- action: "record",+ ...decision,recordInputs: input.recordInputs && decision.recordInputs,recordOutputs: input.recordOutputs && decision.recordOutputs,},