Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix: honor explicit trace content capture for private audiences (#4765)

Chad Hietala0fd4240d31c0
diff --git a/.changeset/honest-private-traces.md b/.changeset/honest-private-traces.md
new file mode 100644
index 000000000..4ba3d1202
--- /dev/null
+++ b/.changeset/honest-private-traces.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Honor explicit OpenTelemetry trace policies that enable inputs or outputs for DMs, private channels, and unknown audiences. Default capture remains audience-aware, and inherited capture limits and destination redaction still apply.
diff --git a/e2e/fixtures/agent-basic-runtime/agent/instrumentation/otel.ts b/e2e/fixtures/agent-basic-runtime/agent/instrumentation/otel.ts
index 58f2209af..f540bdf8a 100644
--- a/e2e/fixtures/agent-basic-runtime/agent/instrumentation/otel.ts
+++ b/e2e/fixtures/agent-basic-runtime/agent/instrumentation/otel.ts
@@ -1,3 +1,6 @@
import { otel } from "eve/instrumentation/otel";
-export default otel({ instrumentations: ["fetch"] });
+export default otel({
+ instrumentations: ["fetch"],
+ tracePolicy: () => ({ emit: true, recordInputs: true, recordOutputs: true }),
+});
diff --git a/e2e/fixtures/agent-basic-runtime/agent/lib/trace-audit.ts b/e2e/fixtures/agent-basic-runtime/agent/lib/trace-audit.ts
index e777d6d6c..e56df988d 100644
--- a/e2e/fixtures/agent-basic-runtime/agent/lib/trace-audit.ts
+++ b/e2e/fixtures/agent-basic-runtime/agent/lib/trace-audit.ts
@@ -6,6 +6,8 @@ interface AuditSpan {
spanId: string;
parentSpanId?: string;
attributes: Readonly<Record<string, unknown>>;
+ finished: Promise<void>;
+ finish: () => void;
}
// Instrumentation and tools load in separate bundles, so they share the buffer
@@ -32,20 +34,38 @@ export const traceAuditProcessor: SpanProcessor = {
runs.set(runId, spans);
if (runs.size > 64) runs.delete(runs.keys().next().value!);
}
+ let finish!: () => void;
+ const finished = new Promise<void>((resolve) => {
+ finish = resolve;
+ });
spans.push({
name: span.name,
...reference,
parentSpanId: span.parentSpanContext?.spanId,
attributes: { ...span.attributes },
+ finished,
+ finish,
});
while (spans.length > 128) spans.shift();
},
- onEnd() {},
+ onEnd(value) {
+ const span = value as {
+ attributes: Readonly<Record<string, unknown>>;
+ spanContext: () => { spanId: string };
+ };
+ const runId = span.attributes["agent.run.id"];
+ if (typeof runId !== "string") return;
+ const recorded = runs.get(runId)?.find((entry) => entry.spanId === span.spanContext().spanId);
+ if (recorded !== undefined) {
+ recorded.attributes = { ...span.attributes };
+ recorded.finish();
+ }
+ },
async forceFlush() {},
async shutdown() {},
};
-export function inspectTrace(runId: string) {
+export async function inspectTrace(runId: string) {
const owned = runs.get(runId) ?? [];
// The tool's own span is durable and starts only when the call completes, so
// the audit reads the step that requested it and that step's model call.
@@ -60,8 +80,25 @@ export function inspectTrace(runId: string) {
.reverse()
.find((span) => span.name === "agent.step" && modelOf(span) !== undefined);
const model = step === undefined ? undefined : modelOf(step);
+ // Streaming tool calls can execute before the model's terminal callback.
+ if (model !== undefined) {
+ let timeout: ReturnType<typeof setTimeout> | undefined;
+ try {
+ await Promise.race([
+ model.finished,
+ new Promise<void>((resolve) => {
+ timeout = setTimeout(resolve, 5_000);
+ }),
+ ]);
+ } finally {
+ clearTimeout(timeout);
+ }
+ }
return {
modelStep: step !== undefined && model !== undefined,
+ content:
+ typeof model?.attributes["gen_ai.input.messages"] === "string" &&
+ typeof model?.attributes["gen_ai.output.messages"] === "string",
identity:
step !== undefined &&
model !== undefined &&
diff --git a/e2e/fixtures/agent-basic-runtime/evals/runtime/trace-topology.eval.ts b/e2e/fixtures/agent-basic-runtime/evals/runtime/trace-topology.eval.ts
index c9d578151..bb2adc286 100644
--- a/e2e/fixtures/agent-basic-runtime/evals/runtime/trace-topology.eval.ts
+++ b/e2e/fixtures/agent-basic-runtime/evals/runtime/trace-topology.eval.ts
@@ -3,7 +3,7 @@ import { satisfies } from "eve/evals/expect";
export default defineEval({
description:
- "The installed tracing pipeline retains eve identity and model-step parent relationships.",
+ "The installed tracing pipeline retains eve identity, model-step parents, and explicitly enabled content for the private HTTP channel.",
async test(t) {
const turn = await t.send(
"Alice is checking the support agent's diagnostics. Call inspect_trace once and report its result.",
@@ -18,8 +18,9 @@ export default defineEval({
typeof value === "object" &&
value !== null &&
Reflect.get(value, "modelStep") === true &&
- Reflect.get(value, "identity") === true,
- "the model has its step parent and eve schema identity is preserved",
+ Reflect.get(value, "identity") === true &&
+ Reflect.get(value, "content") === true,
+ "the model has its step parent, eve schema identity, and captured input and output content",
),
);
},
diff --git a/packages/eve/src/instrumentation/runtime.test.ts b/packages/eve/src/instrumentation/runtime.test.ts
index d28134475..510bde974 100644
--- a/packages/eve/src/instrumentation/runtime.test.ts
+++ b/packages/eve/src/instrumentation/runtime.test.ts
@@ -216,6 +216,7 @@ describe("initializeSessionInstrumentation", () => {
action: "record",
recordInputs: true,
recordOutputs: false,
+ explicitContentAudience: "private",
});
await expect(
readTelemetry(
diff --git a/packages/eve/src/shared/instrumentation-content.ts b/packages/eve/src/shared/instrumentation-content.ts
index 5a0631910..65da175d5 100644
--- a/packages/eve/src/shared/instrumentation-content.ts
+++ b/packages/eve/src/shared/instrumentation-content.ts
@@ -14,12 +14,16 @@ export function shouldCaptureInstrumentationContent(
return context.audience === "public" || context.environment === "development";
}
-/** Per-delivery audience is a hard ceiling over the session-level trace decision. */
+/** An explicit policy overrides defaults only for the audience it evaluated. */
export function applyAudienceCeiling(
decision: InstrumentationDecision,
context: InstrumentationContentContext,
): InstrumentationDecision {
- if (decision.action === "drop" || shouldCaptureInstrumentationContent(context)) {
+ if (
+ decision.action === "drop" ||
+ decision.explicitContentAudience === context.audience ||
+ shouldCaptureInstrumentationContent(context)
+ ) {
return decision;
}
return { action: "record", recordInputs: false, recordOutputs: false };
diff --git a/packages/eve/src/shared/instrumentation-decision.ts b/packages/eve/src/shared/instrumentation-decision.ts
index 0d876da6d..8526fb578 100644
--- a/packages/eve/src/shared/instrumentation-decision.ts
+++ b/packages/eve/src/shared/instrumentation-decision.ts
@@ -1,9 +1,13 @@
+import { CHANNEL_AUDIENCES, type ChannelAudience } from "#shared/channel-audience.js";
+
export type InstrumentationDecision =
| { readonly action: "drop" }
| {
readonly action: "record";
readonly recordInputs: boolean;
readonly recordOutputs: boolean;
+ /** The audience for which an explicit policy authorized content. */
+ readonly explicitContentAudience?: ChannelAudience;
};
export const DROP_INSTRUMENTATION: InstrumentationDecision = { action: "drop" };
@@ -23,6 +27,10 @@ export function readInstrumentationDecision(value: unknown): InstrumentationDeci
action: "record",
recordInputs: decision.recordInputs,
recordOutputs: decision.recordOutputs,
+ ...(decision.explicitContentAudience !== undefined &&
+ CHANNEL_AUDIENCES.includes(decision.explicitContentAudience)
+ ? { explicitContentAudience: decision.explicitContentAudience }
+ : undefined),
}
: DROP_INSTRUMENTATION;
}
@@ -32,9 +40,17 @@ export function intersectInstrumentationDecisions(
right: InstrumentationDecision,
): InstrumentationDecision {
if (left.action === "drop" || right.action === "drop") return DROP_INSTRUMENTATION;
+ const explicitContentAudience =
+ left.explicitContentAudience === undefined
+ ? right.explicitContentAudience
+ : right.explicitContentAudience === undefined ||
+ left.explicitContentAudience === right.explicitContentAudience
+ ? left.explicitContentAudience
+ : undefined;
return {
action: "record",
recordInputs: left.recordInputs && right.recordInputs,
recordOutputs: left.recordOutputs && right.recordOutputs,
+ ...(explicitContentAudience !== undefined ? { explicitContentAudience } : undefined),
};
}
diff --git a/packages/eve/src/shared/trace-policy.test.ts b/packages/eve/src/shared/trace-policy.test.ts
index d5642c503..0854e3f04 100644
--- a/packages/eve/src/shared/trace-policy.test.ts
+++ b/packages/eve/src/shared/trace-policy.test.ts
@@ -28,15 +28,30 @@ describe("resolveTracePolicyDecision", () => {
],
[
{ emit: true, recordInputs: true, recordOutputs: false },
- { action: "record", recordInputs: true, recordOutputs: false },
+ {
+ action: "record",
+ recordInputs: true,
+ recordOutputs: false,
+ explicitContentAudience: "private",
+ },
],
[
{ emit: true, recordInputs: false, recordOutputs: true },
- { action: "record", recordInputs: false, recordOutputs: true },
+ {
+ action: "record",
+ recordInputs: false,
+ recordOutputs: true,
+ explicitContentAudience: "private",
+ },
],
[
{ emit: true, recordInputs: true, recordOutputs: true },
- { action: "record", recordInputs: true, recordOutputs: true },
+ {
+ action: "record",
+ recordInputs: true,
+ recordOutputs: true,
+ explicitContentAudience: "private",
+ },
],
] as const)("normalizes the explicit $decision.emit decision", (decision, expected) => {
expect(resolveTracePolicyDecision(decision, contentContext("private"))).toEqual(expected);
diff --git a/packages/eve/src/shared/trace-policy.ts b/packages/eve/src/shared/trace-policy.ts
index 3610cd440..fe0a8adf8 100644
--- a/packages/eve/src/shared/trace-policy.ts
+++ b/packages/eve/src/shared/trace-policy.ts
@@ -29,14 +29,7 @@ export function resolveTracePolicy(
): InstrumentationDecision {
try {
const decision = policy?.(trace);
- return resolveTracePolicyDecision(
- decision ?? {
- emit: true,
- recordInputs: trace.audience === "public" || trace.environment === "development",
- recordOutputs: trace.audience === "public" || trace.environment === "development",
- },
- trace,
- );
+ return resolveTracePolicyDecision(decision ?? true, trace);
} catch (error) {
try {
onError?.(error);
@@ -62,5 +55,8 @@ export function resolveTracePolicyDecision(
action: "record",
recordInputs: decision.recordInputs,
recordOutputs: decision.recordOutputs,
+ ...(decision.recordInputs || decision.recordOutputs
+ ? { explicitContentAudience: context.audience }
+ : undefined),
};
}
diff --git a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts b/packages/eve/src/tracing/eve/agent-otel-provider.test.ts
index 8923c0b87..8dcb824a8 100644
--- a/packages/eve/src/tracing/eve/agent-otel-provider.test.ts
+++ b/packages/eve/src/tracing/eve/agent-otel-provider.test.ts
@@ -58,6 +58,7 @@ import { parseLocalTraceSegment } from "#tracing/local/trace-reader.js";
import { CONTENT_ATTRIBUTE_LIMIT } from "#compiled/@eve/tracing/otel.js";
import type { TraceCapturePolicy } from "#tracing/eve/otel-declaration.js";
import type { TraceCaptureContext } from "#shared/trace-policy.js";
+import { resolveTracePolicyDecision } from "#shared/trace-policy.js";
import {
actionIdempotencyKey,
attemptIdempotencyKey,
@@ -1709,38 +1710,46 @@ describe("createAgentOtelInstrumentation", () => {
);
});
- it("caps forwarded content for an explicitly private delivery", async () => {
- const runtime = createRuntime(new InMemoryAgentTraceStateStore(), null);
- const ctx = new ContextContainer();
- const traceSeed = {
- ...spanContext("1", "2"),
- decision: { action: "record", recordInputs: true, recordOutputs: true },
- forwardedTracePolicy: {
- ceiling: { recordInputs: true, recordOutputs: true },
- originAudience: "public",
- },
- } as const;
- ctx.set(SessionTraceSeedKey, traceSeed);
-
- await expect(
- contextStorage.run(ctx, () =>
- runtime.projectEvent({
- delivery: {
- channelAudience: "private",
- channelKind: "channel:slack",
- channelName: "slack",
- deliveryId: "delivery-private",
- },
- idempotencyKey: "channel-delivery:session-1:delivery-private",
- input: { message: "private" },
- rootSessionId: "session-1",
- sessionId: "session-1",
- traceSeed,
- type: "channel.delivery.started",
- }),
- ),
- ).resolves.toMatchObject({ input: undefined });
- });
+ it.each([false, true])(
+ "caps public-origin content for a private delivery (explicit=%s)",
+ async (explicit) => {
+ const runtime = createRuntime(new InMemoryAgentTraceStateStore(), null);
+ const ctx = new ContextContainer();
+ const traceSeed = {
+ ...spanContext("1", "2"),
+ decision: explicit
+ ? resolveTracePolicyDecision(
+ { emit: true, recordInputs: true, recordOutputs: true },
+ { audience: "public", environment: "production" },
+ )
+ : { action: "record" as const, recordInputs: true, recordOutputs: true },
+ forwardedTracePolicy: {
+ ceiling: { recordInputs: true, recordOutputs: true },
+ originAudience: "public",
+ },
+ } as const;
+ ctx.set(SessionTraceSeedKey, traceSeed);
+
+ await expect(
+ contextStorage.run(ctx, () =>
+ runtime.projectEvent({
+ delivery: {
+ channelAudience: "private",
+ channelKind: "channel:slack",
+ channelName: "slack",
+ deliveryId: "delivery-private",
+ },
+ idempotencyKey: "channel-delivery:session-1:delivery-private",
+ input: { message: "private" },
+ rootSessionId: "session-1",
+ sessionId: "session-1",
+ traceSeed,
+ type: "channel.delivery.started",
+ }),
+ ),
+ ).resolves.toMatchObject({ input: undefined });
+ },
+ );
it("normalizes an event-carried drop before preparing its activation trace", async () => {
const runtime = createRuntime(new InMemoryAgentTraceStateStore(), null);
diff --git a/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts b/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts
index 0295d7fd7..5b615157e 100644
--- a/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts
+++ b/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts
@@ -57,8 +57,10 @@ import { createPresentedRuntimeActionRequestFromToolCall } from "#harness/action
import { taskWaitTool } from "#tools/provided/task-wait.js";
import { taskCancelTool } from "#tools/provided/task-cancel.js";
import { jsonSchema } from "ai";
+import type { ChannelAudience } from "#shared/channel-audience.js";
+import type { TraceCapturePolicy } from "#shared/trace-policy.js";
-const traceContext = (agentName: string, audience: "public" | "private") => ({
+const traceContext = (agentName: string, audience: ChannelAudience) => ({
agentName,
audience,
channel: { kind: "http" as const },
@@ -66,7 +68,7 @@ const traceContext = (agentName: string, audience: "public" | "private") => ({
principalType: "anonymous",
});
-function createRuntime() {
+function createRuntime(tracePolicy?: TraceCapturePolicy) {
const exporter = new InMemorySpanExporter();
const metadata = new InMemorySpanExporter();
const idGenerator = new AgentSpanIdGenerator();
@@ -80,6 +82,7 @@ function createRuntime() {
],
});
const agent = createAgentOtelInstrumentation({
+ tracePolicy,
frameworkVersion: "test",
recordInputs: true,
recordOutputs: true,
@@ -101,13 +104,18 @@ function createRuntime() {
provider,
idGenerator,
ownsAgentSpans: true,
- otelSettings: { recordInputs: true, recordOutputs: true, traceChannelRequests: false },
+ otelSettings: {
+ tracePolicy,
+ recordInputs: true,
+ recordOutputs: true,
+ traceChannelRequests: false,
+ },
forceFlush: () => provider.forceFlush(),
shutdown: () => provider.shutdown(),
};
}
-function contextFor(audience: "public" | "private") {
+function contextFor(audience: ChannelAudience) {
const ctx = new ContextContainer();
ctx.set(ChannelInstrumentationKey, { kind: "http", metadata: {} });
ctx.set(ConversationContextKey, traceContext("parent", audience));
@@ -126,7 +134,7 @@ function contextFor(audience: "public" | "private") {
return ctx;
}
-function scopeFor(sessionId: string, audience: "public" | "private"): InstrumentationAttemptScope {
+function scopeFor(sessionId: string, audience: ChannelAudience): InstrumentationAttemptScope {
return {
attemptId: `${sessionId}:turn_0:0:0`,
attemptIndex: 0,
@@ -660,10 +668,19 @@ describe("exported agent telemetry contract", () => {
}
});
- it.each(["public", "private"] as const)(
- "round-trips the normalized %s v4 trace forest through OTLP",
- async (audience) => {
- const runtime = createRuntime();
+ it.each([
+ { audience: "public", explicit: false, recordInputs: true, recordOutputs: true },
+ { audience: "private", explicit: false, recordInputs: false, recordOutputs: false },
+ { audience: "private", explicit: true, recordInputs: true, recordOutputs: true },
+ { audience: "unknown", explicit: true, recordInputs: true, recordOutputs: true },
+ { audience: "private", explicit: true, recordInputs: true, recordOutputs: false },
+ { audience: "private", explicit: true, recordInputs: false, recordOutputs: true },
+ ] as const)(
+ "round-trips the normalized $audience v4 trace forest through OTLP (explicit=$explicit, inputs=$recordInputs, outputs=$recordOutputs)",
+ async ({ audience, explicit, recordInputs, recordOutputs }) => {
+ const runtime = createRuntime(
+ explicit ? () => ({ emit: true, recordInputs, recordOutputs }) : undefined,
+ );
let parent = contextFor(audience);
parent.set(ConversationIdKey, "original-conversation");
const scope = scopeFor("parent", audience);
@@ -786,6 +803,7 @@ describe("exported agent telemetry contract", () => {
const modelKey = modelCallIdempotencyKey(childScope, 0, 0);
await childHooks.publish({
idempotencyKey: modelKey,
+ input: { instructions: "private instructions", messages: [] },
model: { modelId: "test", provider: "test" },
scope: childScope,
type: "model.call.started",
@@ -1023,8 +1041,15 @@ describe("exported agent telemetry contract", () => {
parsed.map((span) => [span.name, span.name]),
);
expect(metadata).not.toContain("private ");
- if (audience === "private") expect(new TextDecoder().decode(bytes)).not.toContain("private ");
- else expect(new TextDecoder().decode(bytes)).toContain("private reply");
+ const serialized = new TextDecoder().decode(bytes);
+ for (const text of ["private instructions", "private input"]) {
+ if (recordInputs) expect(serialized).toContain(text);
+ else expect(serialized).not.toContain(text);
+ }
+ for (const text of ["private output", "private reply"]) {
+ if (recordOutputs) expect(serialized).toContain(text);
+ else expect(serialized).not.toContain(text);
+ }
await runtime.shutdown();
},
);
diff --git a/packages/eve/src/tracing/eve/invoke-tool-span.ts b/packages/eve/src/tracing/eve/invoke-tool-span.ts
index 5ddd23e55..01694d9d6 100644
--- a/packages/eve/src/tracing/eve/invoke-tool-span.ts
+++ b/packages/eve/src/tracing/eve/invoke-tool-span.ts
@@ -143,7 +143,7 @@ async function traced(
}
const content = applyLiveDeliveryAudienceCeiling(
{
- action: "record",
+ ...decision,
recordInputs: settings.recordInputs && decision.recordInputs,
recordOutputs: settings.recordOutputs && decision.recordOutputs,
},
diff --git a/packages/eve/src/tracing/eve/otel-declaration.ts b/packages/eve/src/tracing/eve/otel-declaration.ts
index 5f82409b6..7efb6a21b 100644
--- a/packages/eve/src/tracing/eve/otel-declaration.ts
+++ b/packages/eve/src/tracing/eve/otel-declaration.ts
@@ -58,14 +58,15 @@ export interface OtelOptions {
/**
* Process-wide trace and content decision. Boolean returns preserve the
* existing audience-aware behavior; explicit decisions can disable emission
- * or select input and output capture independently. Defaults to emitting
- * every audience, with content only for public conversations. The result is
- * an OpenTelemetry capture ceiling: local delivery audience and destination
- * settings can only narrow it. For a trusted remote trace, eve evaluates this
- * policy against the immutable origin audience and intersects it with the
- * parent's effective ceiling, so every hop can only narrow capture. It never
- * changes what lifecycle instrumentation receives. A thrown error rejects trace production without
- * silencing lifecycle providers. The policy may be invoked more than once
+ * or select input and output capture independently for any audience.
+ * Defaults to emitting every audience, with content only for public
+ * conversations or development. Explicit content decisions override defaults
+ * for the evaluated audience, but destination settings can only narrow them.
+ * For a trusted remote trace, eve evaluates this policy against the immutable
+ * origin audience and intersects it with the parent's effective ceiling, so every
+ * hop can only narrow capture. It never changes what lifecycle instrumentation
+ * receives. A thrown error rejects trace production without silencing
+ * lifecycle providers. The policy may be invoked more than once
* while a new session is being prepared, so it must be deterministic for
* consistent results.
*/
diff --git a/packages/eve/src/tracing/eve/policy.ts b/packages/eve/src/tracing/eve/policy.ts
index 88164cf82..00573a46c 100644
--- a/packages/eve/src/tracing/eve/policy.ts
+++ b/packages/eve/src/tracing/eve/policy.ts
@@ -68,7 +68,7 @@ export function createEveCapturePolicy(input: {
decision.action === "drop"
? decision
: {
- action: "record",
+ ...decision,
recordInputs: input.recordInputs && decision.recordInputs,
recordOutputs: input.recordOutputs && decision.recordOutputs,
},