The Open Framework for Building Agents
fix(eve): align activation content markers with trace policy (#4850)
Chad Hietalae4e0cd1757d9
diff --git a/.changeset/accurate-activation-content-markers.md b/.changeset/accurate-activation-content-markers.mdnew file mode 100644index 000000000..407c9366d--- /dev/null+++ b/.changeset/accurate-activation-content-markers.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Fix activation trace-content markers to respect the resolved session policy, provider capture settings, and live audience ceiling. Metadata-only sessions now report withheld content accurately without removing content allowed by an explicit trace policy.diff --git a/packages/eve/src/tracing/eve/agent-otel-provider.ts b/packages/eve/src/tracing/eve/agent-otel-provider.tsindex 5ff8770c9..5b640bf41 100644--- a/packages/eve/src/tracing/eve/agent-otel-provider.ts+++ b/packages/eve/src/tracing/eve/agent-otel-provider.ts@@ -73,10 +73,16 @@ export function createAgentOtelInstrumentation(const recordInputs = input.recordInputs ?? false;const recordOutputs = input.recordOutputs ?? false;const tracing = createEveTracing(input);+ const capturePolicy = createEveCapturePolicy({+ stateStore: input.stateStore,+ environment,+ recordInputs,+ recordOutputs,+ });// Attempts run in one process; a lost serverless worker retries the whole step.const steps = new Map<string, AttemptOperation>();const modelSpans = new Map<string, Map<string, ModelOperation>>();- const sessionTracing = createEveSessionTracing({ ...input, environment, tracing });+ const sessionTracing = createEveSessionTracing({ ...input, environment, tracing, capturePolicy });const { prepareSessionTrace, prepareTurnTrace } = sessionTracing;/** The attempt in its turn's tree, hydrating it when another process started it. */@@ -106,13 +112,6 @@ export function createAgentOtelInstrumentation(});const memory = createAgentMemoryInstrumentation({ ...input, environment, tracing });- const capturePolicy = createEveCapturePolicy({- stateStore: input.stateStore,- environment,- recordInputs,- recordOutputs,- });-function turnState(scope: { sessionId: string; turnId: string }) {return input.stateStore.get("turn", JSON.stringify([scope.sessionId, scope.turnId]));}diff --git a/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts b/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.tsindex cee75743b..2cae0dc32 100644--- a/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts+++ b/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts@@ -16,6 +16,7 @@ import {ParentSessionKey,ParentTraceContextKey,SessionTraceSeedKey,+ SessionTitleKey,TraceRootKey,} from "#context/keys.js";import { deserializeContext, serializeContext } from "#context/serialize.js";@@ -68,7 +69,10 @@ const traceContext = (agentName: string, audience: ChannelAudience) => ({principalType: "anonymous",});-function createRuntime(tracePolicy?: TraceCapturePolicy) {+function createRuntime(+ tracePolicy?: TraceCapturePolicy,+ content = { recordInputs: true, recordOutputs: true },+) {const exporter = new InMemorySpanExporter();const metadata = new InMemorySpanExporter();const idGenerator = new AgentSpanIdGenerator();@@ -84,8 +88,8 @@ function createRuntime(tracePolicy?: TraceCapturePolicy) {const agent = createAgentOtelInstrumentation({tracePolicy,frameworkVersion: "test",- recordInputs: true,- recordOutputs: true,+ environment: "production",+ ...content,stateStore: new ContextAgentTraceStateStore(),telemetry: otelTelemetry({provider,@@ -106,8 +110,7 @@ function createRuntime(tracePolicy?: TraceCapturePolicy) {ownsAgentSpans: true,otelSettings: {tracePolicy,- recordInputs: true,- recordOutputs: true,+ ...content,traceChannelRequests: false,},forceFlush: () => provider.forceFlush(),@@ -670,19 +673,39 @@ describe("exported agent telemetry contract", () => {it.each([{ audience: "public", explicit: false, recordInputs: true, recordOutputs: true },+ { audience: "public", explicit: true, recordInputs: false, recordOutputs: false },{ audience: "private", explicit: false, recordInputs: false, recordOutputs: false },+ { audience: "unknown", explicit: false, recordInputs: false, recordOutputs: false },{ audience: "private", explicit: true, recordInputs: true, recordOutputs: true },{ audience: "unknown", explicit: true, recordInputs: true, recordOutputs: true },{ audience: "private", explicit: true, recordInputs: true, recordOutputs: false },{ audience: "private", explicit: true, recordInputs: false, recordOutputs: true },+ {+ audience: "public",+ explicit: false,+ recordInputs: false,+ recordOutputs: true,+ providerContent: { recordInputs: false, recordOutputs: true },+ },+ {+ audience: "public",+ explicit: false,+ recordInputs: true,+ recordOutputs: false,+ providerContent: { recordInputs: true, recordOutputs: false },+ },] as const)("round-trips the normalized $audience v4 trace forest through OTLP (explicit=$explicit, inputs=$recordInputs, outputs=$recordOutputs)",- async ({ audience, explicit, recordInputs, recordOutputs }) => {+ async (testCase) => {+ const { audience, explicit, recordInputs, recordOutputs } = testCase;+ const includesIds = audience === "public" && (!explicit || (recordInputs && recordOutputs));const runtime = createRuntime(explicit ? () => ({ emit: true, recordInputs, recordOutputs }) : undefined,+ "providerContent" in testCase ? testCase.providerContent : undefined,);let parent = contextFor(audience);parent.set(ConversationIdKey, "original-conversation");+ parent.set(SessionTitleKey, "private initial title");const scope = scopeFor("parent", audience);const hooks = runtime.hooks.forTrace!(traceContext("parent", audience));const actionKey = actionIdempotencyKey("parent", "turn_0", "workflow");@@ -972,13 +995,22 @@ describe("exported agent telemetry contract", () => {}expect(parsed.filter(isAgentTurnSpan)).toHaveLength(4);for (const activation of parsed.filter(isAgentTurnSpan)) {+ expect(activation.attributes).toMatchObject({+ "agent.trace.content.input": recordInputs,+ "agent.trace.content.output": recordOutputs,+ });+ expect(activation.attributes["agent.session.title"]).toBe(+ activation.attributes["agent.run.id"] === "parent" && recordInputs+ ? "private initial title"+ : undefined,+ );expect(activation.attributes["agent.principal.current.type"]).toBe("service");expect(activation.attributes["agent.principal.initiator.type"]).toBe("user");expect(activation.attributes["agent.principal.current.id"]).toBe(- audience === "public" ? "current-user" : undefined,+ includesIds ? "current-user" : undefined,);expect(activation.attributes["agent.principal.initiator.id"]).toBe(- audience === "public" ? "initiator-user" : undefined,+ includesIds ? "initiator-user" : undefined,);}expect(new TextDecoder().decode(bytes)).not.toContain("auth-only-secret");@@ -1041,6 +1073,12 @@ describe("exported agent telemetry contract", () => {parsed.map((span) => [span.name, span.name]),);expect(metadata).not.toContain("private ");+ for (const activation of metadataSpans.filter(isAgentTurnSpan)) {+ expect(activation.attributes).toMatchObject({+ "agent.trace.content.input": false,+ "agent.trace.content.output": false,+ });+ }const serialized = new TextDecoder().decode(bytes);for (const text of ["private instructions", "private input"]) {if (recordInputs) expect(serialized).toContain(text);@@ -1111,21 +1149,22 @@ describe("exported agent telemetry contract", () => {});it.each([- ["private", false, false],- ["private", true, true],- ["public", false, true],- ["public", true, false],- ["public", true, true],+ ["private", false, false, "public"],+ ["private", true, true, "public"],+ ["public", false, true, "public"],+ ["public", true, false, "public"],+ ["public", true, true, "public"],+ ["public", true, true, "private"],] as const)(- "bounds public-channel principal IDs by origin %s and input/output ceiling %s/%s",- async (originAudience, recordInputs, recordOutputs) => {+ "bounds principal IDs and content markers by origin %s, input/output ceiling %s/%s, and live audience %s",+ async (originAudience, recordInputs, recordOutputs, liveAudience) => {const logs = captureLogRecords();const runtime = createRuntime();- const hooks = runtime.hooks.forTrace!(traceContext("child", "public"));+ const hooks = runtime.hooks.forTrace!(traceContext("child", liveAudience));const registered = vi.spyOn(runtimeGlobal, "getInstrumentationRuntime").mockReturnValue(runtime);- let ctx = contextFor("public");+ let ctx = contextFor(liveAudience);ctx.set(ParentTraceContextKey, {forwardedTracePolicy: {ceiling: { recordInputs, recordOutputs },@@ -1135,7 +1174,8 @@ describe("exported agent telemetry contract", () => {traceFlags: 1,traceId: "d".repeat(32),});- const includesIds = originAudience === "public" && recordInputs && recordOutputs;+ const includesIds =+ liveAudience === "public" && originAudience === "public" && recordInputs && recordOutputs;try {instrumentation.initializeSessionInstrumentation({ agentName: "child", ctx });expect(logs.records).toContainEqual(@@ -1182,6 +1222,12 @@ describe("exported agent telemetry contract", () => {const spans = runtime.exporter.getFinishedSpans();expect(spans).toHaveLength(2);for (const span of spans) {+ expect(span.attributes).toMatchObject({+ "agent.trace.content.input":+ liveAudience === "public" && originAudience === "public" && recordInputs,+ "agent.trace.content.output":+ liveAudience === "public" && originAudience === "public" && recordOutputs,+ });expect(span.attributes["agent.principal.current.type"]).toBe("service");expect(span.attributes["agent.principal.initiator.type"]).toBe("user");expect(span.attributes["agent.principal.current.id"]).toBe(diff --git a/packages/eve/src/tracing/eve/operation-input.ts b/packages/eve/src/tracing/eve/operation-input.tsindex 400142c62..b7ad25acc 100644--- a/packages/eve/src/tracing/eve/operation-input.ts+++ b/packages/eve/src/tracing/eve/operation-input.ts@@ -1,5 +1,3 @@-import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";-import { resolveConversationId } from "#shared/conversation-identity.js";import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";import {@@ -40,15 +38,6 @@ export function eveTurnIdentity(input: {};}-/** Content flags are recorded by eve's content processor, not withheld by the library. */-export function eveCapture(reference: SpanContext | undefined) {- return {- emit: ((reference?.traceFlags ?? 1) & 1) !== 0,- recordInputs: true,- recordOutputs: true,- };-}-/** eve's agent tracing, checkpointed in its workflow-context trace state. */export function createEveTracing(input: {readonly telemetry: AgentTelemetry;diff --git a/packages/eve/src/tracing/eve/policy.ts b/packages/eve/src/tracing/eve/policy.tsindex 00573a46c..aa08f1702 100644--- a/packages/eve/src/tracing/eve/policy.ts+++ b/packages/eve/src/tracing/eve/policy.ts@@ -15,6 +15,7 @@ import {import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";import { normalizeChannelAudience, type ChannelAudience } from "#shared/channel-audience.js";import type { ConversationEnvironment } from "#shared/conversation-context.js";+import { ConversationContextKey } from "#shared/conversation-context.js";export function createEveCapturePolicy(input: {readonly stateStore: AgentTraceStateStore;@@ -80,16 +81,21 @@ export function createEveCapturePolicy(input: {{ applyAudienceCeiling: false },);},- async forOperation(scope: InstrumentationAttemptScope) {+ async forOperation(scope: Pick<InstrumentationAttemptScope, "sessionId" | "channelAudience">) {const session = await input.stateStore.get("session", scope.sessionId);- const seed = resolveForwardedTraceSeed(contextStorage.getStore()?.get(SessionTraceSeedKey));+ const context = contextStorage.getStore();+ const seed = resolveForwardedTraceSeed(context?.get(SessionTraceSeedKey));const decision = seed?.decision ?? session?.decision;const effective =decision === undefined? undefined: applyLiveDeliveryAudienceCeiling(decision,- normalizeChannelAudience(scope.channelAudience),+ normalizeChannelAudience(+ scope.channelAudience ??+ context?.get(ConversationContextKey)?.audience ??+ session?.channelAudience,+ ),seed?.forwardedTracePolicy,input.environment,);diff --git a/packages/eve/src/tracing/eve/session.ts b/packages/eve/src/tracing/eve/session.tsindex 583d56ea1..31da18d53 100644--- a/packages/eve/src/tracing/eve/session.ts+++ b/packages/eve/src/tracing/eve/session.ts@@ -22,7 +22,8 @@ import type {} from "#tracing/eve/agent-trace-state.js";import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";import { eveActivationMetadata } from "#tracing/eve/metadata.js";-import { eveCapture, eveTurnIdentity } from "#tracing/eve/operation-input.js";+import { eveTurnIdentity } from "#tracing/eve/operation-input.js";+import type { createEveCapturePolicy } from "#tracing/eve/policy.js";import type { AgentTurnTraceState } from "#tracing/eve/agent-trace-state.js";import { applyPrincipalTraceDecision } from "#instrumentation/principal-summary.js";import { normalizeInstrumentationChannelKind } from "#internal/instrumentation.js";@@ -46,6 +47,7 @@ interface EveSessionTracingInput {readonly frameworkVersion: string;readonly stateStore: AgentTraceStateStore;readonly tracePolicy?: TraceCapturePolicy;+ readonly capturePolicy: ReturnType<typeof createEveCapturePolicy>;}type SessionMetadata = Omit<InstrumentationSessionStartedEvent, "idempotencyKey" | "type">;@@ -171,7 +173,7 @@ export function createEveSessionTracing(input: EveSessionTracingInput): EveSessiagentName,identity: eveTurnIdentity({ ...event, sessionId: event.sessionId }),framework: { name: "eve", version: input.frameworkVersion },- capture: eveCapture(reserved),+ capture: { emit: true, ...(await input.capturePolicy.forOperation(event)) },reference: reserved,parent: nested ? adoptedSpanContext(caller) : undefined,links: activationLinks(turn),