Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): align activation content markers with trace policy (#4850)

Chad Hietalae4e0cd1757d9
diff --git a/.changeset/accurate-activation-content-markers.md b/.changeset/accurate-activation-content-markers.md
new file mode 100644
index 000000000..407c9366d
--- /dev/null
+++ b/.changeset/accurate-activation-content-markers.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Fix activation trace-content markers to respect the resolved session policy, provider capture settings, and live audience ceiling. Metadata-only sessions now report withheld content accurately without removing content allowed by an explicit trace policy.
diff --git a/packages/eve/src/tracing/eve/agent-otel-provider.ts b/packages/eve/src/tracing/eve/agent-otel-provider.ts
index 5ff8770c9..5b640bf41 100644
--- a/packages/eve/src/tracing/eve/agent-otel-provider.ts
+++ b/packages/eve/src/tracing/eve/agent-otel-provider.ts
@@ -73,10 +73,16 @@ export function createAgentOtelInstrumentation(
const recordInputs = input.recordInputs ?? false;
const recordOutputs = input.recordOutputs ?? false;
const tracing = createEveTracing(input);
+ const capturePolicy = createEveCapturePolicy({
+ stateStore: input.stateStore,
+ environment,
+ recordInputs,
+ recordOutputs,
+ });
// Attempts run in one process; a lost serverless worker retries the whole step.
const steps = new Map<string, AttemptOperation>();
const modelSpans = new Map<string, Map<string, ModelOperation>>();
- const sessionTracing = createEveSessionTracing({ ...input, environment, tracing });
+ const sessionTracing = createEveSessionTracing({ ...input, environment, tracing, capturePolicy });
const { prepareSessionTrace, prepareTurnTrace } = sessionTracing;
/** The attempt in its turn's tree, hydrating it when another process started it. */
@@ -106,13 +112,6 @@ export function createAgentOtelInstrumentation(
});
const memory = createAgentMemoryInstrumentation({ ...input, environment, tracing });
- const capturePolicy = createEveCapturePolicy({
- stateStore: input.stateStore,
- environment,
- recordInputs,
- recordOutputs,
- });
-
function turnState(scope: { sessionId: string; turnId: string }) {
return input.stateStore.get("turn", JSON.stringify([scope.sessionId, scope.turnId]));
}
diff --git a/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts b/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts
index cee75743b..2cae0dc32 100644
--- a/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts
+++ b/packages/eve/src/tracing/eve/agent-telemetry-contract.integration.test.ts
@@ -16,6 +16,7 @@ import {
ParentSessionKey,
ParentTraceContextKey,
SessionTraceSeedKey,
+ SessionTitleKey,
TraceRootKey,
} from "#context/keys.js";
import { deserializeContext, serializeContext } from "#context/serialize.js";
@@ -68,7 +69,10 @@ const traceContext = (agentName: string, audience: ChannelAudience) => ({
principalType: "anonymous",
});
-function createRuntime(tracePolicy?: TraceCapturePolicy) {
+function createRuntime(
+ tracePolicy?: TraceCapturePolicy,
+ content = { recordInputs: true, recordOutputs: true },
+) {
const exporter = new InMemorySpanExporter();
const metadata = new InMemorySpanExporter();
const idGenerator = new AgentSpanIdGenerator();
@@ -84,8 +88,8 @@ function createRuntime(tracePolicy?: TraceCapturePolicy) {
const agent = createAgentOtelInstrumentation({
tracePolicy,
frameworkVersion: "test",
- recordInputs: true,
- recordOutputs: true,
+ environment: "production",
+ ...content,
stateStore: new ContextAgentTraceStateStore(),
telemetry: otelTelemetry({
provider,
@@ -106,8 +110,7 @@ function createRuntime(tracePolicy?: TraceCapturePolicy) {
ownsAgentSpans: true,
otelSettings: {
tracePolicy,
- recordInputs: true,
- recordOutputs: true,
+ ...content,
traceChannelRequests: false,
},
forceFlush: () => provider.forceFlush(),
@@ -670,19 +673,39 @@ describe("exported agent telemetry contract", () => {
it.each([
{ audience: "public", explicit: false, recordInputs: true, recordOutputs: true },
+ { audience: "public", explicit: true, recordInputs: false, recordOutputs: false },
{ audience: "private", explicit: false, recordInputs: false, recordOutputs: false },
+ { audience: "unknown", explicit: false, recordInputs: false, recordOutputs: false },
{ audience: "private", explicit: true, recordInputs: true, recordOutputs: true },
{ audience: "unknown", explicit: true, recordInputs: true, recordOutputs: true },
{ audience: "private", explicit: true, recordInputs: true, recordOutputs: false },
{ audience: "private", explicit: true, recordInputs: false, recordOutputs: true },
+ {
+ audience: "public",
+ explicit: false,
+ recordInputs: false,
+ recordOutputs: true,
+ providerContent: { recordInputs: false, recordOutputs: true },
+ },
+ {
+ audience: "public",
+ explicit: false,
+ recordInputs: true,
+ recordOutputs: false,
+ providerContent: { recordInputs: true, recordOutputs: false },
+ },
] as const)(
"round-trips the normalized $audience v4 trace forest through OTLP (explicit=$explicit, inputs=$recordInputs, outputs=$recordOutputs)",
- async ({ audience, explicit, recordInputs, recordOutputs }) => {
+ async (testCase) => {
+ const { audience, explicit, recordInputs, recordOutputs } = testCase;
+ const includesIds = audience === "public" && (!explicit || (recordInputs && recordOutputs));
const runtime = createRuntime(
explicit ? () => ({ emit: true, recordInputs, recordOutputs }) : undefined,
+ "providerContent" in testCase ? testCase.providerContent : undefined,
);
let parent = contextFor(audience);
parent.set(ConversationIdKey, "original-conversation");
+ parent.set(SessionTitleKey, "private initial title");
const scope = scopeFor("parent", audience);
const hooks = runtime.hooks.forTrace!(traceContext("parent", audience));
const actionKey = actionIdempotencyKey("parent", "turn_0", "workflow");
@@ -972,13 +995,22 @@ describe("exported agent telemetry contract", () => {
}
expect(parsed.filter(isAgentTurnSpan)).toHaveLength(4);
for (const activation of parsed.filter(isAgentTurnSpan)) {
+ expect(activation.attributes).toMatchObject({
+ "agent.trace.content.input": recordInputs,
+ "agent.trace.content.output": recordOutputs,
+ });
+ expect(activation.attributes["agent.session.title"]).toBe(
+ activation.attributes["agent.run.id"] === "parent" && recordInputs
+ ? "private initial title"
+ : undefined,
+ );
expect(activation.attributes["agent.principal.current.type"]).toBe("service");
expect(activation.attributes["agent.principal.initiator.type"]).toBe("user");
expect(activation.attributes["agent.principal.current.id"]).toBe(
- audience === "public" ? "current-user" : undefined,
+ includesIds ? "current-user" : undefined,
);
expect(activation.attributes["agent.principal.initiator.id"]).toBe(
- audience === "public" ? "initiator-user" : undefined,
+ includesIds ? "initiator-user" : undefined,
);
}
expect(new TextDecoder().decode(bytes)).not.toContain("auth-only-secret");
@@ -1041,6 +1073,12 @@ describe("exported agent telemetry contract", () => {
parsed.map((span) => [span.name, span.name]),
);
expect(metadata).not.toContain("private ");
+ for (const activation of metadataSpans.filter(isAgentTurnSpan)) {
+ expect(activation.attributes).toMatchObject({
+ "agent.trace.content.input": false,
+ "agent.trace.content.output": false,
+ });
+ }
const serialized = new TextDecoder().decode(bytes);
for (const text of ["private instructions", "private input"]) {
if (recordInputs) expect(serialized).toContain(text);
@@ -1111,21 +1149,22 @@ describe("exported agent telemetry contract", () => {
});
it.each([
- ["private", false, false],
- ["private", true, true],
- ["public", false, true],
- ["public", true, false],
- ["public", true, true],
+ ["private", false, false, "public"],
+ ["private", true, true, "public"],
+ ["public", false, true, "public"],
+ ["public", true, false, "public"],
+ ["public", true, true, "public"],
+ ["public", true, true, "private"],
] as const)(
- "bounds public-channel principal IDs by origin %s and input/output ceiling %s/%s",
- async (originAudience, recordInputs, recordOutputs) => {
+ "bounds principal IDs and content markers by origin %s, input/output ceiling %s/%s, and live audience %s",
+ async (originAudience, recordInputs, recordOutputs, liveAudience) => {
const logs = captureLogRecords();
const runtime = createRuntime();
- const hooks = runtime.hooks.forTrace!(traceContext("child", "public"));
+ const hooks = runtime.hooks.forTrace!(traceContext("child", liveAudience));
const registered = vi
.spyOn(runtimeGlobal, "getInstrumentationRuntime")
.mockReturnValue(runtime);
- let ctx = contextFor("public");
+ let ctx = contextFor(liveAudience);
ctx.set(ParentTraceContextKey, {
forwardedTracePolicy: {
ceiling: { recordInputs, recordOutputs },
@@ -1135,7 +1174,8 @@ describe("exported agent telemetry contract", () => {
traceFlags: 1,
traceId: "d".repeat(32),
});
- const includesIds = originAudience === "public" && recordInputs && recordOutputs;
+ const includesIds =
+ liveAudience === "public" && originAudience === "public" && recordInputs && recordOutputs;
try {
instrumentation.initializeSessionInstrumentation({ agentName: "child", ctx });
expect(logs.records).toContainEqual(
@@ -1182,6 +1222,12 @@ describe("exported agent telemetry contract", () => {
const spans = runtime.exporter.getFinishedSpans();
expect(spans).toHaveLength(2);
for (const span of spans) {
+ expect(span.attributes).toMatchObject({
+ "agent.trace.content.input":
+ liveAudience === "public" && originAudience === "public" && recordInputs,
+ "agent.trace.content.output":
+ liveAudience === "public" && originAudience === "public" && recordOutputs,
+ });
expect(span.attributes["agent.principal.current.type"]).toBe("service");
expect(span.attributes["agent.principal.initiator.type"]).toBe("user");
expect(span.attributes["agent.principal.current.id"]).toBe(
diff --git a/packages/eve/src/tracing/eve/operation-input.ts b/packages/eve/src/tracing/eve/operation-input.ts
index 400142c62..b7ad25acc 100644
--- a/packages/eve/src/tracing/eve/operation-input.ts
+++ b/packages/eve/src/tracing/eve/operation-input.ts
@@ -1,5 +1,3 @@
-import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";
-
import { resolveConversationId } from "#shared/conversation-identity.js";
import type { AgentTraceStateStore } from "#tracing/eve/agent-trace-state.js";
import {
@@ -40,15 +38,6 @@ export function eveTurnIdentity(input: {
};
}
-/** Content flags are recorded by eve's content processor, not withheld by the library. */
-export function eveCapture(reference: SpanContext | undefined) {
- return {
- emit: ((reference?.traceFlags ?? 1) & 1) !== 0,
- recordInputs: true,
- recordOutputs: true,
- };
-}
-
/** eve's agent tracing, checkpointed in its workflow-context trace state. */
export function createEveTracing(input: {
readonly telemetry: AgentTelemetry;
diff --git a/packages/eve/src/tracing/eve/policy.ts b/packages/eve/src/tracing/eve/policy.ts
index 00573a46c..aa08f1702 100644
--- a/packages/eve/src/tracing/eve/policy.ts
+++ b/packages/eve/src/tracing/eve/policy.ts
@@ -15,6 +15,7 @@ import {
import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";
import { normalizeChannelAudience, type ChannelAudience } from "#shared/channel-audience.js";
import type { ConversationEnvironment } from "#shared/conversation-context.js";
+import { ConversationContextKey } from "#shared/conversation-context.js";
export function createEveCapturePolicy(input: {
readonly stateStore: AgentTraceStateStore;
@@ -80,16 +81,21 @@ export function createEveCapturePolicy(input: {
{ applyAudienceCeiling: false },
);
},
- async forOperation(scope: InstrumentationAttemptScope) {
+ async forOperation(scope: Pick<InstrumentationAttemptScope, "sessionId" | "channelAudience">) {
const session = await input.stateStore.get("session", scope.sessionId);
- const seed = resolveForwardedTraceSeed(contextStorage.getStore()?.get(SessionTraceSeedKey));
+ const context = contextStorage.getStore();
+ const seed = resolveForwardedTraceSeed(context?.get(SessionTraceSeedKey));
const decision = seed?.decision ?? session?.decision;
const effective =
decision === undefined
? undefined
: applyLiveDeliveryAudienceCeiling(
decision,
- normalizeChannelAudience(scope.channelAudience),
+ normalizeChannelAudience(
+ scope.channelAudience ??
+ context?.get(ConversationContextKey)?.audience ??
+ session?.channelAudience,
+ ),
seed?.forwardedTracePolicy,
input.environment,
);
diff --git a/packages/eve/src/tracing/eve/session.ts b/packages/eve/src/tracing/eve/session.ts
index 583d56ea1..31da18d53 100644
--- a/packages/eve/src/tracing/eve/session.ts
+++ b/packages/eve/src/tracing/eve/session.ts
@@ -22,7 +22,8 @@ import type {
} from "#tracing/eve/agent-trace-state.js";
import { readInstrumentationDecision } from "#shared/instrumentation-decision.js";
import { eveActivationMetadata } from "#tracing/eve/metadata.js";
-import { eveCapture, eveTurnIdentity } from "#tracing/eve/operation-input.js";
+import { eveTurnIdentity } from "#tracing/eve/operation-input.js";
+import type { createEveCapturePolicy } from "#tracing/eve/policy.js";
import type { AgentTurnTraceState } from "#tracing/eve/agent-trace-state.js";
import { applyPrincipalTraceDecision } from "#instrumentation/principal-summary.js";
import { normalizeInstrumentationChannelKind } from "#internal/instrumentation.js";
@@ -46,6 +47,7 @@ interface EveSessionTracingInput {
readonly frameworkVersion: string;
readonly stateStore: AgentTraceStateStore;
readonly tracePolicy?: TraceCapturePolicy;
+ readonly capturePolicy: ReturnType<typeof createEveCapturePolicy>;
}
type SessionMetadata = Omit<InstrumentationSessionStartedEvent, "idempotencyKey" | "type">;
@@ -171,7 +173,7 @@ export function createEveSessionTracing(input: EveSessionTracingInput): EveSessi
agentName,
identity: eveTurnIdentity({ ...event, sessionId: event.sessionId }),
framework: { name: "eve", version: input.frameworkVersion },
- capture: eveCapture(reserved),
+ capture: { emit: true, ...(await input.capturePolicy.forOperation(event)) },
reference: reserved,
parent: nested ? adoptedSpanContext(caller) : undefined,
links: activationLinks(turn),