The Open Framework for Building Agents
fix(eve): classify open agent steps (#3989)
Chad Hietalacd3f26386fc9
diff --git a/.changeset/classify-open-agent-steps.md b/.changeset/classify-open-agent-steps.mdnew file mode 100644index 000000000..2d708517e--- /dev/null+++ b/.changeset/classify-open-agent-steps.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Include channel kind and origin on agent step spans so traces can be classified while a turn is still running or waiting for input.diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex 166fb38ba..4d4cea292 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -322,6 +322,7 @@ async function emitAttempt(input: {async function publishTurnStarted(input: {readonly channelAudience?: ChannelAudience;+ readonly channelKind?: string;readonly currentPrincipal?: InstrumentationPrincipalSummary;readonly hooks: InstrumentationHooks;readonly initiatorPrincipal?: InstrumentationPrincipalSummary;@@ -339,7 +340,7 @@ async function publishTurnStarted(input: {await input.hooks.publish({agentName: "weather",channelAudience: input.channelAudience ?? "public",- channelKind: "http",+ channelKind: input.channelKind ?? "http",idempotencyKey: sessionIdempotencyKey(input.sessionId),parentLineage: input.parentLineage,parentTraceContext: input.parentTraceContext,@@ -393,6 +394,53 @@ function nanos(hrTime: readonly [number, number]): bigint {}describe("createAgentOtelInstrumentation", () => {+ it.each([+ { channelKind: "channel:eve", expectedKind: "channel:eve", expectedOrigin: "channel" },+ { channelKind: "unknown", expectedKind: undefined, expectedOrigin: undefined },+ ])("classifies a step before the invocation span exists ($channelKind)", async (input) => {+ const runtime = createRuntime();+ const scope: InstrumentationAttemptScope = {+ attemptId: "session-1:turn_0:0:0",+ attemptIndex: 0,+ functionId: "weather",+ sessionId: "session-1",+ stepIndex: 0,+ turnId: "turn_0",+ };+ await publishTurnStarted({+ channelKind: input.channelKind,+ hooks: runtime.hooks,+ sessionId: scope.sessionId,+ turnId: scope.turnId,+ turnSequence: 0,+ });+ await runtime.hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },+ scope,+ type: "step.attempt.started",+ });+ await runtime.hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ type: "step.attempt.completed",+ });+ await runtime.provider.forceFlush();++ const spans = runtime.exporter.getFinishedSpans();+ expect(byName(spans, "invoke_agent weather")).toHaveLength(0);+ const attributes = byName(spans, "agent.step")[0]?.attributes;+ if (input.expectedKind === undefined) {+ expect(attributes).not.toHaveProperty("agent.channel.kind");+ expect(attributes).not.toHaveProperty("agent.session.origin");+ } else {+ expect(attributes).toMatchObject({+ "agent.channel.kind": input.expectedKind,+ "agent.session.origin": input.expectedOrigin,+ });+ }+ });+it.each(["public", "private"] as const)("exports explicit names for %s spans across bounded activation traces",async (channelAudience) => {@@ -731,6 +779,25 @@ describe("createAgentOtelInstrumentation", () => {turnId: "child-turn",turnSequence: 0,});+ const scope: InstrumentationAttemptScope = {+ attemptId: "child-session:child-turn:0:0",+ attemptIndex: 0,+ functionId: "weather",+ sessionId: "child-session",+ stepIndex: 0,+ turnId: "child-turn",+ };+ await runtime.hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },+ scope,+ type: "step.attempt.started",+ });+ await runtime.hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ type: "step.attempt.completed",+ });await completeTurn(runtime.hooks, "child-session", "child-turn");await runtime.provider.forceFlush();@@ -762,6 +829,9 @@ describe("createAgentOtelInstrumentation", () => {expect(invocation.attributes).not.toHaveProperty("agent.schedule.id");expect(invocation.attributes).not.toHaveProperty("agent.session.origin");expect(invocation.attributes).not.toHaveProperty("agent.session.title");+ const step = byName(spans, "agent.step")[0]!;+ expect(step.attributes).not.toHaveProperty("agent.channel.kind");+ expect(step.attributes).not.toHaveProperty("agent.session.origin");expect(invocation.attributes).not.toHaveProperty("agent.principal.initiator.id");expect(invocation.attributes).not.toHaveProperty("agent.root_run.id");expect(invocation.attributes).not.toHaveProperty("agent.session.id");diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex 7449a8526..c837ea34a 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -221,6 +221,7 @@ export function createAgentOtelInstrumentation(const onStepStarted = async (event: InstrumentationStepAttemptStartedEvent): Promise<void> => {const turn = await input.stateStore.getTurn(event.scope.sessionId, event.scope.turnId);if (turn === undefined || !isSampledTrace(turn.context)) return;+ const session = await input.stateStore.getSession(event.scope.sessionId);const turnContext = withChannelAudience(contextFromSpanContext(turn.context),event.scope.channelAudience,@@ -232,20 +233,12 @@ export function createAgentOtelInstrumentation(input.tracer.startSpan(AGENT_SPAN_NAMES.step,{- attributes: {- "agent.framework.name": "eve",- "agent.framework.version": input.frameworkVersion,- "agent.step.attempt": event.scope.attemptIndex,- "agent.step.index": event.scope.stepIndex,- "agent.turn.id": event.scope.turnId,- "agent.name": event.scope.functionId,- ...agentSpanNamingAttributes("agent.step"),- ...agentTraceIdentityAttributes({- rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,- sessionId: event.scope.sessionId,- }),- ...runtimeAttributes.runtimeContextAttributes(event.runtimeContext),- },+ attributes: runtimeAttributes.agentStepAttributes({+ event,+ frameworkVersion: input.frameworkVersion,+ session,+ turn,+ }),links:activeSpanContext === undefined || activeSpanContext.traceId === turn.context.traceId? undefineddiff --git a/packages/eve/src/tracing/agent-otel-runtime-context.ts b/packages/eve/src/tracing/agent-otel-runtime-context.tsindex 2284bf773..ba37d9fb4 100644--- a/packages/eve/src/tracing/agent-otel-runtime-context.ts+++ b/packages/eve/src/tracing/agent-otel-runtime-context.ts@@ -1,4 +1,5 @@import type { AgentSessionTraceState, AgentTurnTraceState } from "#tracing/agent-trace-state.js";+import type { InstrumentationStepAttemptStartedEvent } from "#instrumentation/lifecycle.js";import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";import { agentInvocationSpanName } from "#tracing/agent-span-contract.js";import { agentTraceIdentityAttributes } from "#tracing/agent-otel-attributes.js";@@ -20,21 +21,8 @@ export function agentActivationAttributes(input: {const recordsOutputs = recordsTrace && input.session?.decision?.recordOutputs === true;const parentLineage = input.turn.parentLineage ?? input.session?.parentLineage;const isSubagent = parentLineage !== undefined;- const channelKind =- input.turn.channelDelivery?.channelKind ??- (isSubagent- ? undefined- : (input.session?.channelKind ??- (input.session?.channelType === undefined- ? undefined- : normalizeInstrumentationChannelKind(input.session.channelType))));+ const channelClassification = agentChannelClassificationAttributes(input.session, input.turn);const scheduleId = isSubagent ? undefined : input.session?.scheduleId;- const origin =- isSubagent || channelKind === undefined- ? undefined- : scheduleId !== undefined- ? "schedule"- : "channel";return {"agent.framework.name": "eve","agent.framework.version": input.frameworkVersion,@@ -43,14 +31,13 @@ export function agentActivationAttributes(input: {...agentPrincipalAttributes(input.turn),"agent.channel.delivery.id": input.turn.channelDelivery?.deliveryId,"agent.channel.delivery.input": input.turn.channelDelivery?.inputAttribute,- "agent.channel.kind": channelKind,+ ...channelClassification,"agent.channel.name": input.turn.channelDelivery?.channelName,"agent.channel.request.id": input.turn.channelDelivery?.requestId,"agent.parent_call.id": parentLineage?.callId,"agent.parent_run.id": parentLineage?.sessionId,"agent.run.type": isSubagent ? "subagent" : "session","agent.schedule.id": scheduleId,- "agent.session.origin": origin,"agent.session.title": !isSubagent && recordsInputs ? input.session?.title : undefined,"agent.subagent.name": input.turn.subagentName,"agent.trace.content.input": recordsInputs,@@ -67,6 +54,62 @@ export function agentActivationAttributes(input: {};}+export function agentChannelClassificationAttributes(+ session: AgentSessionTraceState | undefined,+ turn: AgentTurnTraceState,+): {+ readonly "agent.channel.kind": string | undefined;+ readonly "agent.session.origin": string | undefined;+} {+ const isSubagent = (turn.parentLineage ?? session?.parentLineage) !== undefined;+ const channelKind =+ turn.channelDelivery?.channelKind ??+ (isSubagent+ ? undefined+ : (session?.channelKind ??+ (session?.channelType === undefined+ ? undefined+ : normalizeInstrumentationChannelKind(session.channelType))));+ const origin =+ isSubagent || channelKind === undefined+ ? undefined+ : session?.scheduleId !== undefined+ ? "schedule"+ : "channel";+ return {+ "agent.channel.kind": channelKind,+ "agent.session.origin": origin,+ };+}++export function agentStepAttributes(input: {+ readonly event: InstrumentationStepAttemptStartedEvent;+ readonly frameworkVersion: string;+ readonly session?: AgentSessionTraceState;+ readonly turn: AgentTurnTraceState;+}) {+ const { event, session, turn } = input;+ const channelClassification = agentChannelClassificationAttributes(session, turn);+ return {+ "agent.framework.name": "eve",+ "agent.framework.version": input.frameworkVersion,+ "agent.step.attempt": event.scope.attemptIndex,+ "agent.step.index": event.scope.stepIndex,+ "agent.turn.id": event.scope.turnId,+ "agent.name": event.scope.functionId,+ // Leave an unresolved kind open for a later delivery to classify.+ ...(channelClassification["agent.channel.kind"] === "unknown"+ ? undefined+ : channelClassification),+ ...agentSpanNamingAttributes("agent.step"),+ ...agentTraceIdentityAttributes({+ rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,+ sessionId: event.scope.sessionId,+ }),+ ...runtimeContextAttributes(event.runtimeContext),+ };+}+export function agentPrincipalAttributes(turn: AgentTurnTraceState): Record<string, string> {const attributes: Record<string, string> = {};setOptionalAttribute(attributes, "agent.principal.current.id", turn.currentPrincipal?.id);