Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): classify open agent steps (#3989)

Chad Hietalacd3f26386fc9
diff --git a/.changeset/classify-open-agent-steps.md b/.changeset/classify-open-agent-steps.md
new file mode 100644
index 000000000..2d708517e
--- /dev/null
+++ b/.changeset/classify-open-agent-steps.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Include channel kind and origin on agent step spans so traces can be classified while a turn is still running or waiting for input.
diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.ts
index 166fb38ba..4d4cea292 100644
--- a/packages/eve/src/tracing/agent-otel-provider.test.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts
@@ -322,6 +322,7 @@ async function emitAttempt(input: {
async function publishTurnStarted(input: {
readonly channelAudience?: ChannelAudience;
+ readonly channelKind?: string;
readonly currentPrincipal?: InstrumentationPrincipalSummary;
readonly hooks: InstrumentationHooks;
readonly initiatorPrincipal?: InstrumentationPrincipalSummary;
@@ -339,7 +340,7 @@ async function publishTurnStarted(input: {
await input.hooks.publish({
agentName: "weather",
channelAudience: input.channelAudience ?? "public",
- channelKind: "http",
+ channelKind: input.channelKind ?? "http",
idempotencyKey: sessionIdempotencyKey(input.sessionId),
parentLineage: input.parentLineage,
parentTraceContext: input.parentTraceContext,
@@ -393,6 +394,53 @@ function nanos(hrTime: readonly [number, number]): bigint {
}
describe("createAgentOtelInstrumentation", () => {
+ it.each([
+ { channelKind: "channel:eve", expectedKind: "channel:eve", expectedOrigin: "channel" },
+ { channelKind: "unknown", expectedKind: undefined, expectedOrigin: undefined },
+ ])("classifies a step before the invocation span exists ($channelKind)", async (input) => {
+ const runtime = createRuntime();
+ const scope: InstrumentationAttemptScope = {
+ attemptId: "session-1:turn_0:0:0",
+ attemptIndex: 0,
+ functionId: "weather",
+ sessionId: "session-1",
+ stepIndex: 0,
+ turnId: "turn_0",
+ };
+ await publishTurnStarted({
+ channelKind: input.channelKind,
+ hooks: runtime.hooks,
+ sessionId: scope.sessionId,
+ turnId: scope.turnId,
+ turnSequence: 0,
+ });
+ await runtime.hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },
+ scope,
+ type: "step.attempt.started",
+ });
+ await runtime.hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ scope,
+ type: "step.attempt.completed",
+ });
+ await runtime.provider.forceFlush();
+
+ const spans = runtime.exporter.getFinishedSpans();
+ expect(byName(spans, "invoke_agent weather")).toHaveLength(0);
+ const attributes = byName(spans, "agent.step")[0]?.attributes;
+ if (input.expectedKind === undefined) {
+ expect(attributes).not.toHaveProperty("agent.channel.kind");
+ expect(attributes).not.toHaveProperty("agent.session.origin");
+ } else {
+ expect(attributes).toMatchObject({
+ "agent.channel.kind": input.expectedKind,
+ "agent.session.origin": input.expectedOrigin,
+ });
+ }
+ });
+
it.each(["public", "private"] as const)(
"exports explicit names for %s spans across bounded activation traces",
async (channelAudience) => {
@@ -731,6 +779,25 @@ describe("createAgentOtelInstrumentation", () => {
turnId: "child-turn",
turnSequence: 0,
});
+ const scope: InstrumentationAttemptScope = {
+ attemptId: "child-session:child-turn:0:0",
+ attemptIndex: 0,
+ functionId: "weather",
+ sessionId: "child-session",
+ stepIndex: 0,
+ turnId: "child-turn",
+ };
+ await runtime.hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },
+ scope,
+ type: "step.attempt.started",
+ });
+ await runtime.hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ scope,
+ type: "step.attempt.completed",
+ });
await completeTurn(runtime.hooks, "child-session", "child-turn");
await runtime.provider.forceFlush();
@@ -762,6 +829,9 @@ describe("createAgentOtelInstrumentation", () => {
expect(invocation.attributes).not.toHaveProperty("agent.schedule.id");
expect(invocation.attributes).not.toHaveProperty("agent.session.origin");
expect(invocation.attributes).not.toHaveProperty("agent.session.title");
+ const step = byName(spans, "agent.step")[0]!;
+ expect(step.attributes).not.toHaveProperty("agent.channel.kind");
+ expect(step.attributes).not.toHaveProperty("agent.session.origin");
expect(invocation.attributes).not.toHaveProperty("agent.principal.initiator.id");
expect(invocation.attributes).not.toHaveProperty("agent.root_run.id");
expect(invocation.attributes).not.toHaveProperty("agent.session.id");
diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.ts
index 7449a8526..c837ea34a 100644
--- a/packages/eve/src/tracing/agent-otel-provider.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.ts
@@ -221,6 +221,7 @@ export function createAgentOtelInstrumentation(
const onStepStarted = async (event: InstrumentationStepAttemptStartedEvent): Promise<void> => {
const turn = await input.stateStore.getTurn(event.scope.sessionId, event.scope.turnId);
if (turn === undefined || !isSampledTrace(turn.context)) return;
+ const session = await input.stateStore.getSession(event.scope.sessionId);
const turnContext = withChannelAudience(
contextFromSpanContext(turn.context),
event.scope.channelAudience,
@@ -232,20 +233,12 @@ export function createAgentOtelInstrumentation(
input.tracer.startSpan(
AGENT_SPAN_NAMES.step,
{
- attributes: {
- "agent.framework.name": "eve",
- "agent.framework.version": input.frameworkVersion,
- "agent.step.attempt": event.scope.attemptIndex,
- "agent.step.index": event.scope.stepIndex,
- "agent.turn.id": event.scope.turnId,
- "agent.name": event.scope.functionId,
- ...agentSpanNamingAttributes("agent.step"),
- ...agentTraceIdentityAttributes({
- rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,
- sessionId: event.scope.sessionId,
- }),
- ...runtimeAttributes.runtimeContextAttributes(event.runtimeContext),
- },
+ attributes: runtimeAttributes.agentStepAttributes({
+ event,
+ frameworkVersion: input.frameworkVersion,
+ session,
+ turn,
+ }),
links:
activeSpanContext === undefined || activeSpanContext.traceId === turn.context.traceId
? undefined
diff --git a/packages/eve/src/tracing/agent-otel-runtime-context.ts b/packages/eve/src/tracing/agent-otel-runtime-context.ts
index 2284bf773..ba37d9fb4 100644
--- a/packages/eve/src/tracing/agent-otel-runtime-context.ts
+++ b/packages/eve/src/tracing/agent-otel-runtime-context.ts
@@ -1,4 +1,5 @@
import type { AgentSessionTraceState, AgentTurnTraceState } from "#tracing/agent-trace-state.js";
+import type { InstrumentationStepAttemptStartedEvent } from "#instrumentation/lifecycle.js";
import { agentSpanNamingAttributes } from "#tracing/agent-span-naming.js";
import { agentInvocationSpanName } from "#tracing/agent-span-contract.js";
import { agentTraceIdentityAttributes } from "#tracing/agent-otel-attributes.js";
@@ -20,21 +21,8 @@ export function agentActivationAttributes(input: {
const recordsOutputs = recordsTrace && input.session?.decision?.recordOutputs === true;
const parentLineage = input.turn.parentLineage ?? input.session?.parentLineage;
const isSubagent = parentLineage !== undefined;
- const channelKind =
- input.turn.channelDelivery?.channelKind ??
- (isSubagent
- ? undefined
- : (input.session?.channelKind ??
- (input.session?.channelType === undefined
- ? undefined
- : normalizeInstrumentationChannelKind(input.session.channelType))));
+ const channelClassification = agentChannelClassificationAttributes(input.session, input.turn);
const scheduleId = isSubagent ? undefined : input.session?.scheduleId;
- const origin =
- isSubagent || channelKind === undefined
- ? undefined
- : scheduleId !== undefined
- ? "schedule"
- : "channel";
return {
"agent.framework.name": "eve",
"agent.framework.version": input.frameworkVersion,
@@ -43,14 +31,13 @@ export function agentActivationAttributes(input: {
...agentPrincipalAttributes(input.turn),
"agent.channel.delivery.id": input.turn.channelDelivery?.deliveryId,
"agent.channel.delivery.input": input.turn.channelDelivery?.inputAttribute,
- "agent.channel.kind": channelKind,
+ ...channelClassification,
"agent.channel.name": input.turn.channelDelivery?.channelName,
"agent.channel.request.id": input.turn.channelDelivery?.requestId,
"agent.parent_call.id": parentLineage?.callId,
"agent.parent_run.id": parentLineage?.sessionId,
"agent.run.type": isSubagent ? "subagent" : "session",
"agent.schedule.id": scheduleId,
- "agent.session.origin": origin,
"agent.session.title": !isSubagent && recordsInputs ? input.session?.title : undefined,
"agent.subagent.name": input.turn.subagentName,
"agent.trace.content.input": recordsInputs,
@@ -67,6 +54,62 @@ export function agentActivationAttributes(input: {
};
}
+export function agentChannelClassificationAttributes(
+ session: AgentSessionTraceState | undefined,
+ turn: AgentTurnTraceState,
+): {
+ readonly "agent.channel.kind": string | undefined;
+ readonly "agent.session.origin": string | undefined;
+} {
+ const isSubagent = (turn.parentLineage ?? session?.parentLineage) !== undefined;
+ const channelKind =
+ turn.channelDelivery?.channelKind ??
+ (isSubagent
+ ? undefined
+ : (session?.channelKind ??
+ (session?.channelType === undefined
+ ? undefined
+ : normalizeInstrumentationChannelKind(session.channelType))));
+ const origin =
+ isSubagent || channelKind === undefined
+ ? undefined
+ : session?.scheduleId !== undefined
+ ? "schedule"
+ : "channel";
+ return {
+ "agent.channel.kind": channelKind,
+ "agent.session.origin": origin,
+ };
+}
+
+export function agentStepAttributes(input: {
+ readonly event: InstrumentationStepAttemptStartedEvent;
+ readonly frameworkVersion: string;
+ readonly session?: AgentSessionTraceState;
+ readonly turn: AgentTurnTraceState;
+}) {
+ const { event, session, turn } = input;
+ const channelClassification = agentChannelClassificationAttributes(session, turn);
+ return {
+ "agent.framework.name": "eve",
+ "agent.framework.version": input.frameworkVersion,
+ "agent.step.attempt": event.scope.attemptIndex,
+ "agent.step.index": event.scope.stepIndex,
+ "agent.turn.id": event.scope.turnId,
+ "agent.name": event.scope.functionId,
+ // Leave an unresolved kind open for a later delivery to classify.
+ ...(channelClassification["agent.channel.kind"] === "unknown"
+ ? undefined
+ : channelClassification),
+ ...agentSpanNamingAttributes("agent.step"),
+ ...agentTraceIdentityAttributes({
+ rootSessionId: event.scope.rootSessionId ?? event.scope.sessionId,
+ sessionId: event.scope.sessionId,
+ }),
+ ...runtimeContextAttributes(event.runtimeContext),
+ };
+}
+
export function agentPrincipalAttributes(turn: AgentTurnTraceState): Record<string, string> {
const attributes: Record<string, string> = {};
setOptionalAttribute(attributes, "agent.principal.current.id", turn.currentPrincipal?.id);