Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(eve): add agent-semantic OTel provider (#1192)

Chad Hietala08de13fce849
diff --git a/packages/eve/package.json b/packages/eve/package.json
index 487bff7b3..39cc9873f 100644
--- a/packages/eve/package.json
+++ b/packages/eve/package.json
@@ -315,6 +315,7 @@
"@chat-adapter/twilio": "4.34.0",
"@clack/core": "1.3.1",
"@nuxt/kit": "^4.0.0",
+ "@opentelemetry/sdk-trace-base": "^2.6.1",
"@standard-schema/spec": "1.1.0",
"@sveltejs/kit": "^2.0.0",
"@types/json-schema": "7.0.15",
diff --git a/packages/eve/src/harness/agent-otel-provider.test.ts b/packages/eve/src/harness/agent-otel-provider.test.ts
new file mode 100644
index 000000000..87c42a3a2
--- /dev/null
+++ b/packages/eve/src/harness/agent-otel-provider.test.ts
@@ -0,0 +1,283 @@
+import { SpanStatusCode } from "@opentelemetry/api";
+import {
+ BasicTracerProvider,
+ InMemorySpanExporter,
+ SimpleSpanProcessor,
+ type ReadableSpan,
+} from "@opentelemetry/sdk-trace-base";
+import { describe, expect, it } from "vitest";
+
+import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";
+import {
+ createAgentOtelInstrumentation,
+ InMemoryAgentTraceStateStore,
+} from "#harness/agent-otel-provider.js";
+import {
+ createInstrumentationHooks,
+ type InstrumentationAttemptScope,
+ type InstrumentationContextRunner,
+ type InstrumentationHooks,
+} from "#harness/instrumentation-lifecycle.js";
+
+interface TestRuntime {
+ readonly exporter: InMemorySpanExporter;
+ readonly hooks: InstrumentationHooks;
+ readonly provider: BasicTracerProvider;
+ readonly runInContext: InstrumentationContextRunner;
+}
+
+function createRuntime(stateStore = new InMemoryAgentTraceStateStore()): TestRuntime {
+ const exporter = new InMemorySpanExporter();
+ const provider = new BasicTracerProvider({
+ spanProcessors: [new SimpleSpanProcessor(exporter)],
+ });
+ const agentOtel = createAgentOtelInstrumentation({
+ frameworkVersion: "test",
+ stateStore,
+ tracer: provider.getTracer("eve.agent"),
+ });
+ const hooks = createInstrumentationHooks([agentOtel.hook]);
+ return { exporter, hooks, provider, runInContext: agentOtel.runInContext };
+}
+
+async function emitAttempt(input: {
+ readonly attemptIndex?: number;
+ readonly hooks: InstrumentationHooks;
+ readonly runInContext: InstrumentationContextRunner;
+ readonly sessionId: string;
+ readonly toolError?: Error;
+ readonly turnAlreadyStarted?: boolean;
+ readonly turnId: string;
+ readonly turnSequence: number;
+}): Promise<void> {
+ const scope: InstrumentationAttemptScope = {
+ attemptId: `${input.sessionId}:${input.turnId}:0:${input.attemptIndex ?? 0}`,
+ attemptIndex: input.attemptIndex ?? 0,
+ functionId: "weather",
+ sessionId: input.sessionId,
+ stepIndex: 0,
+ turnId: input.turnId,
+ };
+ if (input.turnAlreadyStarted !== true) {
+ await publishTurnStarted(input);
+ }
+
+ const bridge = createAiSdkHookBridge(scope, input.hooks, input.runInContext);
+ Reflect.apply(bridge.onStart!, bridge, [
+ {
+ callId: "call-1",
+ instructions: "private prompt",
+ messages: [{ content: "private message", role: "user" }],
+ modelId: "claude-test",
+ operationId: "ai.streamText",
+ provider: "anthropic",
+ },
+ ]);
+ await Reflect.apply(bridge.onStepStart!, bridge, [{ callId: "call-1", stepNumber: 0 }]);
+ await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [
+ { callId: "call-1", messages: [], modelId: "claude-test", provider: "anthropic" },
+ ]);
+ await bridge.executeLanguageModelCall!({ callId: "call-1", execute: async () => undefined });
+ await Reflect.apply(bridge.onLanguageModelCallEnd!, bridge, [
+ {
+ callId: "call-1",
+ content: [],
+ finishReason: "tool-calls",
+ performance: { responseTimeMs: 10 },
+ responseId: "response-1",
+ usage: { inputTokens: 10, outputTokens: 5 },
+ },
+ ]);
+ await Reflect.apply(bridge.onToolExecutionStart!, bridge, [
+ {
+ callId: "call-1",
+ toolCall: { input: { secret: "value" }, toolCallId: "tool-1", toolName: "weather" },
+ },
+ ]);
+ await bridge.executeTool!({
+ callId: "call-1",
+ execute: async () => undefined,
+ toolCallId: "tool-1",
+ });
+ await Reflect.apply(bridge.onToolExecutionEnd!, bridge, [
+ {
+ callId: "call-1",
+ messages: [],
+ toolCall: { input: {}, toolCallId: "tool-1", toolName: "weather" },
+ toolExecutionMs: 1,
+ toolOutput:
+ input.toolError === undefined
+ ? { output: { temperature: 72 }, type: "tool-result" }
+ : { error: input.toolError, type: "tool-error" },
+ },
+ ]);
+
+ await input.hooks.publish({ scope, type: "attempt.completed" });
+ await input.hooks.publish({
+ sessionId: input.sessionId,
+ turnId: input.turnId,
+ type: "turn.completed",
+ });
+ await input.hooks.publish({
+ sessionId: input.sessionId,
+ turnId: input.turnId,
+ type: "session.waiting",
+ });
+}
+
+async function publishTurnStarted(input: {
+ readonly hooks: InstrumentationHooks;
+ readonly sessionId: string;
+ readonly turnId: string;
+ readonly turnSequence: number;
+}): Promise<void> {
+ await input.hooks.publish({
+ agentName: "weather",
+ channelKind: "http",
+ rootSessionId: input.sessionId,
+ sessionId: input.sessionId,
+ type: "session.started",
+ });
+ await input.hooks.publish({
+ rootSessionId: input.sessionId,
+ sequence: input.turnSequence,
+ sessionId: input.sessionId,
+ turnId: input.turnId,
+ type: "turn.started",
+ });
+}
+
+function byName(spans: readonly ReadableSpan[], name: string): ReadableSpan[] {
+ return spans.filter((span) => span.name === name);
+}
+
+describe("createAgentOtelInstrumentation", () => {
+ it("emits the agent hierarchy in one session trace", async () => {
+ const runtime = createRuntime();
+ await emitAttempt({
+ hooks: runtime.hooks,
+ runInContext: runtime.runInContext,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ turnSequence: 0,
+ });
+ await runtime.provider.forceFlush();
+
+ const spans = runtime.exporter.getFinishedSpans();
+ const session = byName(spans, "agent.session")[0]!;
+ const turn = byName(spans, "agent.turn")[0]!;
+ const turnTerminal = byName(spans, "agent.turn.terminal")[0]!;
+ const step = byName(spans, "agent.step")[0]!;
+ const action = byName(spans, "agent.action")[0]!;
+
+ expect(session.parentSpanContext).toBeUndefined();
+ expect(turn.parentSpanContext?.spanId).toBe(session.spanContext().spanId);
+ expect(turnTerminal.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);
+ expect(step.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);
+ expect(action.parentSpanContext?.spanId).toBe(step.spanContext().spanId);
+ expect(new Set(spans.map((span) => span.spanContext().traceId))).toHaveLength(1);
+ expect(turn.events.map((event) => event.name)).toEqual(["turn.started", "session.started"]);
+ expect(turnTerminal.events.map((event) => event.name)).toEqual([
+ "turn.completed",
+ "session.waiting",
+ ]);
+ expect(step.attributes).toMatchObject({
+ "agent.framework.name": "eve",
+ "agent.model.id": "claude-test",
+ "agent.model.provider": "anthropic",
+ "agent.root.session.id": "session-1",
+ "agent.usage.input_tokens": 10,
+ "agent.usage.output_tokens": 5,
+ });
+ expect(action.attributes).toMatchObject({
+ "agent.action.kind": "tool",
+ "agent.action.name": "weather",
+ "agent.framework.name": "eve",
+ "agent.root.session.id": "session-1",
+ });
+ expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("private");
+ });
+
+ it("reuses one trace id across turns", async () => {
+ const stateStore = new InMemoryAgentTraceStateStore();
+ const firstRuntime = createRuntime(stateStore);
+ await emitAttempt({
+ hooks: firstRuntime.hooks,
+ runInContext: firstRuntime.runInContext,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ turnSequence: 0,
+ });
+ await firstRuntime.provider.forceFlush();
+
+ // A new provider instance models resumption in another Workflow worker.
+ const secondRuntime = createRuntime(stateStore);
+ await emitAttempt({
+ hooks: secondRuntime.hooks,
+ runInContext: secondRuntime.runInContext,
+ sessionId: "session-1",
+ turnId: "turn-2",
+ turnSequence: 1,
+ });
+ await secondRuntime.provider.forceFlush();
+
+ const turns = [
+ ...byName(firstRuntime.exporter.getFinishedSpans(), "agent.turn"),
+ ...byName(secondRuntime.exporter.getFinishedSpans(), "agent.turn"),
+ ];
+ expect(turns).toHaveLength(2);
+ expect(turns[0]!.spanContext().traceId).toBe(turns[1]!.spanContext().traceId);
+ expect(
+ turns.flatMap((turn) => turn.events).filter((event) => event.name === "session.started"),
+ ).toHaveLength(1);
+ });
+
+ it("restores durable turn context when a replacement worker runs the attempt", async () => {
+ const stateStore = new InMemoryAgentTraceStateStore();
+ const firstRuntime = createRuntime(stateStore);
+ await publishTurnStarted({
+ hooks: firstRuntime.hooks,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ turnSequence: 0,
+ });
+ await firstRuntime.provider.forceFlush();
+
+ const replacementRuntime = createRuntime(stateStore);
+ await emitAttempt({
+ hooks: replacementRuntime.hooks,
+ runInContext: replacementRuntime.runInContext,
+ sessionId: "session-1",
+ turnAlreadyStarted: true,
+ turnId: "turn-1",
+ turnSequence: 0,
+ });
+ await replacementRuntime.provider.forceFlush();
+
+ const turn = byName(firstRuntime.exporter.getFinishedSpans(), "agent.turn")[0]!;
+ const replacementSpans = replacementRuntime.exporter.getFinishedSpans();
+ const step = byName(replacementSpans, "agent.step")[0]!;
+ const action = byName(replacementSpans, "agent.action")[0]!;
+
+ expect(step.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);
+ expect(action.parentSpanContext?.spanId).toBe(step.spanContext().spanId);
+ expect(step.spanContext().traceId).toBe(turn.spanContext().traceId);
+ });
+
+ it("marks a failed action without failing its turn", async () => {
+ const runtime = createRuntime();
+ await emitAttempt({
+ hooks: runtime.hooks,
+ runInContext: runtime.runInContext,
+ sessionId: "session-1",
+ toolError: new Error("tool failed"),
+ turnId: "turn-1",
+ turnSequence: 0,
+ });
+ await runtime.provider.forceFlush();
+
+ const spans = runtime.exporter.getFinishedSpans();
+ expect(byName(spans, "agent.action")[0]!.status.code).toBe(SpanStatusCode.ERROR);
+ expect(byName(spans, "agent.turn")[0]!.status.code).toBe(SpanStatusCode.UNSET);
+ });
+});
diff --git a/packages/eve/src/harness/agent-otel-provider.ts b/packages/eve/src/harness/agent-otel-provider.ts
new file mode 100644
index 000000000..7d0a8edd6
--- /dev/null
+++ b/packages/eve/src/harness/agent-otel-provider.ts
@@ -0,0 +1,408 @@
+import {
+ ROOT_CONTEXT,
+ SpanStatusCode,
+ context,
+ type Context,
+ type Span,
+ type SpanContext,
+ type Tracer,
+ trace,
+} from "@opentelemetry/api";
+
+import type {
+ InstrumentationAttemptScope,
+ InstrumentationAttemptStartedEvent,
+ InstrumentationAttemptTerminalEvent,
+ InstrumentationContextRunner,
+ InstrumentationModelCallTerminalEvent,
+ InstrumentationModelCallStartedEvent,
+ InstrumentationProviderDefinition,
+ InstrumentationSessionStartedEvent,
+ InstrumentationSessionTransitionEvent,
+ InstrumentationToolCallStartedEvent,
+ InstrumentationToolCallTerminalEvent,
+ InstrumentationTurnStartedEvent,
+ InstrumentationTurnTerminalEvent,
+} from "#harness/instrumentation-lifecycle.js";
+
+interface SpanState {
+ readonly context: Context;
+ readonly span: Span;
+}
+
+export interface AgentSessionTraceState {
+ readonly agentName?: string;
+ readonly channelKind?: string;
+ readonly context: SpanContext;
+ readonly pendingStarted: boolean;
+ readonly rootSessionId: string;
+}
+
+export interface AgentTurnTraceState {
+ readonly context: SpanContext;
+ readonly rootSessionId: string;
+ readonly sequence: number;
+ readonly terminal?: {
+ readonly error?: unknown;
+ readonly type: InstrumentationTurnTerminalEvent["type"];
+ };
+}
+
+/** Provider-owned serializable storage for durable agent trace state. */
+export interface AgentTraceStateStore {
+ deleteSession(sessionId: string): void | PromiseLike<void>;
+ deleteTurn(sessionId: string, turnId: string): void | PromiseLike<void>;
+ getSession(
+ sessionId: string,
+ ): AgentSessionTraceState | undefined | PromiseLike<AgentSessionTraceState | undefined>;
+ getTurn(
+ sessionId: string,
+ turnId: string,
+ ): AgentTurnTraceState | undefined | PromiseLike<AgentTurnTraceState | undefined>;
+ setSession(sessionId: string, state: AgentSessionTraceState): void | PromiseLike<void>;
+ setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void | PromiseLike<void>;
+}
+
+/** In-memory trace state used by tests and non-durable runtimes. */
+export class InMemoryAgentTraceStateStore implements AgentTraceStateStore {
+ readonly #sessions = new Map<string, AgentSessionTraceState>();
+ readonly #turns = new Map<string, AgentTurnTraceState>();
+
+ deleteSession(sessionId: string): void {
+ this.#sessions.delete(sessionId);
+ }
+
+ deleteTurn(sessionId: string, turnId: string): void {
+ this.#turns.delete(turnKey(sessionId, turnId));
+ }
+
+ getSession(sessionId: string): AgentSessionTraceState | undefined {
+ return this.#sessions.get(sessionId);
+ }
+
+ getTurn(sessionId: string, turnId: string): AgentTurnTraceState | undefined {
+ return this.#turns.get(turnKey(sessionId, turnId));
+ }
+
+ setSession(sessionId: string, state: AgentSessionTraceState): void {
+ this.#sessions.set(sessionId, state);
+ }
+
+ setTurn(sessionId: string, turnId: string, state: AgentTurnTraceState): void {
+ this.#turns.set(turnKey(sessionId, turnId), state);
+ }
+}
+
+export interface AgentOtelInstrumentationInput {
+ readonly frameworkVersion: string;
+ readonly stateStore: AgentTraceStateStore;
+ readonly tracer: Tracer;
+}
+
+/** OTel event definition and its trusted framework context runner. */
+export interface AgentOtelInstrumentation {
+ readonly hook: InstrumentationProviderDefinition;
+ readonly runInContext: InstrumentationContextRunner;
+}
+
+/** Creates OTel instrumentation for eve's structural `agent.*` convention. */
+export function createAgentOtelInstrumentation(
+ input: AgentOtelInstrumentationInput,
+): AgentOtelInstrumentation {
+ const executionContexts = new WeakMap<
+ InstrumentationAttemptScope,
+ { readonly models: Map<string, Context>; readonly tools: Map<string, Context> }
+ >();
+ // A serverless turn runs inside one `turnStep` "use step" invocation. If
+ // that worker is lost, Workflow retries the whole step from entry rather
+ // than resuming this callback sequence in a replacement process.
+ const steps = new WeakMap<InstrumentationAttemptScope, SpanState>();
+
+ const onSessionStarted = async (event: InstrumentationSessionStartedEvent): Promise<void> => {
+ await ensureSessionContext(event);
+ };
+
+ const onTurnStarted = async (event: InstrumentationTurnStartedEvent): Promise<void> => {
+ const session = await ensureSessionContext({
+ agentName: undefined,
+ channelKind: undefined,
+ rootSessionId: event.rootSessionId,
+ sessionId: event.sessionId,
+ type: "session.started",
+ });
+ const span = input.tracer.startSpan(
+ "agent.turn",
+ {
+ attributes: {
+ "agent.framework.name": "eve",
+ "agent.framework.version": input.frameworkVersion,
+ "agent.name": session.agentName,
+ "agent.root.session.id": event.rootSessionId,
+ "agent.session.id": event.sessionId,
+ "agent.turn.id": event.turnId,
+ "agent.turn.sequence": event.sequence,
+ },
+ },
+ contextFromSpanContext(session.context),
+ );
+ span.addEvent("turn.started");
+ if (session.pendingStarted) {
+ span.addEvent("session.started");
+ }
+ await input.stateStore.setSession(event.sessionId, {
+ ...session,
+ pendingStarted: false,
+ });
+ await input.stateStore.setTurn(event.sessionId, event.turnId, {
+ context: span.spanContext(),
+ rootSessionId: event.rootSessionId,
+ sequence: event.sequence,
+ });
+ span.end();
+ };
+
+ const onAttemptStarted = async (event: InstrumentationAttemptStartedEvent): Promise<void> => {
+ const turn = await input.stateStore.getTurn(event.scope.sessionId, event.scope.turnId);
+ if (turn === undefined) return;
+ const turnContext = contextFromSpanContext(turn.context);
+ const span = input.tracer.startSpan(
+ "agent.step",
+ {
+ attributes: {
+ "agent.session.id": event.scope.sessionId,
+ "agent.framework.name": "eve",
+ "agent.framework.version": input.frameworkVersion,
+ "agent.root.session.id": event.scope.rootSessionId ?? event.scope.sessionId,
+ "agent.step.attempt": event.scope.attemptIndex,
+ "agent.step.index": event.scope.stepIndex,
+ "agent.turn.id": event.scope.turnId,
+ "agent.name": event.scope.functionId,
+ },
+ },
+ turnContext,
+ );
+ span.addEvent("step.started");
+ steps.set(event.scope, {
+ context: trace.setSpan(turnContext, span),
+ span,
+ });
+ };
+
+ const onAttemptTerminal = (event: InstrumentationAttemptTerminalEvent): void => {
+ executionContexts.delete(event.scope);
+ const step = steps.get(event.scope);
+ if (step === undefined) return;
+ step.span.addEvent(event.type === "attempt.completed" ? "step.completed" : "step.failed");
+ if (event.type === "attempt.failed") recordError(step.span, event.error);
+ step.span.end();
+ steps.delete(event.scope);
+ };
+
+ const onTurnTerminal = async (event: InstrumentationTurnTerminalEvent): Promise<void> => {
+ const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);
+ if (turn === undefined) return;
+ await input.stateStore.setTurn(event.sessionId, event.turnId, {
+ ...turn,
+ terminal: { error: event.error, type: event.type },
+ });
+ };
+
+ const onSessionTransition = async (
+ event: InstrumentationSessionTransitionEvent,
+ ): Promise<void> => {
+ if (event.turnId !== undefined) {
+ const turn = await input.stateStore.getTurn(event.sessionId, event.turnId);
+ if (turn !== undefined) {
+ const span = input.tracer.startSpan(
+ "agent.turn.terminal",
+ {
+ attributes: {
+ "agent.root.session.id": turn.rootSessionId,
+ "agent.session.id": event.sessionId,
+ "agent.turn.id": event.turnId,
+ "agent.turn.sequence": turn.sequence,
+ },
+ },
+ contextFromSpanContext(turn.context),
+ );
+ if (turn.terminal !== undefined) {
+ span.addEvent(turn.terminal.type);
+ if (turn.terminal.type === "turn.failed") {
+ recordError(span, turn.terminal.error);
+ }
+ }
+ span.addEvent(event.type);
+ span.end();
+ await input.stateStore.deleteTurn(event.sessionId, event.turnId);
+ }
+ }
+ // `session.waiting` is not terminal — the session may resume with a new
+ // turn that still needs its metadata — so only release session-scoped
+ // state on terminal transitions.
+ if (event.type === "session.completed" || event.type === "session.failed") {
+ await input.stateStore.deleteSession(event.sessionId);
+ }
+ };
+
+ const beforeModelCall = (event: InstrumentationModelCallStartedEvent): SpanState | undefined => {
+ const step = steps.get(event.scope);
+ if (step === undefined) return undefined;
+ step.span.setAttribute("agent.model.id", event.source.modelId);
+ step.span.setAttribute("agent.model.provider", event.source.provider);
+ getExecutionContexts(event.scope).models.set(event.id, step.context);
+ return step;
+ };
+
+ const afterModelCall = (event: InstrumentationModelCallTerminalEvent, state: unknown): void => {
+ executionContexts.get(event.scope)?.models.delete(event.id);
+ if (!isSpanState(state)) return;
+ if (event.type === "model.call.failed") {
+ recordError(state.span, event.error);
+ return;
+ }
+ setUsage(state.span, event.source.usage);
+ };
+
+ const beforeToolCall = (event: InstrumentationToolCallStartedEvent): SpanState | undefined => {
+ const step = steps.get(event.scope);
+ if (step === undefined) return undefined;
+ const span = input.tracer.startSpan(
+ "agent.action",
+ {
+ attributes: {
+ "agent.action.call_id": event.source.toolCall.toolCallId,
+ "agent.action.kind": "tool",
+ "agent.action.name": event.source.toolCall.toolName,
+ "agent.framework.name": "eve",
+ "agent.framework.version": input.frameworkVersion,
+ "agent.root.session.id": event.scope.rootSessionId ?? event.scope.sessionId,
+ "agent.session.id": event.scope.sessionId,
+ "agent.step.attempt": event.scope.attemptIndex,
+ "agent.step.index": event.scope.stepIndex,
+ "agent.turn.id": event.scope.turnId,
+ },
+ },
+ step.context,
+ );
+ const state = { context: trace.setSpan(step.context, span), span };
+ getExecutionContexts(event.scope).tools.set(event.id, state.context);
+ return state;
+ };
+
+ const afterToolCall = (event: InstrumentationToolCallTerminalEvent, state: unknown): void => {
+ executionContexts.get(event.scope)?.tools.delete(event.id);
+ if (!isSpanState(state)) return;
+ if (event.type === "tool.call.failed") {
+ recordError(state.span, event.error);
+ } else if (event.source.toolOutput.type !== "tool-result") {
+ recordError(state.span, event.source.toolOutput.error);
+ }
+ state.span.end();
+ };
+
+ const ensureSessionContext = async (
+ event: InstrumentationSessionStartedEvent,
+ ): Promise<AgentSessionTraceState> => {
+ let state = await input.stateStore.getSession(event.sessionId);
+ if (state === undefined) {
+ const marker = input.tracer.startSpan(
+ "agent.session",
+ {
+ attributes: {
+ "agent.channel.kind": event.channelKind,
+ "agent.framework.name": "eve",
+ "agent.framework.version": input.frameworkVersion,
+ "agent.name": event.agentName,
+ "agent.root.session.id": event.rootSessionId,
+ "agent.session.id": event.sessionId,
+ },
+ root: true,
+ },
+ ROOT_CONTEXT,
+ );
+ state = {
+ agentName: event.agentName,
+ channelKind: event.channelKind,
+ context: marker.spanContext(),
+ pendingStarted: true,
+ rootSessionId: event.rootSessionId,
+ };
+ await input.stateStore.setSession(event.sessionId, state);
+ marker.end();
+ }
+ return state;
+ };
+
+ return {
+ hook: {
+ events: {
+ "attempt.completed": onAttemptTerminal,
+ "attempt.failed": onAttemptTerminal,
+ "attempt.started": onAttemptStarted,
+ "model.call": { after: afterModelCall, before: beforeModelCall },
+ "session.completed": onSessionTransition,
+ "session.failed": onSessionTransition,
+ "session.started": onSessionStarted,
+ "session.waiting": onSessionTransition,
+ "tool.call": { after: afterToolCall, before: beforeToolCall },
+ "turn.cancelled": onTurnTerminal,
+ "turn.completed": onTurnTerminal,
+ "turn.failed": onTurnTerminal,
+ "turn.started": onTurnStarted,
+ },
+ },
+ runInContext(operation, execute) {
+ const contexts = executionContexts.get(operation.scope);
+ const parent =
+ operation.type === "model.call"
+ ? contexts?.models.get(operation.id)
+ : contexts?.tools.get(operation.id);
+ return parent === undefined ? execute() : context.with(parent, execute);
+ },
+ };
+
+ function getExecutionContexts(scope: InstrumentationAttemptScope): {
+ readonly models: Map<string, Context>;
+ readonly tools: Map<string, Context>;
+ } {
+ let state = executionContexts.get(scope);
+ if (state === undefined) {
+ state = { models: new Map(), tools: new Map() };
+ executionContexts.set(scope, state);
+ }
+ return state;
+ }
+}
+
+function turnKey(sessionId: string, turnId: string): string {
+ return `${sessionId}:${turnId}`;
+}
+
+function contextFromSpanContext(spanContext: SpanContext): Context {
+ return trace.setSpan(ROOT_CONTEXT, trace.wrapSpanContext(spanContext));
+}
+
+function isSpanState(value: unknown): value is SpanState {
+ return typeof value === "object" && value !== null && "context" in value && "span" in value;
+}
+
+function setUsage(
+ span: Span,
+ usage: { readonly inputTokens?: number; readonly outputTokens?: number },
+): void {
+ if (usage.inputTokens !== undefined) {
+ span.setAttribute("agent.usage.input_tokens", usage.inputTokens);
+ }
+ if (usage.outputTokens !== undefined) {
+ span.setAttribute("agent.usage.output_tokens", usage.outputTokens);
+ }
+}
+
+function recordError(span: Span, error: unknown): void {
+ if (error instanceof Error) {
+ span.recordException(error);
+ span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+ } else {
+ span.setStatus({ code: SpanStatusCode.ERROR });
+ }
+}
diff --git a/packages/eve/src/harness/instrumentation-lifecycle.ts b/packages/eve/src/harness/instrumentation-lifecycle.ts
index e58f1beb2..c0e71992b 100644
--- a/packages/eve/src/harness/instrumentation-lifecycle.ts
+++ b/packages/eve/src/harness/instrumentation-lifecycle.ts
@@ -9,6 +9,7 @@ export interface InstrumentationAttemptScope {
readonly attemptId: string;
readonly attemptIndex: number;
readonly functionId?: string;
+ readonly rootSessionId?: string;
readonly sessionId: string;
readonly stepIndex: number;
readonly turnId: string;
@@ -21,6 +22,42 @@ export interface InstrumentationAttemptStartedEvent {
readonly step: TelemetryEvent<"onStepStart">;
}
+export interface InstrumentationSessionStartedEvent {
+ readonly type: "session.started";
+ readonly agentName?: string;
+ readonly channelKind?: string;
+ readonly rootSessionId: string;
+ readonly sessionId: string;
+}
+
+export interface InstrumentationSessionTransitionEvent {
+ readonly type: "session.completed" | "session.failed" | "session.waiting";
+ readonly error?: unknown;
+ readonly sessionId: string;
+ readonly turnId?: string;
+}
+
+export interface InstrumentationTurnStartedEvent {
+ readonly type: "turn.started";
+ readonly rootSessionId: string;
+ readonly sequence: number;
+ readonly sessionId: string;
+ readonly turnId: string;
+}
+
+export interface InstrumentationTurnTerminalEvent {
+ readonly type: "turn.cancelled" | "turn.completed" | "turn.failed";
+ readonly error?: unknown;
+ readonly sessionId: string;
+ readonly turnId: string;
+}
+
+export interface InstrumentationAttemptTerminalEvent {
+ readonly type: "attempt.completed" | "attempt.failed";
+ readonly error?: unknown;
+ readonly scope: InstrumentationAttemptScope;
+}
+
export interface InstrumentationModelCallStartedEvent {
readonly type: "model.call.started";
readonly id: string;
@@ -86,10 +123,36 @@ export interface InstrumentationProviderDefinition {
readonly "attempt.started"?: (
event: InstrumentationAttemptStartedEvent,
) => void | PromiseLike<void>;
+ readonly "attempt.completed"?: (
+ event: InstrumentationAttemptTerminalEvent,
+ ) => void | PromiseLike<void>;
+ readonly "attempt.failed"?: (
+ event: InstrumentationAttemptTerminalEvent,
+ ) => void | PromiseLike<void>;
+ readonly "session.completed"?: (
+ event: InstrumentationSessionTransitionEvent,
+ ) => void | PromiseLike<void>;
+ readonly "session.failed"?: (
+ event: InstrumentationSessionTransitionEvent,
+ ) => void | PromiseLike<void>;
+ readonly "session.started"?: (
+ event: InstrumentationSessionStartedEvent,
+ ) => void | PromiseLike<void>;
+ readonly "session.waiting"?: (
+ event: InstrumentationSessionTransitionEvent,
+ ) => void | PromiseLike<void>;
readonly "tool.call"?: RelatedLifecycleHook<
InstrumentationToolCallStartedEvent,
InstrumentationToolCallTerminalEvent
>;
+ readonly "turn.cancelled"?: (
+ event: InstrumentationTurnTerminalEvent,
+ ) => void | PromiseLike<void>;
+ readonly "turn.completed"?: (
+ event: InstrumentationTurnTerminalEvent,
+ ) => void | PromiseLike<void>;
+ readonly "turn.failed"?: (event: InstrumentationTurnTerminalEvent) => void | PromiseLike<void>;
+ readonly "turn.started"?: (event: InstrumentationTurnStartedEvent) => void | PromiseLike<void>;
};
}
@@ -106,7 +169,13 @@ export interface InstrumentationRelatedEventMap {
export type InstrumentationRelatedEventName = keyof InstrumentationRelatedEventMap;
-export type InstrumentationPointEvent = InstrumentationAttemptStartedEvent;
+export type InstrumentationPointEvent =
+ | InstrumentationAttemptStartedEvent
+ | InstrumentationAttemptTerminalEvent
+ | InstrumentationSessionStartedEvent
+ | InstrumentationSessionTransitionEvent
+ | InstrumentationTurnStartedEvent
+ | InstrumentationTurnTerminalEvent;
/** Trusted framework operation for activating context around AI SDK execution. */
export type InstrumentationContextRunner = <T>(
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 0205a198e..53cd8500a 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1173,6 +1173,9 @@ importers:
'@nuxt/kit':
specifier: ^4.0.0
version: 4.4.6([email protected])
+ '@opentelemetry/sdk-trace-base':
+ specifier: ^2.6.1
+ version: 2.6.1(@opentelemetry/[email protected])
'@standard-schema/spec':
specifier: 1.1.0
version: 1.1.0