The Open Framework for Building Agents
feat(eve): publish canonical harness lifecycle events (#1197)
Chad Hietalae68667bfd497
diff --git a/packages/eve/src/harness/instrumentation-native-events.test.ts b/packages/eve/src/harness/instrumentation-native-events.test.tsnew file mode 100644index 000000000..96ae284c8--- /dev/null+++ b/packages/eve/src/harness/instrumentation-native-events.test.ts@@ -0,0 +1,73 @@+import { describe, expect, it } from "vitest";++import {+ createSessionStartedEvent,+ createSessionWaitingEvent,+ createStepCompletedEvent,+ createStepStartedEvent,+ createTurnCompletedEvent,+ createTurnStartedEvent,+} from "#protocol/message.js";+import { createInstrumentationHandleEvent } from "#harness/instrumentation-native-events.js";+import type { InstrumentationHooks } from "#harness/instrumentation-lifecycle.js";++describe("createInstrumentationHandleEvent", () => {+ it("publishes native lifecycle transitions after durable handling", async () => {+ const order: string[] = [];+ const hooks: InstrumentationHooks = {+ after: async () => {},+ before: async () => {},+ publish: async (event) => {+ order.push(`lifecycle:${event.type}`);+ },+ };+ const handleEvent = createInstrumentationHandleEvent({+ agentName: "weather",+ handleEvent: async (event) => {+ order.push(`durable:${event.type}`);+ },+ hooks,+ sessionId: "session-1",+ })!;++ await handleEvent(createSessionStartedEvent());+ await handleEvent(createTurnStartedEvent({ sequence: 0, turnId: "turn-1" }));+ await handleEvent(createStepStartedEvent({ sequence: 0, stepIndex: 0, turnId: "turn-1" }));+ await handleEvent(+ createStepCompletedEvent({+ finishReason: "stop",+ sequence: 0,+ stepIndex: 0,+ turnId: "turn-1",+ }),+ );+ await handleEvent(createTurnCompletedEvent({ sequence: 0, turnId: "turn-1" }));+ await handleEvent(createSessionWaitingEvent("continue-1"));++ expect(order).toEqual([+ "durable:session.started",+ "lifecycle:session.started",+ "durable:turn.started",+ "lifecycle:turn.started",+ "durable:step.started",+ "durable:step.completed",+ "durable:turn.completed",+ "lifecycle:turn.completed",+ "durable:session.waiting",+ "lifecycle:session.waiting",+ ]);+ });++ it("does not change execution mode when hooks have no durable handler", () => {+ expect(+ createInstrumentationHandleEvent({+ hooks: {+ after: async () => {},+ before: async () => {},+ publish: async () => {},+ },+ sessionId: "session-1",+ }),+ ).toBeUndefined();+ });+});diff --git a/packages/eve/src/harness/instrumentation-native-events.ts b/packages/eve/src/harness/instrumentation-native-events.tsnew file mode 100644index 000000000..a720c657d--- /dev/null+++ b/packages/eve/src/harness/instrumentation-native-events.ts@@ -0,0 +1,78 @@+import type { HandleMessageStreamEvent } from "#protocol/message.js";+import type {+ InstrumentationHooks,+ InstrumentationPointEvent,+} from "#harness/instrumentation-lifecycle.js";+import type { HandleEventFn } from "#harness/types.js";++export interface CreateInstrumentationHandleEventInput {+ readonly agentName?: string;+ readonly handleEvent?: HandleEventFn;+ readonly hooks?: InstrumentationHooks;+ readonly rootSessionId?: string;+ readonly sessionId: string;+}++/** Publishes eve-native lifecycle transitions after durable event acceptance. */+export function createInstrumentationHandleEvent(+ input: CreateInstrumentationHandleEventInput,+): HandleEventFn | undefined {+ if (input.hooks === undefined) return input.handleEvent;+ if (input.handleEvent === undefined) return undefined;++ const handleEvent = input.handleEvent;+ const hooks = input.hooks;+ let activeTurnId: string | undefined;+ return async (event, messages) => {+ await handleEvent(event, messages);+ const lifecycleEvent = toLifecycleEvent(event, input, activeTurnId);+ if (event.type === "turn.started") activeTurnId = event.data.turnId;+ if (lifecycleEvent !== undefined) await hooks.publish(lifecycleEvent);+ };+}++function toLifecycleEvent(+ event: HandleMessageStreamEvent,+ input: CreateInstrumentationHandleEventInput,+ activeTurnId: string | undefined,+): InstrumentationPointEvent | undefined {+ switch (event.type) {+ case "session.started":+ return {+ agentName: input.agentName,+ rootSessionId: input.rootSessionId ?? input.sessionId,+ sessionId: input.sessionId,+ type: "session.started",+ };+ case "session.completed":+ case "session.waiting":+ return { sessionId: input.sessionId, turnId: activeTurnId, type: event.type };+ case "session.failed":+ return {+ error: new Error(event.data.message),+ sessionId: input.sessionId,+ turnId: activeTurnId,+ type: "session.failed",+ };+ case "turn.started":+ return {+ rootSessionId: input.rootSessionId ?? input.sessionId,+ sequence: event.data.sequence,+ sessionId: input.sessionId,+ turnId: event.data.turnId,+ type: "turn.started",+ };+ case "turn.completed":+ case "turn.cancelled":+ return { sessionId: input.sessionId, turnId: event.data.turnId, type: event.type };+ case "turn.failed":+ return {+ error: new Error(event.data.message),+ sessionId: input.sessionId,+ turnId: event.data.turnId,+ type: "turn.failed",+ };+ default:+ return undefined;+ }+}diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex afaa058ea..b25ee2d52 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -8849,7 +8849,10 @@ describe("createToolLoopHarness", () => {toolCalls: [],toolResults: [],});- const hooks = createInstrumentationHooks([]);+ const attemptCompleted = vi.fn();+ const hooks = createInstrumentationHooks([+ { events: { "attempt.completed": attemptCompleted } },+ ]);const config = createTestConfig("conversation", undefined, {instrumentationHooks: hooks,});@@ -8880,6 +8883,12 @@ describe("createToolLoopHarness", () => {recordInputs: true,recordOutputs: true,});+ expect(attemptCompleted).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({+ scope: expect.objectContaining({ attemptIndex: 0 }),+ type: "attempt.completed",+ }),+ );});it("composes lifecycle hooks with existing authored OTel", async () => {diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.tsindex 370ee1296..45a95f741 100644--- a/packages/eve/src/harness/tool-loop.ts+++ b/packages/eve/src/harness/tool-loop.ts@@ -105,6 +105,8 @@ import { createToolResultMessagePartFromToolError } from "#harness/action-resultimport { activeTurnId } from "#harness/active-turn-id.js";import { buildTelemetryRuntimeContext } from "#harness/instrumentation-runtime-context.js";import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";+import { createInstrumentationHandleEvent } from "#harness/instrumentation-native-events.js";+import type { InstrumentationAttemptScope } from "#harness/instrumentation-lifecycle.js";import {consumeDeferredStepInput,getApprovedTools,@@ -475,7 +477,7 @@ function resolveStepOtelContext(}export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {- const emit = config.handleEvent;+ const baseEmit = config.handleEvent;const telemetryConfig = getInstrumentationConfig();if (telemetryConfig !== undefined) {ensureOtelIntegration();@@ -534,6 +536,14 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {}let emissionState = getHarnessEmissionState(session.state);+ const parent = contextStorage.getStore()?.get(ParentSessionKey);+ const emit = createInstrumentationHandleEvent({+ agentName: config.runtimeIdentity?.agentName,+ handleEvent: baseEmit,+ hooks: config.instrumentationHooks,+ rootSessionId: parent?.rootSessionId,+ sessionId: session.sessionId,+ });// Resolve deferred input, runtime actions, then HITL input; each stage// may park when its resume payload has not arrived.@@ -889,20 +899,22 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {const instrumentationHooks = config.instrumentationHooks;const instrumentationTurnId = activeTurnId(emissionState);- const bridgeIntegration =+ const attemptScope: InstrumentationAttemptScope | undefined =instrumentationHooks === undefined? undefined- : createAiSdkHookBridge(- {- attemptId: `${session.sessionId}:${instrumentationTurnId}:${emissionState.stepIndex}:${opts.attemptIndex}`,- attemptIndex: opts.attemptIndex,- functionId: telemetryConfig?.functionId ?? agentName,- sessionId: session.sessionId,- stepIndex: emissionState.stepIndex,- turnId: instrumentationTurnId,- },- instrumentationHooks,- );+ : {+ attemptId: `${session.sessionId}:${instrumentationTurnId}:${emissionState.stepIndex}:${opts.attemptIndex}`,+ attemptIndex: opts.attemptIndex,+ functionId: telemetryConfig?.functionId ?? agentName,+ rootSessionId: parent?.rootSessionId ?? session.sessionId,+ sessionId: session.sessionId,+ stepIndex: emissionState.stepIndex,+ turnId: instrumentationTurnId,+ };+ const bridgeIntegration =+ attemptScope === undefined || instrumentationHooks === undefined+ ? undefined+ : createAiSdkHookBridge(attemptScope, instrumentationHooks);const hooks = buildStepHooks({cachePath,@@ -1027,7 +1039,22 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {});};- return executeModelCall().catch(rethrowNoOutputAsEmptyResponse);+ try {+ const result = await executeModelCall();+ if (attemptScope !== undefined) {+ await instrumentationHooks?.publish({ scope: attemptScope, type: "attempt.completed" });+ }+ return result;+ } catch (error) {+ if (attemptScope !== undefined) {+ await instrumentationHooks?.publish({+ error,+ scope: attemptScope,+ type: "attempt.failed",+ });+ }+ return rethrowNoOutputAsEmptyResponse(error);+ }};let nextModelAttemptIndex = 0;