The Open Framework for Building Agents
feat(eve): add replay-stable instrumentation event identity (#1859)
Chad Hietalaf06633b20201
diff --git a/.changeset/instrumentation-event-identity.md b/.changeset/instrumentation-event-identity.mdnew file mode 100644index 000000000..b660b00cd--- /dev/null+++ b/.changeset/instrumentation-event-identity.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Give every instrumentation lifecycle event a replay-stable `idempotencyKey` derived from durable eve identity, allowing providers to upsert one record across retries and worker replays.diff --git a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts b/packages/eve/src/harness/ai-sdk-hook-bridge.test.tsindex 7b6017360..fa647500f 100644--- a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts+++ b/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts@@ -29,11 +29,13 @@ describe("createAiSdkHookBridge", () => {return {events: {"model.call.started"(event) {- calls.push(`${name}:started:${event.id}`);- states.set(event.id, `${name}-state`);+ calls.push(`${name}:started:${event.idempotencyKey}`);+ states.set(event.idempotencyKey, `${name}-state`);},"model.call.completed"(event) {- calls.push(`${name}:completed:${event.id}:${String(states.get(event.id))}`);+ calls.push(+ `${name}:completed:${event.idempotencyKey}:${String(states.get(event.idempotencyKey))}`,+ );},},};@@ -55,7 +57,7 @@ describe("createAiSdkHookBridge", () => {},]);- const id = `${scope.attemptId}:model:call-1:0`;+ const id = `model:${scope.attemptId}:0`;expect(calls).toEqual([`a:started:${id}`,`b:started:${id}`,@@ -91,10 +93,10 @@ describe("createAiSdkHookBridge", () => {it("passes the identity captured at model-call start to the context runner", async () => {const ids: string[] = [];const hooks = createInstrumentationHooks([- { events: { "model.call.started": (event) => void ids.push(event.id) } },+ { events: { "model.call.started": (event) => void ids.push(event.idempotencyKey) } },]);const bridge = createAiSdkHookBridge(scope, hooks, (operation, execute) => {- ids.push(operation.id);+ ids.push(operation.idempotencyKey);return execute();});Reflect.apply(bridge.onStart!, bridge, [@@ -108,7 +110,7 @@ describe("createAiSdkHookBridge", () => {await bridge.executeLanguageModelCall!({ callId: "call-1", execute: async () => "result" });- const expected = `${scope.attemptId}:model:call-1:0`;+ const expected = `model:${scope.attemptId}:0`;expect(ids).toEqual([expected, expected]);});@@ -126,6 +128,23 @@ describe("createAiSdkHookBridge", () => {expect(adapterCalls).toBe(0);});+ it("derives replay-stable model identity without the AI SDK call ID", async () => {+ const keys: string[] = [];+ const hooks = createInstrumentationHooks([+ { events: { "model.call.started": (event) => void keys.push(event.idempotencyKey) } },+ ]);++ for (const callId of ["sdk-random-1", "sdk-random-2"]) {+ const bridge = createAiSdkHookBridge(scope, hooks);+ await Reflect.apply(bridge.onStepStart!, bridge, [{ callId, stepNumber: 2 }]);+ await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [+ { callId, messages: [], modelId: "model", provider: "test", tools: undefined },+ ]);+ }++ expect(keys).toEqual([`model:${scope.attemptId}:2`, `model:${scope.attemptId}:2`]);+ });+it("publishes step provider metadata as step.metadata, skipping steps without any", async () => {const events: InstrumentationStepAttemptMetadataEvent[] = [];const hooks = createInstrumentationHooks([@@ -146,6 +165,7 @@ describe("createAiSdkHookBridge", () => {expect(events).toEqual([{+ idempotencyKey: `step:${scope.attemptId}`,providerMetadata: { gateway: { cost: "0.000082" } },scope,type: "step.attempt.metadata",@@ -240,6 +260,7 @@ describe("createAiSdkHookBridge", () => {await Reflect.apply(bridge.onStepStart!, bridge, [{ callId: "call-1", stepNumber: 0 }]);const expected = {+ idempotencyKey: `step:${scope.attemptId}`,operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },scope,type: "step.attempt.started",@@ -313,7 +334,7 @@ describe("createAiSdkHookBridge", () => {]);expect(before).toHaveBeenCalledExactlyOnceWith({- id: `${scope.attemptId}:model:call-1:0`,+ idempotencyKey: `model:${scope.attemptId}:0`,input: { instructions: "be brief", messages: [{ content: "hi", role: "user" }] },model: { modelId: "model", provider: "test" },scope,@@ -342,7 +363,7 @@ describe("createAiSdkHookBridge", () => {},],finishReason: "tool-calls",- id: `${scope.attemptId}:model:call-1:0`,+ idempotencyKey: `model:${scope.attemptId}:0`,scope,type: "model.call.completed",usage: {@@ -386,14 +407,14 @@ describe("createAiSdkHookBridge", () => {expect(before).toHaveBeenCalledExactlyOnceWith({callId: "tool-1",- id: `${scope.attemptId}:tool:tool-1:0`,+ idempotencyKey: `tool:${scope.attemptId}:tool-1:0`,input: { q: "eve" },scope,toolName: "search",type: "tool.call.started",});expect(after).toHaveBeenCalledExactlyOnceWith({- id: `${scope.attemptId}:tool:tool-1:0`,+ idempotencyKey: `tool:${scope.attemptId}:tool-1:0`,output: expected,scope,type: "tool.call.completed",@@ -408,9 +429,9 @@ describe("createAiSdkHookBridge", () => {return {events: {"model.call.completed": (event) => {- observed.set(name, own.get(event.id));+ observed.set(name, own.get(event.idempotencyKey));},- "model.call.started": (event) => void own.set(event.id, `${name}-state`),+ "model.call.started": (event) => void own.set(event.idempotencyKey, `${name}-state`),},};};@@ -469,14 +490,14 @@ describe("createAiSdkHookBridge", () => {events: {async "tool.call.started"(event) {started.set(- event.id,+ event.idempotencyKey,await new Promise<string>((resolve) => {- resolvers.set(event.id, () => resolve(`state:${event.id}`));+ resolvers.set(event.idempotencyKey, () => resolve(`state:${event.idempotencyKey}`));}),);},"tool.call.completed"(event) {- terminalStates.set(event.id, started.get(event.id));+ terminalStates.set(event.idempotencyKey, started.get(event.idempotencyKey));},},},@@ -493,8 +514,8 @@ describe("createAiSdkHookBridge", () => {const second = start("tool-2");await vi.waitFor(() => expect(resolvers.size).toBe(2));- const firstId = `${scope.attemptId}:tool:tool-1:0`;- const secondId = `${scope.attemptId}:tool:tool-2:0`;+ const firstId = `tool:${scope.attemptId}:tool-1:0`;+ const secondId = `tool:${scope.attemptId}:tool-2:0`;resolvers.get(secondId)!();resolvers.get(firstId)!();await Promise.all([first, second]);diff --git a/packages/eve/src/harness/ai-sdk-hook-bridge.ts b/packages/eve/src/harness/ai-sdk-hook-bridge.tsindex d0239cf18..0807e7ea1 100644--- a/packages/eve/src/harness/ai-sdk-hook-bridge.ts+++ b/packages/eve/src/harness/ai-sdk-hook-bridge.ts@@ -14,13 +14,18 @@ import type {InstrumentationToolOutput,InstrumentationUsage,} from "#harness/instrumentation-lifecycle.js";+import {+ attemptIdempotencyKey,+ modelCallIdempotencyKey,+ toolCallIdempotencyKey,+} from "#harness/instrumentation-lifecycle.js";type TelemetryEvent<TKey extends keyof Telemetry> = Parameters<NonNullable<Telemetry[TKey]>>[0];interface AttemptState {- readonly modelIds: Map<string, string>;+ readonly modelKeys: Map<string, string>;readonly scope: InstrumentationAttemptScope;- readonly toolIds: Map<string, string>;+ readonly toolKeys: Map<string, string>;operation?: InstrumentationOperationRef;// Only the number is kept: it disambiguates call identities within an attempt.stepNumber?: number;@@ -33,9 +38,9 @@ export function createAiSdkHookBridge(runInContext: InstrumentationContextRunner = directRunInContext,): Telemetry {const state: AttemptState = {- modelIds: new Map(),+ modelKeys: new Map(),scope,- toolIds: new Map(),+ toolKeys: new Map(),};return {@@ -52,22 +57,22 @@ export function createAiSdkHookBridge(if (started !== undefined) await hooks.publish(started);},async onLanguageModelCallStart(event) {- const id = createModelCallIdentity(state, event.callId);- state.modelIds.set(event.callId, id);- const started = toModelCallStarted(state, id, event);+ const key = modelCallIdempotencyKey(state.scope, state.stepNumber ?? 0);+ state.modelKeys.set(event.callId, key);+ const started = toModelCallStarted(state, key, event);await hooks.publish(started);},executeLanguageModelCall({ callId, execute }) {- const id = state.modelIds.get(callId);- return id === undefined+ const key = state.modelKeys.get(callId);+ return key === undefined? execute()- : runInContext({ id, scope, type: "model.call" }, execute);+ : runInContext({ idempotencyKey: key, scope, type: "model.call" }, execute);},async onLanguageModelCallEnd(event) {- const id = state.modelIds.get(event.callId);- if (id === undefined) return;- state.modelIds.delete(event.callId);- const completed = toModelCallCompleted(state, id, event);+ const key = state.modelKeys.get(event.callId);+ if (key === undefined) return;+ state.modelKeys.delete(event.callId);+ const completed = toModelCallCompleted(state, key, event);await hooks.publish(completed);},async onStepEnd(event) {@@ -77,6 +82,7 @@ export function createAiSdkHookBridge(if (event.providerMetadata === undefined) return;await hooks.publish(Object.freeze({+ idempotencyKey: attemptIdempotencyKey(state.scope),providerMetadata: event.providerMetadata,scope: state.scope,type: "step.attempt.metadata",@@ -84,21 +90,27 @@ export function createAiSdkHookBridge();},async onToolExecutionStart(event) {- const id = createToolCallIdentity(state, event.toolCall.toolCallId);- state.toolIds.set(event.toolCall.toolCallId, id);- const started = toToolCallStarted(state, id, event);+ const key = toolCallIdempotencyKey(+ state.scope,+ event.toolCall.toolCallId,+ state.stepNumber ?? 0,+ );+ state.toolKeys.set(event.toolCall.toolCallId, key);+ const started = toToolCallStarted(state, key, event);await hooks.publish(started);},executeTool({ toolCallId, execute }) {- const id = state.toolIds.get(toolCallId);- return id === undefined ? execute() : runInContext({ id, scope, type: "tool.call" }, execute);+ const key = state.toolKeys.get(toolCallId);+ return key === undefined+ ? execute()+ : runInContext({ idempotencyKey: key, scope, type: "tool.call" }, execute);},async onToolExecutionEnd(event) {const toolCallId = event.toolCall.toolCallId;- const id = state.toolIds.get(toolCallId);- if (id === undefined) return;- state.toolIds.delete(toolCallId);- const completed = toToolCallCompleted(state, id, event);+ const key = state.toolKeys.get(toolCallId);+ if (key === undefined) return;+ state.toolKeys.delete(toolCallId);+ const completed = toToolCallCompleted(state, key, event);await hooks.publish(completed);},async onAbort(event) {@@ -111,14 +123,18 @@ export function createAiSdkHookBridge(async function failOpenOperations(error: unknown): Promise<void> {const pending: Promise<void>[] = [];- for (const id of state.modelIds.values()) {- pending.push(hooks.publish(Object.freeze({ error, id, scope, type: "model.call.failed" })));+ for (const idempotencyKey of state.modelKeys.values()) {+ pending.push(+ hooks.publish(Object.freeze({ error, idempotencyKey, scope, type: "model.call.failed" })),+ );}- for (const id of state.toolIds.values()) {- pending.push(hooks.publish(Object.freeze({ error, id, scope, type: "tool.call.failed" })));+ for (const idempotencyKey of state.toolKeys.values()) {+ pending.push(+ hooks.publish(Object.freeze({ error, idempotencyKey, scope, type: "tool.call.failed" })),+ );}- state.modelIds.clear();- state.toolIds.clear();+ state.modelKeys.clear();+ state.toolKeys.clear();await Promise.all(pending);}}@@ -130,23 +146,20 @@ function toStepAttemptStarted(): InstrumentationStepAttemptStartedEvent | undefined {if (state.operation === undefined || state.stepNumber === undefined) return undefined;return Object.freeze({+ idempotencyKey: attemptIdempotencyKey(state.scope),operation: state.operation,scope: state.scope,type: "step.attempt.started",});}-function createModelCallIdentity(state: AttemptState, callId: string): string {- return `${state.scope.attemptId}:model:${callId}:${state.stepNumber ?? 0}`;-}-function toModelCallStarted(state: AttemptState,- id: string,+ idempotencyKey: string,source: TelemetryEvent<"onLanguageModelCallStart">,): InstrumentationModelCallStartedEvent {return Object.freeze({- id,+ idempotencyKey,input: Object.freeze({instructions: source.instructions,messages: Object.freeze([...source.messages]),@@ -159,13 +172,13 @@ function toModelCallStarted(function toModelCallCompleted(state: AttemptState,- id: string,+ idempotencyKey: string,source: TelemetryEvent<"onLanguageModelCallEnd">,): InstrumentationModelCallCompletedEvent {return Object.freeze({content: toContentParts(source.content),finishReason: source.finishReason,- id,+ idempotencyKey,scope: state.scope,type: "model.call.completed",usage: toUsage(source.usage),@@ -233,18 +246,14 @@ function toContentParts(return Object.freeze(parts);}-function createToolCallIdentity(state: AttemptState, toolCallId: string): string {- return `${state.scope.attemptId}:tool:${toolCallId}:${state.stepNumber ?? 0}`;-}-function toToolCallStarted(state: AttemptState,- id: string,+ idempotencyKey: string,source: TelemetryEvent<"onToolExecutionStart">,): InstrumentationToolCallStartedEvent {return Object.freeze({callId: source.toolCall.toolCallId,- id,+ idempotencyKey,input: source.toolCall.input,scope: state.scope,toolName: source.toolCall.toolName,@@ -254,11 +263,11 @@ function toToolCallStarted(function toToolCallCompleted(state: AttemptState,- id: string,+ idempotencyKey: string,source: TelemetryEvent<"onToolExecutionEnd">,): InstrumentationToolCallCompletedEvent {return Object.freeze({- id,+ idempotencyKey,output: toToolOutput(source.toolOutput),scope: state.scope,type: "tool.call.completed",diff --git a/packages/eve/src/harness/instrumentation-lifecycle.test.ts b/packages/eve/src/harness/instrumentation-lifecycle.test.tsnew file mode 100644index 000000000..59af53480--- /dev/null+++ b/packages/eve/src/harness/instrumentation-lifecycle.test.ts@@ -0,0 +1,50 @@+import { describe, expect, it } from "vitest";++import {+ attemptIdempotencyKey,+ inputIdempotencyKey,+ modelCallIdempotencyKey,+ sessionIdempotencyKey,+ toolCallIdempotencyKey,+ turnIdempotencyKey,+ type InstrumentationAttemptScope,+} from "#harness/instrumentation-lifecycle.js";++const scope: InstrumentationAttemptScope = {+ attemptId: "session-1:turn-1:0:0",+ attemptIndex: 0,+ sessionId: "session-1",+ stepIndex: 0,+ turnId: "turn-1",+};++describe("instrumentation idempotency keys", () => {+ it("separates classes that share an identifier", () => {+ expect(sessionIdempotencyKey("shared")).not.toBe(turnIdempotencyKey("shared", "shared"));+ });++ it("separates identical turn IDs in different sessions", () => {+ expect(turnIdempotencyKey("session-1", "turn_0")).not.toBe(+ turnIdempotencyKey("session-2", "turn_0"),+ );+ });++ it("scopes input requests to their originating turn", () => {+ expect(inputIdempotencyKey("session-1", "turn-1", "request-1")).toBe(+ "input:session-1:turn-1:request-1",+ );+ expect(inputIdempotencyKey("session-1", "turn-1", "request-1")).not.toBe(+ inputIdempotencyKey("session-1", "turn-2", "request-1"),+ );+ });++ it("derives model identity without an AI SDK call ID", () => {+ expect(modelCallIdempotencyKey(scope, 2)).toBe("model:session-1:turn-1:0:0:2");+ });++ it("separates model attempts and SDK tool calls", () => {+ const retry = { ...scope, attemptId: "session-1:turn-1:0:1", attemptIndex: 1 };+ expect(attemptIdempotencyKey(scope)).not.toBe(attemptIdempotencyKey(retry));+ expect(modelCallIdempotencyKey(scope, 0)).not.toBe(toolCallIdempotencyKey(scope, "call-1", 0));+ });+});diff --git a/packages/eve/src/harness/instrumentation-lifecycle.ts b/packages/eve/src/harness/instrumentation-lifecycle.tsindex 8ec601e80..5f78e03e2 100644--- a/packages/eve/src/harness/instrumentation-lifecycle.ts+++ b/packages/eve/src/harness/instrumentation-lifecycle.ts@@ -81,6 +81,39 @@ export type InstrumentationToolOutput =| { readonly type: "result"; readonly output: unknown }| { readonly type: "error"; readonly error: unknown };+/** Replay-stable row identity for every lifecycle operation. */+export function sessionIdempotencyKey(sessionId: string): string {+ return `session:${sessionId}`;+}++export function turnIdempotencyKey(sessionId: string, turnId: string): string {+ return `turn:${sessionId}:${turnId}`;+}++export function attemptIdempotencyKey(scope: InstrumentationAttemptScope): string {+ return `step:${scope.attemptId}`;+}++/** One model call occurs per AI SDK step within an eve attempt. */+export function modelCallIdempotencyKey(+ scope: InstrumentationAttemptScope,+ stepNumber: number,+): string {+ return `model:${scope.attemptId}:${String(stepNumber)}`;+}++export function toolCallIdempotencyKey(+ scope: InstrumentationAttemptScope,+ callId: string,+ stepNumber: number,+): string {+ return `tool:${scope.attemptId}:${callId}:${String(stepNumber)}`;+}++export function inputIdempotencyKey(sessionId: string, turnId: string, requestId: string): string {+ return `input:${sessionId}:${turnId}:${requestId}`;+}+/** Framework-owned reason an agent suspended for user input. */export type InstrumentationInputKind = "question" | "session-limit" | "tool-approval";@@ -120,7 +153,7 @@ export interface InstrumentationInputRequestedEvent {readonly callId: string;readonly name: string;};- readonly id: string;+ readonly idempotencyKey: string;readonly kind: InstrumentationInputKind;readonly request: InstrumentationInputRequest;readonly requestId: string;@@ -130,7 +163,7 @@ export interface InstrumentationInputRequestedEvent {export interface InstrumentationInputResolvedEvent {readonly type: "input.resolved";readonly error?: unknown;- readonly id: string;+ readonly idempotencyKey: string;readonly kind: InstrumentationInputKind;readonly outcome: InstrumentationInputOutcome;readonly requestId: string;@@ -140,6 +173,7 @@ export interface InstrumentationInputResolvedEvent {export interface InstrumentationStepAttemptStartedEvent {readonly type: "step.attempt.started";+ readonly idempotencyKey: string;readonly operation: InstrumentationOperationRef;readonly scope: InstrumentationAttemptScope;}@@ -148,6 +182,7 @@ export interface InstrumentationSessionStartedEvent {readonly type: "session.started";readonly agentName?: string;readonly channelKind?: string;+ readonly idempotencyKey: string;readonly parentTraceContext?: InstrumentationTraceContext;readonly rootSessionId: string;readonly sessionId: string;@@ -179,6 +214,7 @@ export interface InstrumentationParentLineage {*/export interface InstrumentationSessionSettledEvent {readonly type: "session.completed" | "session.waiting";+ readonly idempotencyKey: string;readonly sessionId: string;readonly turnId?: string;}@@ -186,6 +222,7 @@ export interface InstrumentationSessionSettledEvent {export interface InstrumentationSessionFailedEvent {readonly type: "session.failed";readonly error: unknown;+ readonly idempotencyKey: string;readonly sessionId: string;readonly turnId?: string;}@@ -196,6 +233,7 @@ export type InstrumentationSessionTransitionEvent =export interface InstrumentationTurnStartedEvent {readonly type: "turn.started";+ readonly idempotencyKey: string;readonly parentLineage?: InstrumentationParentLineage;readonly parentTraceContext?: InstrumentationTraceContext;readonly rootSessionId: string;@@ -204,19 +242,42 @@ export interface InstrumentationTurnStartedEvent {readonly turnId: string;}-export interface InstrumentationTurnTerminalEvent {- readonly type: "turn.cancelled" | "turn.completed" | "turn.failed";- readonly error?: unknown;+export interface InstrumentationTurnSettledEvent {+ readonly type: "turn.cancelled" | "turn.completed";+ readonly idempotencyKey: string;readonly sessionId: string;readonly turnId: string;}-export interface InstrumentationStepAttemptTerminalEvent {- readonly type: "step.attempt.completed" | "step.attempt.failed";- readonly error?: unknown;+export interface InstrumentationTurnFailedEvent {+ readonly type: "turn.failed";+ readonly error: unknown;+ readonly idempotencyKey: string;+ readonly sessionId: string;+ readonly turnId: string;+}++export type InstrumentationTurnTerminalEvent =+ | InstrumentationTurnSettledEvent+ | InstrumentationTurnFailedEvent;++export interface InstrumentationStepAttemptCompletedEvent {+ readonly type: "step.attempt.completed";+ readonly idempotencyKey: string;+ readonly scope: InstrumentationAttemptScope;+}++export interface InstrumentationStepAttemptFailedEvent {+ readonly type: "step.attempt.failed";+ readonly error: unknown;+ readonly idempotencyKey: string;readonly scope: InstrumentationAttemptScope;}+export type InstrumentationStepAttemptTerminalEvent =+ | InstrumentationStepAttemptCompletedEvent+ | InstrumentationStepAttemptFailedEvent;+/*** Provider metadata for one completed attempt, as reported by the AI SDK* (`StepResult.providerMetadata`). Carries Vercel AI Gateway cost data when@@ -224,13 +285,14 @@ export interface InstrumentationStepAttemptTerminalEvent {*/export interface InstrumentationStepAttemptMetadataEvent {readonly type: "step.attempt.metadata";+ readonly idempotencyKey: string;readonly scope: InstrumentationAttemptScope;readonly providerMetadata: Readonly<Record<string, unknown>>;}export interface InstrumentationModelCallStartedEvent {readonly type: "model.call.started";- readonly id: string;+ readonly idempotencyKey: string;readonly input: InstrumentationModelInput;readonly model: InstrumentationModelRef;readonly scope: InstrumentationAttemptScope;@@ -240,7 +302,7 @@ export interface InstrumentationModelCallCompletedEvent {readonly type: "model.call.completed";readonly content: readonly InstrumentationContentPart[];readonly finishReason: string;- readonly id: string;+ readonly idempotencyKey: string;readonly scope: InstrumentationAttemptScope;readonly usage: InstrumentationUsage;}@@ -248,7 +310,7 @@ export interface InstrumentationModelCallCompletedEvent {export interface InstrumentationModelCallFailedEvent {readonly type: "model.call.failed";readonly error: unknown;- readonly id: string;+ readonly idempotencyKey: string;readonly scope: InstrumentationAttemptScope;}@@ -259,7 +321,7 @@ export type InstrumentationModelCallTerminalEvent =export interface InstrumentationToolCallStartedEvent {readonly type: "tool.call.started";readonly callId: string;- readonly id: string;+ readonly idempotencyKey: string;readonly input: unknown;readonly scope: InstrumentationAttemptScope;readonly toolName: string;@@ -267,7 +329,7 @@ export interface InstrumentationToolCallStartedEvent {export interface InstrumentationToolCallCompletedEvent {readonly type: "tool.call.completed";- readonly id: string;+ readonly idempotencyKey: string;readonly output: InstrumentationToolOutput;readonly scope: InstrumentationAttemptScope;}@@ -275,7 +337,7 @@ export interface InstrumentationToolCallCompletedEvent {export interface InstrumentationToolCallFailedEvent {readonly type: "tool.call.failed";readonly error: unknown;- readonly id: string;+ readonly idempotencyKey: string;readonly scope: InstrumentationAttemptScope;}@@ -295,8 +357,8 @@ export interface InstrumentationProviderDefinition {readonly name?: string;readonly events?: {readonly "step.attempt.started"?: InstrumentationEventHandler<InstrumentationStepAttemptStartedEvent>;- readonly "step.attempt.completed"?: InstrumentationEventHandler<InstrumentationStepAttemptTerminalEvent>;- readonly "step.attempt.failed"?: InstrumentationEventHandler<InstrumentationStepAttemptTerminalEvent>;+ readonly "step.attempt.completed"?: InstrumentationEventHandler<InstrumentationStepAttemptCompletedEvent>;+ readonly "step.attempt.failed"?: InstrumentationEventHandler<InstrumentationStepAttemptFailedEvent>;readonly "step.attempt.metadata"?: InstrumentationEventHandler<InstrumentationStepAttemptMetadataEvent>;readonly "model.call.started"?: InstrumentationEventHandler<InstrumentationModelCallStartedEvent>;readonly "model.call.completed"?: InstrumentationEventHandler<InstrumentationModelCallCompletedEvent>;@@ -310,16 +372,16 @@ export interface InstrumentationProviderDefinition {readonly "tool.call.started"?: InstrumentationEventHandler<InstrumentationToolCallStartedEvent>;readonly "tool.call.completed"?: InstrumentationEventHandler<InstrumentationToolCallCompletedEvent>;readonly "tool.call.failed"?: InstrumentationEventHandler<InstrumentationToolCallFailedEvent>;- readonly "turn.cancelled"?: InstrumentationEventHandler<InstrumentationTurnTerminalEvent>;- readonly "turn.completed"?: InstrumentationEventHandler<InstrumentationTurnTerminalEvent>;- readonly "turn.failed"?: InstrumentationEventHandler<InstrumentationTurnTerminalEvent>;+ readonly "turn.cancelled"?: InstrumentationEventHandler<InstrumentationTurnSettledEvent>;+ readonly "turn.completed"?: InstrumentationEventHandler<InstrumentationTurnSettledEvent>;+ readonly "turn.failed"?: InstrumentationEventHandler<InstrumentationTurnFailedEvent>;readonly "turn.started"?: InstrumentationEventHandler<InstrumentationTurnStartedEvent>;};readonly flush?: () => void | PromiseLike<void>;readonly shutdown?: () => void | PromiseLike<void>;}-/** Events that carry an operation `id`, pairing a start with its terminal. */+/** Events that pair a start with its terminal under one `idempotencyKey`. */export type InstrumentationCorrelatedEvent =| InstrumentationInputRequestedEvent| InstrumentationInputResolvedEvent@@ -348,12 +410,12 @@ export type InstrumentationContextRunner = <T>(/** Stable identity supplied only to a trusted framework context runner. */export type InstrumentationExecutionOperation =| {- readonly id: string;+ readonly idempotencyKey: string;readonly scope: InstrumentationAttemptScope;readonly type: "model.call";}| {- readonly id: string;+ readonly idempotencyKey: string;readonly scope: InstrumentationAttemptScope;readonly type: "tool.call";};diff --git a/packages/eve/src/harness/instrumentation-native-events.test.ts b/packages/eve/src/harness/instrumentation-native-events.test.tsindex 7f6bdd8c3..6c883a5e7 100644--- a/packages/eve/src/harness/instrumentation-native-events.test.ts+++ b/packages/eve/src/harness/instrumentation-native-events.test.ts@@ -89,7 +89,14 @@ describe("createInstrumentationHandleEvent", () => {await handleEvent(createSessionWaitingEvent());- expect(events).toEqual([{ sessionId: "session-1", turnId: "turn-1", type: "session.waiting" }]);+ expect(events).toEqual([+ {+ idempotencyKey: "session:session-1",+ sessionId: "session-1",+ turnId: "turn-1",+ type: "session.waiting",+ },+ ]);});it("carries the dispatch lineage onto every turn a child session starts", async () => {@@ -117,6 +124,7 @@ describe("createInstrumentationHandleEvent", () => {expect(events.filter((event) => event.type === "turn.started")).toEqual([{+ idempotencyKey: "turn:child-1:child-turn-1",parentLineage,parentTraceContext: undefined,rootSessionId: "session-1",@@ -126,6 +134,7 @@ describe("createInstrumentationHandleEvent", () => {type: "turn.started",},{+ idempotencyKey: "turn:child-1:child-turn-2",parentLineage,parentTraceContext: undefined,rootSessionId: "session-1",diff --git a/packages/eve/src/harness/instrumentation-native-events.ts b/packages/eve/src/harness/instrumentation-native-events.tsindex 78615ade6..9d0de169c 100644--- a/packages/eve/src/harness/instrumentation-native-events.ts+++ b/packages/eve/src/harness/instrumentation-native-events.ts@@ -5,6 +5,7 @@ import type {InstrumentationPointEvent,InstrumentationTraceContext,} from "#harness/instrumentation-lifecycle.js";+import { sessionIdempotencyKey, turnIdempotencyKey } from "#harness/instrumentation-lifecycle.js";import type { HandleEventFn } from "#harness/types.js";export interface CreateInstrumentationHandleEventInput {@@ -45,6 +46,7 @@ function toLifecycleEvent(case "session.started":return {agentName: input.agentName,+ idempotencyKey: sessionIdempotencyKey(input.sessionId),parentTraceContext: input.parentTraceContext,rootSessionId: input.rootSessionId ?? input.sessionId,sessionId: input.sessionId,@@ -52,16 +54,23 @@ function toLifecycleEvent(};case "session.completed":case "session.waiting":- return { sessionId: input.sessionId, turnId: activeTurnId, type: event.type };+ return {+ idempotencyKey: sessionIdempotencyKey(input.sessionId),+ sessionId: input.sessionId,+ turnId: activeTurnId,+ type: event.type,+ };case "session.failed":return {error: new Error(event.data.message),+ idempotencyKey: sessionIdempotencyKey(input.sessionId),sessionId: input.sessionId,turnId: activeTurnId,type: "session.failed",};case "turn.started":return {+ idempotencyKey: turnIdempotencyKey(input.sessionId, event.data.turnId),parentLineage: input.parentLineage,parentTraceContext: input.parentTraceContext,rootSessionId: input.rootSessionId ?? input.sessionId,@@ -72,10 +81,16 @@ function toLifecycleEvent(};case "turn.completed":case "turn.cancelled":- return { sessionId: input.sessionId, turnId: event.data.turnId, type: event.type };+ return {+ idempotencyKey: turnIdempotencyKey(input.sessionId, event.data.turnId),+ sessionId: input.sessionId,+ turnId: event.data.turnId,+ type: event.type,+ };case "turn.failed":return {error: new Error(event.data.message),+ idempotencyKey: turnIdempotencyKey(input.sessionId, event.data.turnId),sessionId: input.sessionId,turnId: event.data.turnId,type: "turn.failed",diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.tsindex caee8d531..d91ce21ff 100644--- a/packages/eve/src/harness/tool-loop.ts+++ b/packages/eve/src/harness/tool-loop.ts@@ -134,6 +134,7 @@ import { buildTelemetryRuntimeContext } from "#harness/instrumentation-runtime-cimport { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";import { createInstrumentationHandleEvent } from "#harness/instrumentation-native-events.js";import type { InstrumentationAttemptScope } from "#harness/instrumentation-lifecycle.js";+import { attemptIdempotencyKey } from "#harness/instrumentation-lifecycle.js";import { resolveParentLineage } from "#harness/parent-lineage.js";import { ChannelKey } from "#runtime/sessions/runtime-context-keys.js";import {@@ -1433,6 +1434,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {const result = await executeModelCall();if (attemptScope !== undefined) {await instrumentationHooks?.publish({+ idempotencyKey: attemptIdempotencyKey(attemptScope),scope: attemptScope,type: "step.attempt.completed",});@@ -1442,6 +1444,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {if (attemptScope !== undefined) {await instrumentationHooks?.publish({error,+ idempotencyKey: attemptIdempotencyKey(attemptScope),scope: attemptScope,type: "step.attempt.failed",});diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex 93bd6df43..2f617df9d 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -15,7 +15,10 @@ import {SESSION_WINDOW_TURN_LIMIT,} from "#tracing/agent-trace-state.js";import {+ attemptIdempotencyKey,createInstrumentationHooks,+ sessionIdempotencyKey,+ turnIdempotencyKey,type InstrumentationAttemptScope,type InstrumentationContextRunner,type InstrumentationHooks,@@ -163,6 +166,7 @@ async function emitAttempt(input: {if (input.providerMetadata !== undefined) {await input.hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),providerMetadata: input.providerMetadata,scope,type: "step.attempt.metadata",@@ -171,15 +175,26 @@ async function emitAttempt(input: {await input.hooks.publish(input.attemptError === undefined- ? { scope, type: "step.attempt.completed" }- : { error: input.attemptError, scope, type: "step.attempt.failed" },+ ? {+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ type: "step.attempt.completed",+ }+ : {+ error: input.attemptError,+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ type: "step.attempt.failed",+ },);await input.hooks.publish({+ idempotencyKey: turnIdempotencyKey(input.sessionId, input.turnId),sessionId: input.sessionId,turnId: input.turnId,type: "turn.completed",});await input.hooks.publish({+ idempotencyKey: sessionIdempotencyKey(input.sessionId),sessionId: input.sessionId,turnId: input.turnId,type: "session.waiting",@@ -199,12 +214,14 @@ async function publishTurnStarted(input: {await input.hooks.publish({agentName: "weather",channelKind: "http",+ idempotencyKey: sessionIdempotencyKey(input.sessionId),parentTraceContext: input.parentTraceContext,rootSessionId,sessionId: input.sessionId,type: "session.started",});await input.hooks.publish({+ idempotencyKey: turnIdempotencyKey(input.sessionId, input.turnId),parentLineage: input.parentLineage,parentTraceContext: input.parentTraceContext,rootSessionId,@@ -221,8 +238,18 @@ async function completeTurn(sessionId: string,turnId: string,): Promise<void> {- await hooks.publish({ sessionId, turnId, type: "turn.completed" });- await hooks.publish({ sessionId, turnId, type: "session.waiting" });+ await hooks.publish({+ idempotencyKey: turnIdempotencyKey(sessionId, turnId),+ sessionId,+ turnId,+ type: "turn.completed",+ });+ await hooks.publish({+ idempotencyKey: sessionIdempotencyKey(sessionId),+ sessionId,+ turnId,+ type: "session.waiting",+ });}function byName(spans: readonly ReadableSpan[], name: string): ReadableSpan[] {@@ -401,11 +428,13 @@ describe("createAgentOtelInstrumentation", () => {await runtime.hooks.publish({agentName: "weather",channelKind: "http",+ idempotencyKey: sessionIdempotencyKey("session-1"),rootSessionId: "session-1",sessionId: "session-1",type: "session.started",});await runtime.hooks.publish({+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),rootSessionId: "session-1",sequence: 0,sessionId: "session-1",diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex c4d67047b..cf711b8b5 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -41,6 +41,7 @@ import type {InstrumentationTurnTerminalEvent,InstrumentationUsage,} from "#harness/instrumentation-lifecycle.js";+import { sessionIdempotencyKey } from "#harness/instrumentation-lifecycle.js";interface SpanState {readonly context: Context;@@ -100,6 +101,7 @@ export function createAgentOtelInstrumentation(await ensureSessionContext({agentName: undefined,channelKind: undefined,+ idempotencyKey: sessionIdempotencyKey(event.sessionId),parentTraceContext: event.parentTraceContext,rootSessionId: event.rootSessionId,sessionId: event.sessionId,@@ -197,7 +199,10 @@ export function createAgentOtelInstrumentation(if (turn === undefined) return;await input.stateStore.setTurn(event.sessionId, event.turnId, {...turn,- terminal: { error: event.error, type: event.type },+ terminal:+ event.type === "turn.failed"+ ? { error: event.error, type: event.type }+ : { type: event.type },});};@@ -276,13 +281,13 @@ export function createAgentOtelInstrumentation(if (system !== undefined) span.setAttribute("ai.prompt.system", system);}const state = { context: trace.setSpan(attempt.operation.context, span), span };- getExecutionContexts(event.scope).models.set(event.id, state.context);- getSpanStates(modelSpans, event.scope).set(event.id, state);+ getExecutionContexts(event.scope).models.set(event.idempotencyKey, state.context);+ getSpanStates(modelSpans, event.scope).set(event.idempotencyKey, state);};const onModelCallTerminal = (event: InstrumentationModelCallTerminalEvent): void => {- executionContexts.get(event.scope)?.models.delete(event.id);- const state = takeSpanState(modelSpans, event.scope, event.id);+ executionContexts.get(event.scope)?.models.delete(event.idempotencyKey);+ const state = takeSpanState(modelSpans, event.scope, event.idempotencyKey);if (state === undefined) return;if (event.type === "model.call.failed") {recordError(state.span, event.error);@@ -385,13 +390,13 @@ export function createAgentOtelInstrumentation(span: actionSpan,toolSpan,};- getExecutionContexts(event.scope).tools.set(event.id, state.context);- getSpanStates(toolSpans, event.scope).set(event.id, state);+ getExecutionContexts(event.scope).tools.set(event.idempotencyKey, state.context);+ getSpanStates(toolSpans, event.scope).set(event.idempotencyKey, state);};const onToolCallTerminal = (event: InstrumentationToolCallTerminalEvent): void => {- executionContexts.get(event.scope)?.tools.delete(event.id);- const state = takeSpanState(toolSpans, event.scope, event.id);+ executionContexts.get(event.scope)?.tools.delete(event.idempotencyKey);+ const state = takeSpanState(toolSpans, event.scope, event.idempotencyKey);if (state === undefined) return;if (event.type === "tool.call.failed") {recordError(state.toolSpan, event.error);@@ -524,8 +529,8 @@ export function createAgentOtelInstrumentation(const contexts = executionContexts.get(operation.scope);const parent =operation.type === "model.call"- ? contexts?.models.get(operation.id)- : contexts?.tools.get(operation.id);+ ? contexts?.models.get(operation.idempotencyKey)+ : contexts?.tools.get(operation.idempotencyKey);return parent === undefined ? execute() : context.with(parent, execute);},};diff --git a/packages/eve/src/tracing/agent-trace-context-store.test.ts b/packages/eve/src/tracing/agent-trace-context-store.test.tsindex 9357f1446..51c36eb09 100644--- a/packages/eve/src/tracing/agent-trace-context-store.test.ts+++ b/packages/eve/src/tracing/agent-trace-context-store.test.ts@@ -53,7 +53,9 @@ describe("ContextAgentTraceStateStore", () => {parentSpanId: "2".repeat(16),startTimeMs: 1_700_000_000_000,});- expect(store.getTurn("session-1", "turn-1")?.terminal?.error).toMatchObject({+ const terminal = store.getTurn("session-1", "turn-1")?.terminal;+ expect(terminal?.type).toBe("turn.failed");+ expect(terminal?.type === "turn.failed" ? terminal.error : undefined).toMatchObject({message: "failed",});});diff --git a/packages/eve/src/tracing/agent-trace-context-store.ts b/packages/eve/src/tracing/agent-trace-context-store.tsindex 16fb85431..3c6111044 100644--- a/packages/eve/src/tracing/agent-trace-context-store.ts+++ b/packages/eve/src/tracing/agent-trace-context-store.ts@@ -112,10 +112,9 @@ function serializeState(state: AgentTraceContextState): unknown {terminal:value.terminal === undefined? undefined- : {- error: serializeError(value.terminal.error),- type: value.terminal.type,- },+ : value.terminal.type === "turn.failed"+ ? { error: serializeError(value.terminal.error), type: value.terminal.type }+ : { type: value.terminal.type },},]),),@@ -187,7 +186,7 @@ function deserializeTerminal(value: unknown): AgentTurnTraceState["terminal"] {if (!isRecord(value) || typeof value.type !== "string") return undefined;const type = value.type;if (!isTurnTerminalType(type)) return undefined;- return { error: deserializeError(value.error), type };+ return type === "turn.failed" ? { error: deserializeError(value.error), type } : { type };}function serializeSpanContext(context: SpanContext): Record<string, unknown> {diff --git a/packages/eve/src/tracing/agent-trace-state.ts b/packages/eve/src/tracing/agent-trace-state.tsindex dc571f48d..93e9f20b6 100644--- a/packages/eve/src/tracing/agent-trace-state.ts+++ b/packages/eve/src/tracing/agent-trace-state.ts@@ -2,7 +2,8 @@ import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";import type {InstrumentationParentLineage,- InstrumentationTurnTerminalEvent,+ InstrumentationTurnFailedEvent,+ InstrumentationTurnSettledEvent,} from "#harness/instrumentation-lifecycle.js";/** Sized so an ordinary session stays one trace and only an outsized one rolls. */@@ -24,10 +25,9 @@ export interface AgentTurnTraceState {readonly rootSessionId: string;readonly sequence: number;readonly startTimeMs: number;- readonly terminal?: {- readonly error?: unknown;- readonly type: InstrumentationTurnTerminalEvent["type"];- };+ readonly terminal?:+ | { readonly error: unknown; readonly type: InstrumentationTurnFailedEvent["type"] }+ | { readonly type: InstrumentationTurnSettledEvent["type"] };}/** Provider-owned serializable storage for durable agent trace state. */diff --git a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.tsindex 34f017215..ec8755b3e 100644--- a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts+++ b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts@@ -11,6 +11,11 @@ import { ContextContainer, contextStorage } from "#context/container.js";import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";import { listLocalTraces } from "#tracing/local-trace-reader.js";import type { InstrumentationAttemptScope } from "#harness/instrumentation-lifecycle.js";+import {+ attemptIdempotencyKey,+ sessionIdempotencyKey,+ turnIdempotencyKey,+} from "#harness/instrumentation-lifecycle.js";import { installLocalInstrumentationRuntime } from "#tracing/local-instrumentation-runtime.js";import { LocalTraceSpanProcessor } from "#tracing/local-trace-span-processor.js";@@ -49,11 +54,13 @@ describe("local instrumentation runtime", () => {await contextStorage.run(new ContextContainer(), async () => {await runtime.hooks.publish({agentName: "weather",+ idempotencyKey: sessionIdempotencyKey("session-1"),rootSessionId: "session-1",sessionId: "session-1",type: "session.started",});await runtime.hooks.publish({+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),rootSessionId: "session-1",sequence: 0,sessionId: "session-1",@@ -117,14 +124,20 @@ describe("local instrumentation runtime", () => {toolOutput: { output: { temperature: 72 }, type: "tool-result" },},]);- await runtime.hooks.publish({ scope, type: "step.attempt.completed" });await runtime.hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ type: "step.attempt.completed",+ });+ await runtime.hooks.publish({+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),sessionId: "session-1",turnId: "turn-1",type: "turn.completed",});// Settling the turn emits the turn span with the pre-allocated id.await runtime.hooks.publish({+ idempotencyKey: sessionIdempotencyKey("session-1"),sessionId: "session-1",turnId: "turn-1",type: "session.waiting",