Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(eve): add replay-stable instrumentation event identity (#1859)

Chad Hietalaf06633b20201
diff --git a/.changeset/instrumentation-event-identity.md b/.changeset/instrumentation-event-identity.md
new file mode 100644
index 000000000..b660b00cd
--- /dev/null
+++ b/.changeset/instrumentation-event-identity.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Give every instrumentation lifecycle event a replay-stable `idempotencyKey` derived from durable eve identity, allowing providers to upsert one record across retries and worker replays.
diff --git a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts b/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts
index 7b6017360..fa647500f 100644
--- a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts
+++ b/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts
@@ -29,11 +29,13 @@ describe("createAiSdkHookBridge", () => {
return {
events: {
"model.call.started"(event) {
- calls.push(`${name}:started:${event.id}`);
- states.set(event.id, `${name}-state`);
+ calls.push(`${name}:started:${event.idempotencyKey}`);
+ states.set(event.idempotencyKey, `${name}-state`);
},
"model.call.completed"(event) {
- calls.push(`${name}:completed:${event.id}:${String(states.get(event.id))}`);
+ calls.push(
+ `${name}:completed:${event.idempotencyKey}:${String(states.get(event.idempotencyKey))}`,
+ );
},
},
};
@@ -55,7 +57,7 @@ describe("createAiSdkHookBridge", () => {
},
]);
- const id = `${scope.attemptId}:model:call-1:0`;
+ const id = `model:${scope.attemptId}:0`;
expect(calls).toEqual([
`a:started:${id}`,
`b:started:${id}`,
@@ -91,10 +93,10 @@ describe("createAiSdkHookBridge", () => {
it("passes the identity captured at model-call start to the context runner", async () => {
const ids: string[] = [];
const hooks = createInstrumentationHooks([
- { events: { "model.call.started": (event) => void ids.push(event.id) } },
+ { events: { "model.call.started": (event) => void ids.push(event.idempotencyKey) } },
]);
const bridge = createAiSdkHookBridge(scope, hooks, (operation, execute) => {
- ids.push(operation.id);
+ ids.push(operation.idempotencyKey);
return execute();
});
Reflect.apply(bridge.onStart!, bridge, [
@@ -108,7 +110,7 @@ describe("createAiSdkHookBridge", () => {
await bridge.executeLanguageModelCall!({ callId: "call-1", execute: async () => "result" });
- const expected = `${scope.attemptId}:model:call-1:0`;
+ const expected = `model:${scope.attemptId}:0`;
expect(ids).toEqual([expected, expected]);
});
@@ -126,6 +128,23 @@ describe("createAiSdkHookBridge", () => {
expect(adapterCalls).toBe(0);
});
+ it("derives replay-stable model identity without the AI SDK call ID", async () => {
+ const keys: string[] = [];
+ const hooks = createInstrumentationHooks([
+ { events: { "model.call.started": (event) => void keys.push(event.idempotencyKey) } },
+ ]);
+
+ for (const callId of ["sdk-random-1", "sdk-random-2"]) {
+ const bridge = createAiSdkHookBridge(scope, hooks);
+ await Reflect.apply(bridge.onStepStart!, bridge, [{ callId, stepNumber: 2 }]);
+ await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [
+ { callId, messages: [], modelId: "model", provider: "test", tools: undefined },
+ ]);
+ }
+
+ expect(keys).toEqual([`model:${scope.attemptId}:2`, `model:${scope.attemptId}:2`]);
+ });
+
it("publishes step provider metadata as step.metadata, skipping steps without any", async () => {
const events: InstrumentationStepAttemptMetadataEvent[] = [];
const hooks = createInstrumentationHooks([
@@ -146,6 +165,7 @@ describe("createAiSdkHookBridge", () => {
expect(events).toEqual([
{
+ idempotencyKey: `step:${scope.attemptId}`,
providerMetadata: { gateway: { cost: "0.000082" } },
scope,
type: "step.attempt.metadata",
@@ -240,6 +260,7 @@ describe("createAiSdkHookBridge", () => {
await Reflect.apply(bridge.onStepStart!, bridge, [{ callId: "call-1", stepNumber: 0 }]);
const expected = {
+ idempotencyKey: `step:${scope.attemptId}`,
operation: { modelId: "model", operationId: "ai.streamText", provider: "test" },
scope,
type: "step.attempt.started",
@@ -313,7 +334,7 @@ describe("createAiSdkHookBridge", () => {
]);
expect(before).toHaveBeenCalledExactlyOnceWith({
- id: `${scope.attemptId}:model:call-1:0`,
+ idempotencyKey: `model:${scope.attemptId}:0`,
input: { instructions: "be brief", messages: [{ content: "hi", role: "user" }] },
model: { modelId: "model", provider: "test" },
scope,
@@ -342,7 +363,7 @@ describe("createAiSdkHookBridge", () => {
},
],
finishReason: "tool-calls",
- id: `${scope.attemptId}:model:call-1:0`,
+ idempotencyKey: `model:${scope.attemptId}:0`,
scope,
type: "model.call.completed",
usage: {
@@ -386,14 +407,14 @@ describe("createAiSdkHookBridge", () => {
expect(before).toHaveBeenCalledExactlyOnceWith({
callId: "tool-1",
- id: `${scope.attemptId}:tool:tool-1:0`,
+ idempotencyKey: `tool:${scope.attemptId}:tool-1:0`,
input: { q: "eve" },
scope,
toolName: "search",
type: "tool.call.started",
});
expect(after).toHaveBeenCalledExactlyOnceWith({
- id: `${scope.attemptId}:tool:tool-1:0`,
+ idempotencyKey: `tool:${scope.attemptId}:tool-1:0`,
output: expected,
scope,
type: "tool.call.completed",
@@ -408,9 +429,9 @@ describe("createAiSdkHookBridge", () => {
return {
events: {
"model.call.completed": (event) => {
- observed.set(name, own.get(event.id));
+ observed.set(name, own.get(event.idempotencyKey));
},
- "model.call.started": (event) => void own.set(event.id, `${name}-state`),
+ "model.call.started": (event) => void own.set(event.idempotencyKey, `${name}-state`),
},
};
};
@@ -469,14 +490,14 @@ describe("createAiSdkHookBridge", () => {
events: {
async "tool.call.started"(event) {
started.set(
- event.id,
+ event.idempotencyKey,
await new Promise<string>((resolve) => {
- resolvers.set(event.id, () => resolve(`state:${event.id}`));
+ resolvers.set(event.idempotencyKey, () => resolve(`state:${event.idempotencyKey}`));
}),
);
},
"tool.call.completed"(event) {
- terminalStates.set(event.id, started.get(event.id));
+ terminalStates.set(event.idempotencyKey, started.get(event.idempotencyKey));
},
},
},
@@ -493,8 +514,8 @@ describe("createAiSdkHookBridge", () => {
const second = start("tool-2");
await vi.waitFor(() => expect(resolvers.size).toBe(2));
- const firstId = `${scope.attemptId}:tool:tool-1:0`;
- const secondId = `${scope.attemptId}:tool:tool-2:0`;
+ const firstId = `tool:${scope.attemptId}:tool-1:0`;
+ const secondId = `tool:${scope.attemptId}:tool-2:0`;
resolvers.get(secondId)!();
resolvers.get(firstId)!();
await Promise.all([first, second]);
diff --git a/packages/eve/src/harness/ai-sdk-hook-bridge.ts b/packages/eve/src/harness/ai-sdk-hook-bridge.ts
index d0239cf18..0807e7ea1 100644
--- a/packages/eve/src/harness/ai-sdk-hook-bridge.ts
+++ b/packages/eve/src/harness/ai-sdk-hook-bridge.ts
@@ -14,13 +14,18 @@ import type {
InstrumentationToolOutput,
InstrumentationUsage,
} from "#harness/instrumentation-lifecycle.js";
+import {
+ attemptIdempotencyKey,
+ modelCallIdempotencyKey,
+ toolCallIdempotencyKey,
+} from "#harness/instrumentation-lifecycle.js";
type TelemetryEvent<TKey extends keyof Telemetry> = Parameters<NonNullable<Telemetry[TKey]>>[0];
interface AttemptState {
- readonly modelIds: Map<string, string>;
+ readonly modelKeys: Map<string, string>;
readonly scope: InstrumentationAttemptScope;
- readonly toolIds: Map<string, string>;
+ readonly toolKeys: Map<string, string>;
operation?: InstrumentationOperationRef;
// Only the number is kept: it disambiguates call identities within an attempt.
stepNumber?: number;
@@ -33,9 +38,9 @@ export function createAiSdkHookBridge(
runInContext: InstrumentationContextRunner = directRunInContext,
): Telemetry {
const state: AttemptState = {
- modelIds: new Map(),
+ modelKeys: new Map(),
scope,
- toolIds: new Map(),
+ toolKeys: new Map(),
};
return {
@@ -52,22 +57,22 @@ export function createAiSdkHookBridge(
if (started !== undefined) await hooks.publish(started);
},
async onLanguageModelCallStart(event) {
- const id = createModelCallIdentity(state, event.callId);
- state.modelIds.set(event.callId, id);
- const started = toModelCallStarted(state, id, event);
+ const key = modelCallIdempotencyKey(state.scope, state.stepNumber ?? 0);
+ state.modelKeys.set(event.callId, key);
+ const started = toModelCallStarted(state, key, event);
await hooks.publish(started);
},
executeLanguageModelCall({ callId, execute }) {
- const id = state.modelIds.get(callId);
- return id === undefined
+ const key = state.modelKeys.get(callId);
+ return key === undefined
? execute()
- : runInContext({ id, scope, type: "model.call" }, execute);
+ : runInContext({ idempotencyKey: key, scope, type: "model.call" }, execute);
},
async onLanguageModelCallEnd(event) {
- const id = state.modelIds.get(event.callId);
- if (id === undefined) return;
- state.modelIds.delete(event.callId);
- const completed = toModelCallCompleted(state, id, event);
+ const key = state.modelKeys.get(event.callId);
+ if (key === undefined) return;
+ state.modelKeys.delete(event.callId);
+ const completed = toModelCallCompleted(state, key, event);
await hooks.publish(completed);
},
async onStepEnd(event) {
@@ -77,6 +82,7 @@ export function createAiSdkHookBridge(
if (event.providerMetadata === undefined) return;
await hooks.publish(
Object.freeze({
+ idempotencyKey: attemptIdempotencyKey(state.scope),
providerMetadata: event.providerMetadata,
scope: state.scope,
type: "step.attempt.metadata",
@@ -84,21 +90,27 @@ export function createAiSdkHookBridge(
);
},
async onToolExecutionStart(event) {
- const id = createToolCallIdentity(state, event.toolCall.toolCallId);
- state.toolIds.set(event.toolCall.toolCallId, id);
- const started = toToolCallStarted(state, id, event);
+ const key = toolCallIdempotencyKey(
+ state.scope,
+ event.toolCall.toolCallId,
+ state.stepNumber ?? 0,
+ );
+ state.toolKeys.set(event.toolCall.toolCallId, key);
+ const started = toToolCallStarted(state, key, event);
await hooks.publish(started);
},
executeTool({ toolCallId, execute }) {
- const id = state.toolIds.get(toolCallId);
- return id === undefined ? execute() : runInContext({ id, scope, type: "tool.call" }, execute);
+ const key = state.toolKeys.get(toolCallId);
+ return key === undefined
+ ? execute()
+ : runInContext({ idempotencyKey: key, scope, type: "tool.call" }, execute);
},
async onToolExecutionEnd(event) {
const toolCallId = event.toolCall.toolCallId;
- const id = state.toolIds.get(toolCallId);
- if (id === undefined) return;
- state.toolIds.delete(toolCallId);
- const completed = toToolCallCompleted(state, id, event);
+ const key = state.toolKeys.get(toolCallId);
+ if (key === undefined) return;
+ state.toolKeys.delete(toolCallId);
+ const completed = toToolCallCompleted(state, key, event);
await hooks.publish(completed);
},
async onAbort(event) {
@@ -111,14 +123,18 @@ export function createAiSdkHookBridge(
async function failOpenOperations(error: unknown): Promise<void> {
const pending: Promise<void>[] = [];
- for (const id of state.modelIds.values()) {
- pending.push(hooks.publish(Object.freeze({ error, id, scope, type: "model.call.failed" })));
+ for (const idempotencyKey of state.modelKeys.values()) {
+ pending.push(
+ hooks.publish(Object.freeze({ error, idempotencyKey, scope, type: "model.call.failed" })),
+ );
}
- for (const id of state.toolIds.values()) {
- pending.push(hooks.publish(Object.freeze({ error, id, scope, type: "tool.call.failed" })));
+ for (const idempotencyKey of state.toolKeys.values()) {
+ pending.push(
+ hooks.publish(Object.freeze({ error, idempotencyKey, scope, type: "tool.call.failed" })),
+ );
}
- state.modelIds.clear();
- state.toolIds.clear();
+ state.modelKeys.clear();
+ state.toolKeys.clear();
await Promise.all(pending);
}
}
@@ -130,23 +146,20 @@ function toStepAttemptStarted(
): InstrumentationStepAttemptStartedEvent | undefined {
if (state.operation === undefined || state.stepNumber === undefined) return undefined;
return Object.freeze({
+ idempotencyKey: attemptIdempotencyKey(state.scope),
operation: state.operation,
scope: state.scope,
type: "step.attempt.started",
});
}
-function createModelCallIdentity(state: AttemptState, callId: string): string {
- return `${state.scope.attemptId}:model:${callId}:${state.stepNumber ?? 0}`;
-}
-
function toModelCallStarted(
state: AttemptState,
- id: string,
+ idempotencyKey: string,
source: TelemetryEvent<"onLanguageModelCallStart">,
): InstrumentationModelCallStartedEvent {
return Object.freeze({
- id,
+ idempotencyKey,
input: Object.freeze({
instructions: source.instructions,
messages: Object.freeze([...source.messages]),
@@ -159,13 +172,13 @@ function toModelCallStarted(
function toModelCallCompleted(
state: AttemptState,
- id: string,
+ idempotencyKey: string,
source: TelemetryEvent<"onLanguageModelCallEnd">,
): InstrumentationModelCallCompletedEvent {
return Object.freeze({
content: toContentParts(source.content),
finishReason: source.finishReason,
- id,
+ idempotencyKey,
scope: state.scope,
type: "model.call.completed",
usage: toUsage(source.usage),
@@ -233,18 +246,14 @@ function toContentParts(
return Object.freeze(parts);
}
-function createToolCallIdentity(state: AttemptState, toolCallId: string): string {
- return `${state.scope.attemptId}:tool:${toolCallId}:${state.stepNumber ?? 0}`;
-}
-
function toToolCallStarted(
state: AttemptState,
- id: string,
+ idempotencyKey: string,
source: TelemetryEvent<"onToolExecutionStart">,
): InstrumentationToolCallStartedEvent {
return Object.freeze({
callId: source.toolCall.toolCallId,
- id,
+ idempotencyKey,
input: source.toolCall.input,
scope: state.scope,
toolName: source.toolCall.toolName,
@@ -254,11 +263,11 @@ function toToolCallStarted(
function toToolCallCompleted(
state: AttemptState,
- id: string,
+ idempotencyKey: string,
source: TelemetryEvent<"onToolExecutionEnd">,
): InstrumentationToolCallCompletedEvent {
return Object.freeze({
- id,
+ idempotencyKey,
output: toToolOutput(source.toolOutput),
scope: state.scope,
type: "tool.call.completed",
diff --git a/packages/eve/src/harness/instrumentation-lifecycle.test.ts b/packages/eve/src/harness/instrumentation-lifecycle.test.ts
new file mode 100644
index 000000000..59af53480
--- /dev/null
+++ b/packages/eve/src/harness/instrumentation-lifecycle.test.ts
@@ -0,0 +1,50 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ attemptIdempotencyKey,
+ inputIdempotencyKey,
+ modelCallIdempotencyKey,
+ sessionIdempotencyKey,
+ toolCallIdempotencyKey,
+ turnIdempotencyKey,
+ type InstrumentationAttemptScope,
+} from "#harness/instrumentation-lifecycle.js";
+
+const scope: InstrumentationAttemptScope = {
+ attemptId: "session-1:turn-1:0:0",
+ attemptIndex: 0,
+ sessionId: "session-1",
+ stepIndex: 0,
+ turnId: "turn-1",
+};
+
+describe("instrumentation idempotency keys", () => {
+ it("separates classes that share an identifier", () => {
+ expect(sessionIdempotencyKey("shared")).not.toBe(turnIdempotencyKey("shared", "shared"));
+ });
+
+ it("separates identical turn IDs in different sessions", () => {
+ expect(turnIdempotencyKey("session-1", "turn_0")).not.toBe(
+ turnIdempotencyKey("session-2", "turn_0"),
+ );
+ });
+
+ it("scopes input requests to their originating turn", () => {
+ expect(inputIdempotencyKey("session-1", "turn-1", "request-1")).toBe(
+ "input:session-1:turn-1:request-1",
+ );
+ expect(inputIdempotencyKey("session-1", "turn-1", "request-1")).not.toBe(
+ inputIdempotencyKey("session-1", "turn-2", "request-1"),
+ );
+ });
+
+ it("derives model identity without an AI SDK call ID", () => {
+ expect(modelCallIdempotencyKey(scope, 2)).toBe("model:session-1:turn-1:0:0:2");
+ });
+
+ it("separates model attempts and SDK tool calls", () => {
+ const retry = { ...scope, attemptId: "session-1:turn-1:0:1", attemptIndex: 1 };
+ expect(attemptIdempotencyKey(scope)).not.toBe(attemptIdempotencyKey(retry));
+ expect(modelCallIdempotencyKey(scope, 0)).not.toBe(toolCallIdempotencyKey(scope, "call-1", 0));
+ });
+});
diff --git a/packages/eve/src/harness/instrumentation-lifecycle.ts b/packages/eve/src/harness/instrumentation-lifecycle.ts
index 8ec601e80..5f78e03e2 100644
--- a/packages/eve/src/harness/instrumentation-lifecycle.ts
+++ b/packages/eve/src/harness/instrumentation-lifecycle.ts
@@ -81,6 +81,39 @@ export type InstrumentationToolOutput =
| { readonly type: "result"; readonly output: unknown }
| { readonly type: "error"; readonly error: unknown };
+/** Replay-stable row identity for every lifecycle operation. */
+export function sessionIdempotencyKey(sessionId: string): string {
+ return `session:${sessionId}`;
+}
+
+export function turnIdempotencyKey(sessionId: string, turnId: string): string {
+ return `turn:${sessionId}:${turnId}`;
+}
+
+export function attemptIdempotencyKey(scope: InstrumentationAttemptScope): string {
+ return `step:${scope.attemptId}`;
+}
+
+/** One model call occurs per AI SDK step within an eve attempt. */
+export function modelCallIdempotencyKey(
+ scope: InstrumentationAttemptScope,
+ stepNumber: number,
+): string {
+ return `model:${scope.attemptId}:${String(stepNumber)}`;
+}
+
+export function toolCallIdempotencyKey(
+ scope: InstrumentationAttemptScope,
+ callId: string,
+ stepNumber: number,
+): string {
+ return `tool:${scope.attemptId}:${callId}:${String(stepNumber)}`;
+}
+
+export function inputIdempotencyKey(sessionId: string, turnId: string, requestId: string): string {
+ return `input:${sessionId}:${turnId}:${requestId}`;
+}
+
/** Framework-owned reason an agent suspended for user input. */
export type InstrumentationInputKind = "question" | "session-limit" | "tool-approval";
@@ -120,7 +153,7 @@ export interface InstrumentationInputRequestedEvent {
readonly callId: string;
readonly name: string;
};
- readonly id: string;
+ readonly idempotencyKey: string;
readonly kind: InstrumentationInputKind;
readonly request: InstrumentationInputRequest;
readonly requestId: string;
@@ -130,7 +163,7 @@ export interface InstrumentationInputRequestedEvent {
export interface InstrumentationInputResolvedEvent {
readonly type: "input.resolved";
readonly error?: unknown;
- readonly id: string;
+ readonly idempotencyKey: string;
readonly kind: InstrumentationInputKind;
readonly outcome: InstrumentationInputOutcome;
readonly requestId: string;
@@ -140,6 +173,7 @@ export interface InstrumentationInputResolvedEvent {
export interface InstrumentationStepAttemptStartedEvent {
readonly type: "step.attempt.started";
+ readonly idempotencyKey: string;
readonly operation: InstrumentationOperationRef;
readonly scope: InstrumentationAttemptScope;
}
@@ -148,6 +182,7 @@ export interface InstrumentationSessionStartedEvent {
readonly type: "session.started";
readonly agentName?: string;
readonly channelKind?: string;
+ readonly idempotencyKey: string;
readonly parentTraceContext?: InstrumentationTraceContext;
readonly rootSessionId: string;
readonly sessionId: string;
@@ -179,6 +214,7 @@ export interface InstrumentationParentLineage {
*/
export interface InstrumentationSessionSettledEvent {
readonly type: "session.completed" | "session.waiting";
+ readonly idempotencyKey: string;
readonly sessionId: string;
readonly turnId?: string;
}
@@ -186,6 +222,7 @@ export interface InstrumentationSessionSettledEvent {
export interface InstrumentationSessionFailedEvent {
readonly type: "session.failed";
readonly error: unknown;
+ readonly idempotencyKey: string;
readonly sessionId: string;
readonly turnId?: string;
}
@@ -196,6 +233,7 @@ export type InstrumentationSessionTransitionEvent =
export interface InstrumentationTurnStartedEvent {
readonly type: "turn.started";
+ readonly idempotencyKey: string;
readonly parentLineage?: InstrumentationParentLineage;
readonly parentTraceContext?: InstrumentationTraceContext;
readonly rootSessionId: string;
@@ -204,19 +242,42 @@ export interface InstrumentationTurnStartedEvent {
readonly turnId: string;
}
-export interface InstrumentationTurnTerminalEvent {
- readonly type: "turn.cancelled" | "turn.completed" | "turn.failed";
- readonly error?: unknown;
+export interface InstrumentationTurnSettledEvent {
+ readonly type: "turn.cancelled" | "turn.completed";
+ readonly idempotencyKey: string;
readonly sessionId: string;
readonly turnId: string;
}
-export interface InstrumentationStepAttemptTerminalEvent {
- readonly type: "step.attempt.completed" | "step.attempt.failed";
- readonly error?: unknown;
+export interface InstrumentationTurnFailedEvent {
+ readonly type: "turn.failed";
+ readonly error: unknown;
+ readonly idempotencyKey: string;
+ readonly sessionId: string;
+ readonly turnId: string;
+}
+
+export type InstrumentationTurnTerminalEvent =
+ | InstrumentationTurnSettledEvent
+ | InstrumentationTurnFailedEvent;
+
+export interface InstrumentationStepAttemptCompletedEvent {
+ readonly type: "step.attempt.completed";
+ readonly idempotencyKey: string;
+ readonly scope: InstrumentationAttemptScope;
+}
+
+export interface InstrumentationStepAttemptFailedEvent {
+ readonly type: "step.attempt.failed";
+ readonly error: unknown;
+ readonly idempotencyKey: string;
readonly scope: InstrumentationAttemptScope;
}
+export type InstrumentationStepAttemptTerminalEvent =
+ | InstrumentationStepAttemptCompletedEvent
+ | InstrumentationStepAttemptFailedEvent;
+
/**
* Provider metadata for one completed attempt, as reported by the AI SDK
* (`StepResult.providerMetadata`). Carries Vercel AI Gateway cost data when
@@ -224,13 +285,14 @@ export interface InstrumentationStepAttemptTerminalEvent {
*/
export interface InstrumentationStepAttemptMetadataEvent {
readonly type: "step.attempt.metadata";
+ readonly idempotencyKey: string;
readonly scope: InstrumentationAttemptScope;
readonly providerMetadata: Readonly<Record<string, unknown>>;
}
export interface InstrumentationModelCallStartedEvent {
readonly type: "model.call.started";
- readonly id: string;
+ readonly idempotencyKey: string;
readonly input: InstrumentationModelInput;
readonly model: InstrumentationModelRef;
readonly scope: InstrumentationAttemptScope;
@@ -240,7 +302,7 @@ export interface InstrumentationModelCallCompletedEvent {
readonly type: "model.call.completed";
readonly content: readonly InstrumentationContentPart[];
readonly finishReason: string;
- readonly id: string;
+ readonly idempotencyKey: string;
readonly scope: InstrumentationAttemptScope;
readonly usage: InstrumentationUsage;
}
@@ -248,7 +310,7 @@ export interface InstrumentationModelCallCompletedEvent {
export interface InstrumentationModelCallFailedEvent {
readonly type: "model.call.failed";
readonly error: unknown;
- readonly id: string;
+ readonly idempotencyKey: string;
readonly scope: InstrumentationAttemptScope;
}
@@ -259,7 +321,7 @@ export type InstrumentationModelCallTerminalEvent =
export interface InstrumentationToolCallStartedEvent {
readonly type: "tool.call.started";
readonly callId: string;
- readonly id: string;
+ readonly idempotencyKey: string;
readonly input: unknown;
readonly scope: InstrumentationAttemptScope;
readonly toolName: string;
@@ -267,7 +329,7 @@ export interface InstrumentationToolCallStartedEvent {
export interface InstrumentationToolCallCompletedEvent {
readonly type: "tool.call.completed";
- readonly id: string;
+ readonly idempotencyKey: string;
readonly output: InstrumentationToolOutput;
readonly scope: InstrumentationAttemptScope;
}
@@ -275,7 +337,7 @@ export interface InstrumentationToolCallCompletedEvent {
export interface InstrumentationToolCallFailedEvent {
readonly type: "tool.call.failed";
readonly error: unknown;
- readonly id: string;
+ readonly idempotencyKey: string;
readonly scope: InstrumentationAttemptScope;
}
@@ -295,8 +357,8 @@ export interface InstrumentationProviderDefinition {
readonly name?: string;
readonly events?: {
readonly "step.attempt.started"?: InstrumentationEventHandler<InstrumentationStepAttemptStartedEvent>;
- readonly "step.attempt.completed"?: InstrumentationEventHandler<InstrumentationStepAttemptTerminalEvent>;
- readonly "step.attempt.failed"?: InstrumentationEventHandler<InstrumentationStepAttemptTerminalEvent>;
+ readonly "step.attempt.completed"?: InstrumentationEventHandler<InstrumentationStepAttemptCompletedEvent>;
+ readonly "step.attempt.failed"?: InstrumentationEventHandler<InstrumentationStepAttemptFailedEvent>;
readonly "step.attempt.metadata"?: InstrumentationEventHandler<InstrumentationStepAttemptMetadataEvent>;
readonly "model.call.started"?: InstrumentationEventHandler<InstrumentationModelCallStartedEvent>;
readonly "model.call.completed"?: InstrumentationEventHandler<InstrumentationModelCallCompletedEvent>;
@@ -310,16 +372,16 @@ export interface InstrumentationProviderDefinition {
readonly "tool.call.started"?: InstrumentationEventHandler<InstrumentationToolCallStartedEvent>;
readonly "tool.call.completed"?: InstrumentationEventHandler<InstrumentationToolCallCompletedEvent>;
readonly "tool.call.failed"?: InstrumentationEventHandler<InstrumentationToolCallFailedEvent>;
- readonly "turn.cancelled"?: InstrumentationEventHandler<InstrumentationTurnTerminalEvent>;
- readonly "turn.completed"?: InstrumentationEventHandler<InstrumentationTurnTerminalEvent>;
- readonly "turn.failed"?: InstrumentationEventHandler<InstrumentationTurnTerminalEvent>;
+ readonly "turn.cancelled"?: InstrumentationEventHandler<InstrumentationTurnSettledEvent>;
+ readonly "turn.completed"?: InstrumentationEventHandler<InstrumentationTurnSettledEvent>;
+ readonly "turn.failed"?: InstrumentationEventHandler<InstrumentationTurnFailedEvent>;
readonly "turn.started"?: InstrumentationEventHandler<InstrumentationTurnStartedEvent>;
};
readonly flush?: () => void | PromiseLike<void>;
readonly shutdown?: () => void | PromiseLike<void>;
}
-/** Events that carry an operation `id`, pairing a start with its terminal. */
+/** Events that pair a start with its terminal under one `idempotencyKey`. */
export type InstrumentationCorrelatedEvent =
| InstrumentationInputRequestedEvent
| InstrumentationInputResolvedEvent
@@ -348,12 +410,12 @@ export type InstrumentationContextRunner = <T>(
/** Stable identity supplied only to a trusted framework context runner. */
export type InstrumentationExecutionOperation =
| {
- readonly id: string;
+ readonly idempotencyKey: string;
readonly scope: InstrumentationAttemptScope;
readonly type: "model.call";
}
| {
- readonly id: string;
+ readonly idempotencyKey: string;
readonly scope: InstrumentationAttemptScope;
readonly type: "tool.call";
};
diff --git a/packages/eve/src/harness/instrumentation-native-events.test.ts b/packages/eve/src/harness/instrumentation-native-events.test.ts
index 7f6bdd8c3..6c883a5e7 100644
--- a/packages/eve/src/harness/instrumentation-native-events.test.ts
+++ b/packages/eve/src/harness/instrumentation-native-events.test.ts
@@ -89,7 +89,14 @@ describe("createInstrumentationHandleEvent", () => {
await handleEvent(createSessionWaitingEvent());
- expect(events).toEqual([{ sessionId: "session-1", turnId: "turn-1", type: "session.waiting" }]);
+ expect(events).toEqual([
+ {
+ idempotencyKey: "session:session-1",
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "session.waiting",
+ },
+ ]);
});
it("carries the dispatch lineage onto every turn a child session starts", async () => {
@@ -117,6 +124,7 @@ describe("createInstrumentationHandleEvent", () => {
expect(events.filter((event) => event.type === "turn.started")).toEqual([
{
+ idempotencyKey: "turn:child-1:child-turn-1",
parentLineage,
parentTraceContext: undefined,
rootSessionId: "session-1",
@@ -126,6 +134,7 @@ describe("createInstrumentationHandleEvent", () => {
type: "turn.started",
},
{
+ idempotencyKey: "turn:child-1:child-turn-2",
parentLineage,
parentTraceContext: undefined,
rootSessionId: "session-1",
diff --git a/packages/eve/src/harness/instrumentation-native-events.ts b/packages/eve/src/harness/instrumentation-native-events.ts
index 78615ade6..9d0de169c 100644
--- a/packages/eve/src/harness/instrumentation-native-events.ts
+++ b/packages/eve/src/harness/instrumentation-native-events.ts
@@ -5,6 +5,7 @@ import type {
InstrumentationPointEvent,
InstrumentationTraceContext,
} from "#harness/instrumentation-lifecycle.js";
+import { sessionIdempotencyKey, turnIdempotencyKey } from "#harness/instrumentation-lifecycle.js";
import type { HandleEventFn } from "#harness/types.js";
export interface CreateInstrumentationHandleEventInput {
@@ -45,6 +46,7 @@ function toLifecycleEvent(
case "session.started":
return {
agentName: input.agentName,
+ idempotencyKey: sessionIdempotencyKey(input.sessionId),
parentTraceContext: input.parentTraceContext,
rootSessionId: input.rootSessionId ?? input.sessionId,
sessionId: input.sessionId,
@@ -52,16 +54,23 @@ function toLifecycleEvent(
};
case "session.completed":
case "session.waiting":
- return { sessionId: input.sessionId, turnId: activeTurnId, type: event.type };
+ return {
+ idempotencyKey: sessionIdempotencyKey(input.sessionId),
+ sessionId: input.sessionId,
+ turnId: activeTurnId,
+ type: event.type,
+ };
case "session.failed":
return {
error: new Error(event.data.message),
+ idempotencyKey: sessionIdempotencyKey(input.sessionId),
sessionId: input.sessionId,
turnId: activeTurnId,
type: "session.failed",
};
case "turn.started":
return {
+ idempotencyKey: turnIdempotencyKey(input.sessionId, event.data.turnId),
parentLineage: input.parentLineage,
parentTraceContext: input.parentTraceContext,
rootSessionId: input.rootSessionId ?? input.sessionId,
@@ -72,10 +81,16 @@ function toLifecycleEvent(
};
case "turn.completed":
case "turn.cancelled":
- return { sessionId: input.sessionId, turnId: event.data.turnId, type: event.type };
+ return {
+ idempotencyKey: turnIdempotencyKey(input.sessionId, event.data.turnId),
+ sessionId: input.sessionId,
+ turnId: event.data.turnId,
+ type: event.type,
+ };
case "turn.failed":
return {
error: new Error(event.data.message),
+ idempotencyKey: turnIdempotencyKey(input.sessionId, event.data.turnId),
sessionId: input.sessionId,
turnId: event.data.turnId,
type: "turn.failed",
diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.ts
index caee8d531..d91ce21ff 100644
--- a/packages/eve/src/harness/tool-loop.ts
+++ b/packages/eve/src/harness/tool-loop.ts
@@ -134,6 +134,7 @@ import { buildTelemetryRuntimeContext } from "#harness/instrumentation-runtime-c
import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";
import { createInstrumentationHandleEvent } from "#harness/instrumentation-native-events.js";
import type { InstrumentationAttemptScope } from "#harness/instrumentation-lifecycle.js";
+import { attemptIdempotencyKey } from "#harness/instrumentation-lifecycle.js";
import { resolveParentLineage } from "#harness/parent-lineage.js";
import { ChannelKey } from "#runtime/sessions/runtime-context-keys.js";
import {
@@ -1433,6 +1434,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
const result = await executeModelCall();
if (attemptScope !== undefined) {
await instrumentationHooks?.publish({
+ idempotencyKey: attemptIdempotencyKey(attemptScope),
scope: attemptScope,
type: "step.attempt.completed",
});
@@ -1442,6 +1444,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
if (attemptScope !== undefined) {
await instrumentationHooks?.publish({
error,
+ idempotencyKey: attemptIdempotencyKey(attemptScope),
scope: attemptScope,
type: "step.attempt.failed",
});
diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.ts
index 93bd6df43..2f617df9d 100644
--- a/packages/eve/src/tracing/agent-otel-provider.test.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts
@@ -15,7 +15,10 @@ import {
SESSION_WINDOW_TURN_LIMIT,
} from "#tracing/agent-trace-state.js";
import {
+ attemptIdempotencyKey,
createInstrumentationHooks,
+ sessionIdempotencyKey,
+ turnIdempotencyKey,
type InstrumentationAttemptScope,
type InstrumentationContextRunner,
type InstrumentationHooks,
@@ -163,6 +166,7 @@ async function emitAttempt(input: {
if (input.providerMetadata !== undefined) {
await input.hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
providerMetadata: input.providerMetadata,
scope,
type: "step.attempt.metadata",
@@ -171,15 +175,26 @@ async function emitAttempt(input: {
await input.hooks.publish(
input.attemptError === undefined
- ? { scope, type: "step.attempt.completed" }
- : { error: input.attemptError, scope, type: "step.attempt.failed" },
+ ? {
+ idempotencyKey: attemptIdempotencyKey(scope),
+ scope,
+ type: "step.attempt.completed",
+ }
+ : {
+ error: input.attemptError,
+ idempotencyKey: attemptIdempotencyKey(scope),
+ scope,
+ type: "step.attempt.failed",
+ },
);
await input.hooks.publish({
+ idempotencyKey: turnIdempotencyKey(input.sessionId, input.turnId),
sessionId: input.sessionId,
turnId: input.turnId,
type: "turn.completed",
});
await input.hooks.publish({
+ idempotencyKey: sessionIdempotencyKey(input.sessionId),
sessionId: input.sessionId,
turnId: input.turnId,
type: "session.waiting",
@@ -199,12 +214,14 @@ async function publishTurnStarted(input: {
await input.hooks.publish({
agentName: "weather",
channelKind: "http",
+ idempotencyKey: sessionIdempotencyKey(input.sessionId),
parentTraceContext: input.parentTraceContext,
rootSessionId,
sessionId: input.sessionId,
type: "session.started",
});
await input.hooks.publish({
+ idempotencyKey: turnIdempotencyKey(input.sessionId, input.turnId),
parentLineage: input.parentLineage,
parentTraceContext: input.parentTraceContext,
rootSessionId,
@@ -221,8 +238,18 @@ async function completeTurn(
sessionId: string,
turnId: string,
): Promise<void> {
- await hooks.publish({ sessionId, turnId, type: "turn.completed" });
- await hooks.publish({ sessionId, turnId, type: "session.waiting" });
+ await hooks.publish({
+ idempotencyKey: turnIdempotencyKey(sessionId, turnId),
+ sessionId,
+ turnId,
+ type: "turn.completed",
+ });
+ await hooks.publish({
+ idempotencyKey: sessionIdempotencyKey(sessionId),
+ sessionId,
+ turnId,
+ type: "session.waiting",
+ });
}
function byName(spans: readonly ReadableSpan[], name: string): ReadableSpan[] {
@@ -401,11 +428,13 @@ describe("createAgentOtelInstrumentation", () => {
await runtime.hooks.publish({
agentName: "weather",
channelKind: "http",
+ idempotencyKey: sessionIdempotencyKey("session-1"),
rootSessionId: "session-1",
sessionId: "session-1",
type: "session.started",
});
await runtime.hooks.publish({
+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),
rootSessionId: "session-1",
sequence: 0,
sessionId: "session-1",
diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.ts
index c4d67047b..cf711b8b5 100644
--- a/packages/eve/src/tracing/agent-otel-provider.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.ts
@@ -41,6 +41,7 @@ import type {
InstrumentationTurnTerminalEvent,
InstrumentationUsage,
} from "#harness/instrumentation-lifecycle.js";
+import { sessionIdempotencyKey } from "#harness/instrumentation-lifecycle.js";
interface SpanState {
readonly context: Context;
@@ -100,6 +101,7 @@ export function createAgentOtelInstrumentation(
await ensureSessionContext({
agentName: undefined,
channelKind: undefined,
+ idempotencyKey: sessionIdempotencyKey(event.sessionId),
parentTraceContext: event.parentTraceContext,
rootSessionId: event.rootSessionId,
sessionId: event.sessionId,
@@ -197,7 +199,10 @@ export function createAgentOtelInstrumentation(
if (turn === undefined) return;
await input.stateStore.setTurn(event.sessionId, event.turnId, {
...turn,
- terminal: { error: event.error, type: event.type },
+ terminal:
+ event.type === "turn.failed"
+ ? { error: event.error, type: event.type }
+ : { type: event.type },
});
};
@@ -276,13 +281,13 @@ export function createAgentOtelInstrumentation(
if (system !== undefined) span.setAttribute("ai.prompt.system", system);
}
const state = { context: trace.setSpan(attempt.operation.context, span), span };
- getExecutionContexts(event.scope).models.set(event.id, state.context);
- getSpanStates(modelSpans, event.scope).set(event.id, state);
+ getExecutionContexts(event.scope).models.set(event.idempotencyKey, state.context);
+ getSpanStates(modelSpans, event.scope).set(event.idempotencyKey, state);
};
const onModelCallTerminal = (event: InstrumentationModelCallTerminalEvent): void => {
- executionContexts.get(event.scope)?.models.delete(event.id);
- const state = takeSpanState(modelSpans, event.scope, event.id);
+ executionContexts.get(event.scope)?.models.delete(event.idempotencyKey);
+ const state = takeSpanState(modelSpans, event.scope, event.idempotencyKey);
if (state === undefined) return;
if (event.type === "model.call.failed") {
recordError(state.span, event.error);
@@ -385,13 +390,13 @@ export function createAgentOtelInstrumentation(
span: actionSpan,
toolSpan,
};
- getExecutionContexts(event.scope).tools.set(event.id, state.context);
- getSpanStates(toolSpans, event.scope).set(event.id, state);
+ getExecutionContexts(event.scope).tools.set(event.idempotencyKey, state.context);
+ getSpanStates(toolSpans, event.scope).set(event.idempotencyKey, state);
};
const onToolCallTerminal = (event: InstrumentationToolCallTerminalEvent): void => {
- executionContexts.get(event.scope)?.tools.delete(event.id);
- const state = takeSpanState(toolSpans, event.scope, event.id);
+ executionContexts.get(event.scope)?.tools.delete(event.idempotencyKey);
+ const state = takeSpanState(toolSpans, event.scope, event.idempotencyKey);
if (state === undefined) return;
if (event.type === "tool.call.failed") {
recordError(state.toolSpan, event.error);
@@ -524,8 +529,8 @@ export function createAgentOtelInstrumentation(
const contexts = executionContexts.get(operation.scope);
const parent =
operation.type === "model.call"
- ? contexts?.models.get(operation.id)
- : contexts?.tools.get(operation.id);
+ ? contexts?.models.get(operation.idempotencyKey)
+ : contexts?.tools.get(operation.idempotencyKey);
return parent === undefined ? execute() : context.with(parent, execute);
},
};
diff --git a/packages/eve/src/tracing/agent-trace-context-store.test.ts b/packages/eve/src/tracing/agent-trace-context-store.test.ts
index 9357f1446..51c36eb09 100644
--- a/packages/eve/src/tracing/agent-trace-context-store.test.ts
+++ b/packages/eve/src/tracing/agent-trace-context-store.test.ts
@@ -53,7 +53,9 @@ describe("ContextAgentTraceStateStore", () => {
parentSpanId: "2".repeat(16),
startTimeMs: 1_700_000_000_000,
});
- expect(store.getTurn("session-1", "turn-1")?.terminal?.error).toMatchObject({
+ const terminal = store.getTurn("session-1", "turn-1")?.terminal;
+ expect(terminal?.type).toBe("turn.failed");
+ expect(terminal?.type === "turn.failed" ? terminal.error : undefined).toMatchObject({
message: "failed",
});
});
diff --git a/packages/eve/src/tracing/agent-trace-context-store.ts b/packages/eve/src/tracing/agent-trace-context-store.ts
index 16fb85431..3c6111044 100644
--- a/packages/eve/src/tracing/agent-trace-context-store.ts
+++ b/packages/eve/src/tracing/agent-trace-context-store.ts
@@ -112,10 +112,9 @@ function serializeState(state: AgentTraceContextState): unknown {
terminal:
value.terminal === undefined
? undefined
- : {
- error: serializeError(value.terminal.error),
- type: value.terminal.type,
- },
+ : value.terminal.type === "turn.failed"
+ ? { error: serializeError(value.terminal.error), type: value.terminal.type }
+ : { type: value.terminal.type },
},
]),
),
@@ -187,7 +186,7 @@ function deserializeTerminal(value: unknown): AgentTurnTraceState["terminal"] {
if (!isRecord(value) || typeof value.type !== "string") return undefined;
const type = value.type;
if (!isTurnTerminalType(type)) return undefined;
- return { error: deserializeError(value.error), type };
+ return type === "turn.failed" ? { error: deserializeError(value.error), type } : { type };
}
function serializeSpanContext(context: SpanContext): Record<string, unknown> {
diff --git a/packages/eve/src/tracing/agent-trace-state.ts b/packages/eve/src/tracing/agent-trace-state.ts
index dc571f48d..93e9f20b6 100644
--- a/packages/eve/src/tracing/agent-trace-state.ts
+++ b/packages/eve/src/tracing/agent-trace-state.ts
@@ -2,7 +2,8 @@ import type { SpanContext } from "#compiled/@opentelemetry/api/index.js";
import type {
InstrumentationParentLineage,
- InstrumentationTurnTerminalEvent,
+ InstrumentationTurnFailedEvent,
+ InstrumentationTurnSettledEvent,
} from "#harness/instrumentation-lifecycle.js";
/** Sized so an ordinary session stays one trace and only an outsized one rolls. */
@@ -24,10 +25,9 @@ export interface AgentTurnTraceState {
readonly rootSessionId: string;
readonly sequence: number;
readonly startTimeMs: number;
- readonly terminal?: {
- readonly error?: unknown;
- readonly type: InstrumentationTurnTerminalEvent["type"];
- };
+ readonly terminal?:
+ | { readonly error: unknown; readonly type: InstrumentationTurnFailedEvent["type"] }
+ | { readonly type: InstrumentationTurnSettledEvent["type"] };
}
/** Provider-owned serializable storage for durable agent trace state. */
diff --git a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts
index 34f017215..ec8755b3e 100644
--- a/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts
+++ b/packages/eve/src/tracing/local-instrumentation-runtime.scenario.test.ts
@@ -11,6 +11,11 @@ import { ContextContainer, contextStorage } from "#context/container.js";
import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";
import { listLocalTraces } from "#tracing/local-trace-reader.js";
import type { InstrumentationAttemptScope } from "#harness/instrumentation-lifecycle.js";
+import {
+ attemptIdempotencyKey,
+ sessionIdempotencyKey,
+ turnIdempotencyKey,
+} from "#harness/instrumentation-lifecycle.js";
import { installLocalInstrumentationRuntime } from "#tracing/local-instrumentation-runtime.js";
import { LocalTraceSpanProcessor } from "#tracing/local-trace-span-processor.js";
@@ -49,11 +54,13 @@ describe("local instrumentation runtime", () => {
await contextStorage.run(new ContextContainer(), async () => {
await runtime.hooks.publish({
agentName: "weather",
+ idempotencyKey: sessionIdempotencyKey("session-1"),
rootSessionId: "session-1",
sessionId: "session-1",
type: "session.started",
});
await runtime.hooks.publish({
+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),
rootSessionId: "session-1",
sequence: 0,
sessionId: "session-1",
@@ -117,14 +124,20 @@ describe("local instrumentation runtime", () => {
toolOutput: { output: { temperature: 72 }, type: "tool-result" },
},
]);
- await runtime.hooks.publish({ scope, type: "step.attempt.completed" });
await runtime.hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ scope,
+ type: "step.attempt.completed",
+ });
+ await runtime.hooks.publish({
+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),
sessionId: "session-1",
turnId: "turn-1",
type: "turn.completed",
});
// Settling the turn emits the turn span with the pre-allocated id.
await runtime.hooks.publish({
+ idempotencyKey: sessionIdempotencyKey("session-1"),
sessionId: "session-1",
turnId: "turn-1",
type: "session.waiting",