Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): preserve model-call retry telemetry (#3893)

Chad Hietalae590bd71dd04
diff --git a/.changeset/fix-model-call-retry-telemetry.md b/.changeset/fix-model-call-retry-telemetry.md
new file mode 100644
index 000000000..3ab4d1d51
--- /dev/null
+++ b/.changeset/fix-model-call-retry-telemetry.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Record provider-call retries as distinct model-call spans and terminalize failed provider attempts.
diff --git a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.integration.test.ts b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.integration.test.ts
new file mode 100644
index 000000000..7d071762e
--- /dev/null
+++ b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.integration.test.ts
@@ -0,0 +1,104 @@
+import { APICallError, generateText } from "ai";
+import { MockLanguageModelV3 } from "ai/test";
+import { describe, expect, it } from "vitest";
+
+import {
+ createInstrumentationHooks as createUnboundInstrumentationHooks,
+ modelCallIdempotencyKey,
+ type InstrumentationAttemptScope,
+ type InstrumentationModelCallStartedEvent,
+ type InstrumentationModelCallTerminalEvent,
+} from "#instrumentation/lifecycle.js";
+import { createAiSdkHookBridge } from "#instrumentation/ai-sdk-hook-bridge.js";
+
+const scope: InstrumentationAttemptScope = {
+ attemptId: "turn-1:step-0:attempt-0",
+ attemptIndex: 0,
+ sessionId: "session-1",
+ stepIndex: 0,
+ turnId: "turn-1",
+};
+
+describe("AI SDK model-call retry telemetry", () => {
+ it("restarts instrumentation when generateText retries without another call-start callback", async () => {
+ const started: InstrumentationModelCallStartedEvent[] = [];
+ const terminal: InstrumentationModelCallTerminalEvent[] = [];
+ const hooks = createUnboundInstrumentationHooks([
+ {
+ events: {
+ "model.call.completed": (event) => void terminal.push(event),
+ "model.call.failed": (event) => void terminal.push(event),
+ "model.call.started": (event) => void started.push(event),
+ },
+ name: "retry",
+ tracePolicy: () => ({ emit: true, recordInputs: true, recordOutputs: true }),
+ },
+ ]).forTrace!({
+ agentName: "test-agent",
+ audience: "unknown",
+ channel: { kind: "http" },
+ environment: "production",
+ principalType: "anonymous",
+ });
+ let providerAttempts = 0;
+ const model = new MockLanguageModelV3({
+ doGenerate: async () => {
+ providerAttempts += 1;
+ if (providerAttempts === 1) {
+ throw new APICallError({
+ isRetryable: true,
+ message: "temporarily unavailable",
+ requestBodyValues: {},
+ statusCode: 503,
+ url: "https://model.example/generate",
+ });
+ }
+ return {
+ content: [{ text: "Recovered answer.", type: "text" }],
+ finishReason: { raw: undefined, unified: "stop" },
+ usage: {
+ inputTokens: { cacheRead: 0, cacheWrite: 0, noCache: 1, total: 1 },
+ outputTokens: { reasoning: 0, text: 1, total: 1 },
+ },
+ warnings: [],
+ };
+ },
+ });
+
+ const contextKeys: string[] = [];
+ const bridge = createAiSdkHookBridge(scope, hooks, (operation, execute) => {
+ contextKeys.push(operation.idempotencyKey);
+ return execute();
+ });
+ const result = await generateText({
+ maxRetries: 1,
+ model,
+ prompt: "Answer Alice's question.",
+ telemetry: { integrations: [bridge], isEnabled: true },
+ });
+
+ expect(result.text).toBe("Recovered answer.");
+ expect(model.doGenerateCalls).toHaveLength(2);
+ expect(started.map((event) => event.idempotencyKey)).toEqual([
+ modelCallIdempotencyKey(scope, 0, 0),
+ modelCallIdempotencyKey(scope, 0, 1),
+ ]);
+ expect(contextKeys).toEqual([
+ modelCallIdempotencyKey(scope, 0, 0),
+ modelCallIdempotencyKey(scope, 0, 1),
+ ]);
+ expect(terminal).toMatchObject([
+ {
+ error: expect.any(APICallError),
+ idempotencyKey: modelCallIdempotencyKey(scope, 0, 0),
+ type: "model.call.failed",
+ },
+ {
+ finishReason: "stop",
+ idempotencyKey: modelCallIdempotencyKey(scope, 0, 1),
+ type: "model.call.completed",
+ usage: { inputTokens: 1, outputTokens: 1 },
+ },
+ ]);
+ });
+});
diff --git a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts
index c38207246..30e861648 100644
--- a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts
+++ b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts
@@ -97,7 +97,7 @@ describe("createAiSdkHookBridge", () => {
},
]);
- const id = modelCallIdempotencyKey(scope, 0);
+ const id = modelCallIdempotencyKey(scope, 0, 0);
expect(calls).toEqual([
`a:started:${id}`,
`b:started:${id}`,
@@ -106,6 +106,75 @@ describe("createAiSdkHookBridge", () => {
]);
});
+ it("gives provider retries distinct identities and terminalizes failed calls", async () => {
+ const started: InstrumentationModelCallStartedEvent[] = [];
+ const terminal: InstrumentationModelCallTerminalEvent[] = [];
+ const hooks = createInstrumentationHooks([
+ {
+ events: {
+ "model.call.completed": (event) => void terminal.push(event),
+ "model.call.failed": (event) => void terminal.push(event),
+ "model.call.started": (event) => void started.push(event),
+ },
+ name: "retry",
+ tracePolicy: contentTracePolicy,
+ },
+ ]);
+ const bridge = createAiSdkHookBridge(scope, hooks);
+ const call = {
+ callId: "call-1",
+ messages: [],
+ modelId: "model",
+ provider: "test",
+ tools: undefined,
+ };
+ await Reflect.apply(bridge.onStepStart!, bridge, [{ callId: "stream-1", stepNumber: 0 }]);
+
+ await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [call]);
+ const retryableError = new Error("provider request failed");
+ await expect(
+ bridge.executeLanguageModelCall!({
+ callId: "call-1",
+ execute: async () => {
+ throw retryableError;
+ },
+ }),
+ ).rejects.toBe(retryableError);
+
+ await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [call]);
+ await bridge.executeLanguageModelCall!({
+ callId: "call-1",
+ execute: async () => "stream",
+ });
+ await Reflect.apply(bridge.onLanguageModelCallEnd!, bridge, [
+ {
+ callId: "call-1",
+ content: [],
+ finishReason: "stop",
+ performance: { responseTimeMs: 1 },
+ responseId: "response-1",
+ usage: { inputTokens: 1, outputTokens: 1 },
+ },
+ ]);
+
+ expect(started.map((event) => event.idempotencyKey)).toEqual([
+ modelCallIdempotencyKey(scope, 0, 0),
+ modelCallIdempotencyKey(scope, 0, 1),
+ ]);
+ expect(terminal).toMatchObject([
+ {
+ error: retryableError,
+ idempotencyKey: modelCallIdempotencyKey(scope, 0, 0),
+ type: "model.call.failed",
+ },
+ {
+ finishReason: "stop",
+ idempotencyKey: modelCallIdempotencyKey(scope, 0, 1),
+ type: "model.call.completed",
+ },
+ ]);
+ });
+
it("publishes usage once when the SDK repeats a model terminal callback", async () => {
let inputTokens = 0;
const completed = vi.fn((event: InstrumentationModelCallTerminalEvent) => {
@@ -190,7 +259,7 @@ describe("createAiSdkHookBridge", () => {
await bridge.executeLanguageModelCall!({ callId: "call-1", execute: async () => "result" });
- const expected = modelCallIdempotencyKey(scope, 0);
+ const expected = modelCallIdempotencyKey(scope, 0, 0);
expect(ids).toEqual([expected, expected]);
});
@@ -225,7 +294,10 @@ describe("createAiSdkHookBridge", () => {
]);
}
- expect(keys).toEqual([modelCallIdempotencyKey(scope, 2), modelCallIdempotencyKey(scope, 2)]);
+ expect(keys).toEqual([
+ modelCallIdempotencyKey(scope, 2, 0),
+ modelCallIdempotencyKey(scope, 2, 0),
+ ]);
});
it("attaches merged runtime context to step and model started events", async () => {
@@ -502,7 +574,7 @@ describe("createAiSdkHookBridge", () => {
expect(before).toHaveBeenCalledExactlyOnceWith(
{
- idempotencyKey: modelCallIdempotencyKey(scope, 0),
+ idempotencyKey: modelCallIdempotencyKey(scope, 0, 0),
input: { instructions: "be brief", messages: [{ content: "hi", role: "user" }] },
model: { modelId: "model", provider: "test" },
scope,
@@ -534,7 +606,7 @@ describe("createAiSdkHookBridge", () => {
},
],
finishReason: "tool-calls",
- idempotencyKey: modelCallIdempotencyKey(scope, 0),
+ idempotencyKey: modelCallIdempotencyKey(scope, 0, 0),
responseModelId: "response-model",
responseId: "response-1",
scope,
diff --git a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts
index 6378a41fe..b9c6390c4 100644
--- a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts
+++ b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts
@@ -26,6 +26,8 @@ type TelemetryEvent<TKey extends keyof Telemetry> = Parameters<NonNullable<Telem
interface AttemptState {
readonly capturesInputs: boolean;
readonly capturesOutputs: boolean;
+ readonly modelCallIndexes: Map<number, number>;
+ readonly modelCallStartEvents: Map<string, ModelCallStartState>;
readonly modelKeys: Map<string, string>;
readonly runtimeContext?: Readonly<Record<string, unknown>>;
readonly scope: InstrumentationAttemptScope;
@@ -35,6 +37,11 @@ interface AttemptState {
stepNumber?: number;
}
+interface ModelCallStartState {
+ readonly event: InstrumentationModelCallStartedEvent;
+ readonly stepNumber: number;
+}
+
/** Creates one provider-neutral AI SDK bridge for one actual model attempt. */
export function createAiSdkHookBridge(
scope: InstrumentationAttemptScope,
@@ -45,6 +52,8 @@ export function createAiSdkHookBridge(
const state: AttemptState = {
capturesInputs: hooks.capturesInputs ?? hooks.capturesContent,
capturesOutputs: hooks.capturesOutputs ?? hooks.capturesContent,
+ modelCallIndexes: new Map(),
+ modelCallStartEvents: new Map(),
modelKeys: new Map(),
runtimeContext:
runtimeContext !== undefined && Object.keys(runtimeContext).length > 0
@@ -53,6 +62,11 @@ export function createAiSdkHookBridge(
scope,
toolKeys: new Map(),
};
+ const nextModelCallKey = (stepNumber: number): string => {
+ const callIndex = state.modelCallIndexes.get(stepNumber) ?? 0;
+ state.modelCallIndexes.set(stepNumber, callIndex + 1);
+ return modelCallIdempotencyKey(state.scope, stepNumber, callIndex);
+ };
return {
onStart(event) {
@@ -68,21 +82,37 @@ export function createAiSdkHookBridge(
if (started !== undefined) await hooks.publish(started);
},
async onLanguageModelCallStart(event) {
- const key = modelCallIdempotencyKey(state.scope, state.stepNumber ?? 0);
+ const stepNumber = state.stepNumber ?? 0;
+ const key = nextModelCallKey(stepNumber);
state.modelKeys.set(event.callId, key);
const started = toModelCallStarted(state, key, event);
+ state.modelCallStartEvents.set(event.callId, { event: started, stepNumber });
await hooks.publish(started);
},
- executeLanguageModelCall({ callId, execute }) {
- const key = state.modelKeys.get(callId);
- return key === undefined
- ? execute()
- : runInContext({ idempotencyKey: key, scope, type: "model.call" }, execute);
+ async executeLanguageModelCall({ callId, execute }) {
+ let key = state.modelKeys.get(callId);
+ if (key === undefined) {
+ const previousStart = state.modelCallStartEvents.get(callId);
+ if (previousStart === undefined) return execute();
+ key = nextModelCallKey(previousStart.stepNumber);
+ state.modelKeys.set(callId, key);
+ await hooks.publish(Object.freeze({ ...previousStart.event, idempotencyKey: key }));
+ }
+ try {
+ return await runInContext({ idempotencyKey: key, scope, type: "model.call" }, execute);
+ } catch (error) {
+ if (state.modelKeys.get(callId) === key) state.modelKeys.delete(callId);
+ await hooks.publish(
+ Object.freeze({ error, idempotencyKey: key, scope, type: "model.call.failed" }),
+ );
+ throw error;
+ }
},
async onLanguageModelCallEnd(event) {
const key = state.modelKeys.get(event.callId);
if (key === undefined) return;
state.modelKeys.delete(event.callId);
+ state.modelCallStartEvents.delete(event.callId);
const completed = toModelCallCompleted(state, key, event);
await hooks.publish(completed);
},
@@ -148,6 +178,7 @@ export function createAiSdkHookBridge(
);
}
state.modelKeys.clear();
+ state.modelCallStartEvents.clear();
state.toolKeys.clear();
await Promise.all(pending);
}
diff --git a/packages/eve/src/instrumentation/eve-channel-audience-production.scenario.test.ts b/packages/eve/src/instrumentation/eve-channel-audience-production.scenario.test.ts
index cb615daf7..b5d991ae3 100644
--- a/packages/eve/src/instrumentation/eve-channel-audience-production.scenario.test.ts
+++ b/packages/eve/src/instrumentation/eve-channel-audience-production.scenario.test.ts
@@ -82,6 +82,7 @@ describe("eve channel production audience", () => {
turnId: "turn-1",
},
0,
+ 0,
),
input: { instructions: "private instructions", messages: [] },
model: { modelId: "test", provider: "test" },
diff --git a/packages/eve/src/instrumentation/lifecycle.test.ts b/packages/eve/src/instrumentation/lifecycle.test.ts
index dd47b4a3c..7a2ce41e9 100644
--- a/packages/eve/src/instrumentation/lifecycle.test.ts
+++ b/packages/eve/src/instrumentation/lifecycle.test.ts
@@ -86,13 +86,16 @@ describe("instrumentation idempotency keys", () => {
});
it("derives model identity without an AI SDK call ID", () => {
- expect(modelCallIdempotencyKey(scope, 2)).toBe("model:session-1:turn-1:0:0:2");
+ expect(modelCallIdempotencyKey(scope, 2, 0)).toBe("model:session-1:turn-1:0:0:2:0");
+ expect(modelCallIdempotencyKey(scope, 2, 0)).not.toBe(modelCallIdempotencyKey(scope, 2, 1));
});
it("separates model attempts and SDK tool calls", () => {
const retry = { ...scope, attemptId: "session-1:turn-1:0:1", attemptIndex: 1 };
expect(attemptIdempotencyKey(scope)).not.toBe(attemptIdempotencyKey(retry));
- expect(modelCallIdempotencyKey(scope, 0)).not.toBe(toolCallIdempotencyKey(scope, "call-1", 0));
+ expect(modelCallIdempotencyKey(scope, 0, 0)).not.toBe(
+ toolCallIdempotencyKey(scope, "call-1", 0),
+ );
});
});
@@ -207,7 +210,7 @@ describe("provider state lifecycle", () => {
});
it("releases unterminated model state when its attempt ends", async () => {
- const modelKey = modelCallIdempotencyKey(scope, 0);
+ const modelKey = modelCallIdempotencyKey(scope, 0, 0);
const hooks = createInstrumentationHooks([
{
events: { "model.call.started": (_event, ctx) => ctx.state.set("open") },
@@ -232,7 +235,7 @@ describe("provider state lifecycle", () => {
});
it("releases attempt-owned state after its provider is removed", async () => {
- const modelKey = modelCallIdempotencyKey(scope, 0);
+ const modelKey = modelCallIdempotencyKey(scope, 0, 0);
const starts = createInstrumentationHooks([
{
events: { "model.call.started": (_event, ctx) => ctx.state.set("open") },
@@ -582,7 +585,7 @@ describe("provider dispatch groups", () => {
const observed: unknown[] = [];
const mutableScope = { ...scope };
const event: InstrumentationModelCallStartedEvent = {
- idempotencyKey: modelCallIdempotencyKey(mutableScope, 0),
+ idempotencyKey: modelCallIdempotencyKey(mutableScope, 0, 0),
input: { messages: [{ content: "private", role: "user" }] },
model: { modelId: "model", provider: "test" },
scope: mutableScope,
@@ -639,7 +642,7 @@ describe("provider dispatch groups", () => {
await contextStorage.run(new ContextContainer(), async () => {
await hooks.publish({
- idempotencyKey: modelCallIdempotencyKey(sharedScope, 0),
+ idempotencyKey: modelCallIdempotencyKey(sharedScope, 0, 0),
input: { messages: [] },
model: { modelId: "model", provider: "test" },
scope: sharedScope,
@@ -648,7 +651,7 @@ describe("provider dispatch groups", () => {
await hooks.publish({
content: [],
finishReason: "stop",
- idempotencyKey: modelCallIdempotencyKey(sharedScope, 0),
+ idempotencyKey: modelCallIdempotencyKey(sharedScope, 0, 0),
scope: sharedScope,
type: "model.call.completed",
usage: {},
diff --git a/packages/eve/src/instrumentation/lifecycle.ts b/packages/eve/src/instrumentation/lifecycle.ts
index 3de2d6cab..2c2f31ff0 100644
--- a/packages/eve/src/instrumentation/lifecycle.ts
+++ b/packages/eve/src/instrumentation/lifecycle.ts
@@ -121,12 +121,13 @@ export function attemptIdempotencyKey(scope: InstrumentationAttemptScope): strin
return `step:${scope.attemptId}`;
}
-/** One model call occurs per AI SDK step within an eve attempt. */
+/** Identifies one provider call, including retries, within an AI SDK step. */
export function modelCallIdempotencyKey(
scope: InstrumentationAttemptScope,
stepNumber: number,
+ callIndex: number,
): string {
- return `model:${scope.attemptId}:${String(stepNumber)}`;
+ return `model:${scope.attemptId}:${String(stepNumber)}:${String(callIndex)}`;
}
export function toolCallIdempotencyKey(
diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.ts
index 7395c7d46..2dda5c680 100644
--- a/packages/eve/src/tracing/agent-otel-provider.test.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts
@@ -3131,7 +3131,7 @@ describe("createAgentOtelInstrumentation", () => {
scope,
type: "step.attempt.started",
});
- const idempotencyKey = modelCallIdempotencyKey(scope, 0);
+ const idempotencyKey = modelCallIdempotencyKey(scope, 0, 0);
await runtime.hooks.publish({
idempotencyKey,
input: { messages: [] },
diff --git a/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts b/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts
index f52d1a93c..19ab071b4 100644
--- a/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts
+++ b/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts
@@ -417,7 +417,7 @@ describe("exported agent telemetry contract", () => {
scope: childScope,
type: "step.attempt.started",
});
- const modelKey = modelCallIdempotencyKey(childScope, 0);
+ const modelKey = modelCallIdempotencyKey(childScope, 0, 0);
await childHooks.publish({
idempotencyKey: modelKey,
model: { modelId: "test", provider: "test" },