The Open Framework for Building Agents
fix(eve): preserve model-call retry telemetry (#3893)
Chad Hietalae590bd71dd04
diff --git a/.changeset/fix-model-call-retry-telemetry.md b/.changeset/fix-model-call-retry-telemetry.mdnew file mode 100644index 000000000..3ab4d1d51--- /dev/null+++ b/.changeset/fix-model-call-retry-telemetry.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Record provider-call retries as distinct model-call spans and terminalize failed provider attempts.diff --git a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.integration.test.ts b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.integration.test.tsnew file mode 100644index 000000000..7d071762e--- /dev/null+++ b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.integration.test.ts@@ -0,0 +1,104 @@+import { APICallError, generateText } from "ai";+import { MockLanguageModelV3 } from "ai/test";+import { describe, expect, it } from "vitest";++import {+ createInstrumentationHooks as createUnboundInstrumentationHooks,+ modelCallIdempotencyKey,+ type InstrumentationAttemptScope,+ type InstrumentationModelCallStartedEvent,+ type InstrumentationModelCallTerminalEvent,+} from "#instrumentation/lifecycle.js";+import { createAiSdkHookBridge } from "#instrumentation/ai-sdk-hook-bridge.js";++const scope: InstrumentationAttemptScope = {+ attemptId: "turn-1:step-0:attempt-0",+ attemptIndex: 0,+ sessionId: "session-1",+ stepIndex: 0,+ turnId: "turn-1",+};++describe("AI SDK model-call retry telemetry", () => {+ it("restarts instrumentation when generateText retries without another call-start callback", async () => {+ const started: InstrumentationModelCallStartedEvent[] = [];+ const terminal: InstrumentationModelCallTerminalEvent[] = [];+ const hooks = createUnboundInstrumentationHooks([+ {+ events: {+ "model.call.completed": (event) => void terminal.push(event),+ "model.call.failed": (event) => void terminal.push(event),+ "model.call.started": (event) => void started.push(event),+ },+ name: "retry",+ tracePolicy: () => ({ emit: true, recordInputs: true, recordOutputs: true }),+ },+ ]).forTrace!({+ agentName: "test-agent",+ audience: "unknown",+ channel: { kind: "http" },+ environment: "production",+ principalType: "anonymous",+ });+ let providerAttempts = 0;+ const model = new MockLanguageModelV3({+ doGenerate: async () => {+ providerAttempts += 1;+ if (providerAttempts === 1) {+ throw new APICallError({+ isRetryable: true,+ message: "temporarily unavailable",+ requestBodyValues: {},+ statusCode: 503,+ url: "https://model.example/generate",+ });+ }+ return {+ content: [{ text: "Recovered answer.", type: "text" }],+ finishReason: { raw: undefined, unified: "stop" },+ usage: {+ inputTokens: { cacheRead: 0, cacheWrite: 0, noCache: 1, total: 1 },+ outputTokens: { reasoning: 0, text: 1, total: 1 },+ },+ warnings: [],+ };+ },+ });++ const contextKeys: string[] = [];+ const bridge = createAiSdkHookBridge(scope, hooks, (operation, execute) => {+ contextKeys.push(operation.idempotencyKey);+ return execute();+ });+ const result = await generateText({+ maxRetries: 1,+ model,+ prompt: "Answer Alice's question.",+ telemetry: { integrations: [bridge], isEnabled: true },+ });++ expect(result.text).toBe("Recovered answer.");+ expect(model.doGenerateCalls).toHaveLength(2);+ expect(started.map((event) => event.idempotencyKey)).toEqual([+ modelCallIdempotencyKey(scope, 0, 0),+ modelCallIdempotencyKey(scope, 0, 1),+ ]);+ expect(contextKeys).toEqual([+ modelCallIdempotencyKey(scope, 0, 0),+ modelCallIdempotencyKey(scope, 0, 1),+ ]);+ expect(terminal).toMatchObject([+ {+ error: expect.any(APICallError),+ idempotencyKey: modelCallIdempotencyKey(scope, 0, 0),+ type: "model.call.failed",+ },+ {+ finishReason: "stop",+ idempotencyKey: modelCallIdempotencyKey(scope, 0, 1),+ type: "model.call.completed",+ usage: { inputTokens: 1, outputTokens: 1 },+ },+ ]);+ });+});diff --git a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.tsindex c38207246..30e861648 100644--- a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts+++ b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.test.ts@@ -97,7 +97,7 @@ describe("createAiSdkHookBridge", () => {},]);- const id = modelCallIdempotencyKey(scope, 0);+ const id = modelCallIdempotencyKey(scope, 0, 0);expect(calls).toEqual([`a:started:${id}`,`b:started:${id}`,@@ -106,6 +106,75 @@ describe("createAiSdkHookBridge", () => {]);});+ it("gives provider retries distinct identities and terminalizes failed calls", async () => {+ const started: InstrumentationModelCallStartedEvent[] = [];+ const terminal: InstrumentationModelCallTerminalEvent[] = [];+ const hooks = createInstrumentationHooks([+ {+ events: {+ "model.call.completed": (event) => void terminal.push(event),+ "model.call.failed": (event) => void terminal.push(event),+ "model.call.started": (event) => void started.push(event),+ },+ name: "retry",+ tracePolicy: contentTracePolicy,+ },+ ]);+ const bridge = createAiSdkHookBridge(scope, hooks);+ const call = {+ callId: "call-1",+ messages: [],+ modelId: "model",+ provider: "test",+ tools: undefined,+ };+ await Reflect.apply(bridge.onStepStart!, bridge, [{ callId: "stream-1", stepNumber: 0 }]);++ await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [call]);+ const retryableError = new Error("provider request failed");+ await expect(+ bridge.executeLanguageModelCall!({+ callId: "call-1",+ execute: async () => {+ throw retryableError;+ },+ }),+ ).rejects.toBe(retryableError);++ await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [call]);+ await bridge.executeLanguageModelCall!({+ callId: "call-1",+ execute: async () => "stream",+ });+ await Reflect.apply(bridge.onLanguageModelCallEnd!, bridge, [+ {+ callId: "call-1",+ content: [],+ finishReason: "stop",+ performance: { responseTimeMs: 1 },+ responseId: "response-1",+ usage: { inputTokens: 1, outputTokens: 1 },+ },+ ]);++ expect(started.map((event) => event.idempotencyKey)).toEqual([+ modelCallIdempotencyKey(scope, 0, 0),+ modelCallIdempotencyKey(scope, 0, 1),+ ]);+ expect(terminal).toMatchObject([+ {+ error: retryableError,+ idempotencyKey: modelCallIdempotencyKey(scope, 0, 0),+ type: "model.call.failed",+ },+ {+ finishReason: "stop",+ idempotencyKey: modelCallIdempotencyKey(scope, 0, 1),+ type: "model.call.completed",+ },+ ]);+ });+it("publishes usage once when the SDK repeats a model terminal callback", async () => {let inputTokens = 0;const completed = vi.fn((event: InstrumentationModelCallTerminalEvent) => {@@ -190,7 +259,7 @@ describe("createAiSdkHookBridge", () => {await bridge.executeLanguageModelCall!({ callId: "call-1", execute: async () => "result" });- const expected = modelCallIdempotencyKey(scope, 0);+ const expected = modelCallIdempotencyKey(scope, 0, 0);expect(ids).toEqual([expected, expected]);});@@ -225,7 +294,10 @@ describe("createAiSdkHookBridge", () => {]);}- expect(keys).toEqual([modelCallIdempotencyKey(scope, 2), modelCallIdempotencyKey(scope, 2)]);+ expect(keys).toEqual([+ modelCallIdempotencyKey(scope, 2, 0),+ modelCallIdempotencyKey(scope, 2, 0),+ ]);});it("attaches merged runtime context to step and model started events", async () => {@@ -502,7 +574,7 @@ describe("createAiSdkHookBridge", () => {expect(before).toHaveBeenCalledExactlyOnceWith({- idempotencyKey: modelCallIdempotencyKey(scope, 0),+ idempotencyKey: modelCallIdempotencyKey(scope, 0, 0),input: { instructions: "be brief", messages: [{ content: "hi", role: "user" }] },model: { modelId: "model", provider: "test" },scope,@@ -534,7 +606,7 @@ describe("createAiSdkHookBridge", () => {},],finishReason: "tool-calls",- idempotencyKey: modelCallIdempotencyKey(scope, 0),+ idempotencyKey: modelCallIdempotencyKey(scope, 0, 0),responseModelId: "response-model",responseId: "response-1",scope,diff --git a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.tsindex 6378a41fe..b9c6390c4 100644--- a/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts+++ b/packages/eve/src/instrumentation/ai-sdk-hook-bridge.ts@@ -26,6 +26,8 @@ type TelemetryEvent<TKey extends keyof Telemetry> = Parameters<NonNullable<Teleminterface AttemptState {readonly capturesInputs: boolean;readonly capturesOutputs: boolean;+ readonly modelCallIndexes: Map<number, number>;+ readonly modelCallStartEvents: Map<string, ModelCallStartState>;readonly modelKeys: Map<string, string>;readonly runtimeContext?: Readonly<Record<string, unknown>>;readonly scope: InstrumentationAttemptScope;@@ -35,6 +37,11 @@ interface AttemptState {stepNumber?: number;}+interface ModelCallStartState {+ readonly event: InstrumentationModelCallStartedEvent;+ readonly stepNumber: number;+}+/** Creates one provider-neutral AI SDK bridge for one actual model attempt. */export function createAiSdkHookBridge(scope: InstrumentationAttemptScope,@@ -45,6 +52,8 @@ export function createAiSdkHookBridge(const state: AttemptState = {capturesInputs: hooks.capturesInputs ?? hooks.capturesContent,capturesOutputs: hooks.capturesOutputs ?? hooks.capturesContent,+ modelCallIndexes: new Map(),+ modelCallStartEvents: new Map(),modelKeys: new Map(),runtimeContext:runtimeContext !== undefined && Object.keys(runtimeContext).length > 0@@ -53,6 +62,11 @@ export function createAiSdkHookBridge(scope,toolKeys: new Map(),};+ const nextModelCallKey = (stepNumber: number): string => {+ const callIndex = state.modelCallIndexes.get(stepNumber) ?? 0;+ state.modelCallIndexes.set(stepNumber, callIndex + 1);+ return modelCallIdempotencyKey(state.scope, stepNumber, callIndex);+ };return {onStart(event) {@@ -68,21 +82,37 @@ export function createAiSdkHookBridge(if (started !== undefined) await hooks.publish(started);},async onLanguageModelCallStart(event) {- const key = modelCallIdempotencyKey(state.scope, state.stepNumber ?? 0);+ const stepNumber = state.stepNumber ?? 0;+ const key = nextModelCallKey(stepNumber);state.modelKeys.set(event.callId, key);const started = toModelCallStarted(state, key, event);+ state.modelCallStartEvents.set(event.callId, { event: started, stepNumber });await hooks.publish(started);},- executeLanguageModelCall({ callId, execute }) {- const key = state.modelKeys.get(callId);- return key === undefined- ? execute()- : runInContext({ idempotencyKey: key, scope, type: "model.call" }, execute);+ async executeLanguageModelCall({ callId, execute }) {+ let key = state.modelKeys.get(callId);+ if (key === undefined) {+ const previousStart = state.modelCallStartEvents.get(callId);+ if (previousStart === undefined) return execute();+ key = nextModelCallKey(previousStart.stepNumber);+ state.modelKeys.set(callId, key);+ await hooks.publish(Object.freeze({ ...previousStart.event, idempotencyKey: key }));+ }+ try {+ return await runInContext({ idempotencyKey: key, scope, type: "model.call" }, execute);+ } catch (error) {+ if (state.modelKeys.get(callId) === key) state.modelKeys.delete(callId);+ await hooks.publish(+ Object.freeze({ error, idempotencyKey: key, scope, type: "model.call.failed" }),+ );+ throw error;+ }},async onLanguageModelCallEnd(event) {const key = state.modelKeys.get(event.callId);if (key === undefined) return;state.modelKeys.delete(event.callId);+ state.modelCallStartEvents.delete(event.callId);const completed = toModelCallCompleted(state, key, event);await hooks.publish(completed);},@@ -148,6 +178,7 @@ export function createAiSdkHookBridge();}state.modelKeys.clear();+ state.modelCallStartEvents.clear();state.toolKeys.clear();await Promise.all(pending);}diff --git a/packages/eve/src/instrumentation/eve-channel-audience-production.scenario.test.ts b/packages/eve/src/instrumentation/eve-channel-audience-production.scenario.test.tsindex cb615daf7..b5d991ae3 100644--- a/packages/eve/src/instrumentation/eve-channel-audience-production.scenario.test.ts+++ b/packages/eve/src/instrumentation/eve-channel-audience-production.scenario.test.ts@@ -82,6 +82,7 @@ describe("eve channel production audience", () => {turnId: "turn-1",},0,+ 0,),input: { instructions: "private instructions", messages: [] },model: { modelId: "test", provider: "test" },diff --git a/packages/eve/src/instrumentation/lifecycle.test.ts b/packages/eve/src/instrumentation/lifecycle.test.tsindex dd47b4a3c..7a2ce41e9 100644--- a/packages/eve/src/instrumentation/lifecycle.test.ts+++ b/packages/eve/src/instrumentation/lifecycle.test.ts@@ -86,13 +86,16 @@ describe("instrumentation idempotency keys", () => {});it("derives model identity without an AI SDK call ID", () => {- expect(modelCallIdempotencyKey(scope, 2)).toBe("model:session-1:turn-1:0:0:2");+ expect(modelCallIdempotencyKey(scope, 2, 0)).toBe("model:session-1:turn-1:0:0:2:0");+ expect(modelCallIdempotencyKey(scope, 2, 0)).not.toBe(modelCallIdempotencyKey(scope, 2, 1));});it("separates model attempts and SDK tool calls", () => {const retry = { ...scope, attemptId: "session-1:turn-1:0:1", attemptIndex: 1 };expect(attemptIdempotencyKey(scope)).not.toBe(attemptIdempotencyKey(retry));- expect(modelCallIdempotencyKey(scope, 0)).not.toBe(toolCallIdempotencyKey(scope, "call-1", 0));+ expect(modelCallIdempotencyKey(scope, 0, 0)).not.toBe(+ toolCallIdempotencyKey(scope, "call-1", 0),+ );});});@@ -207,7 +210,7 @@ describe("provider state lifecycle", () => {});it("releases unterminated model state when its attempt ends", async () => {- const modelKey = modelCallIdempotencyKey(scope, 0);+ const modelKey = modelCallIdempotencyKey(scope, 0, 0);const hooks = createInstrumentationHooks([{events: { "model.call.started": (_event, ctx) => ctx.state.set("open") },@@ -232,7 +235,7 @@ describe("provider state lifecycle", () => {});it("releases attempt-owned state after its provider is removed", async () => {- const modelKey = modelCallIdempotencyKey(scope, 0);+ const modelKey = modelCallIdempotencyKey(scope, 0, 0);const starts = createInstrumentationHooks([{events: { "model.call.started": (_event, ctx) => ctx.state.set("open") },@@ -582,7 +585,7 @@ describe("provider dispatch groups", () => {const observed: unknown[] = [];const mutableScope = { ...scope };const event: InstrumentationModelCallStartedEvent = {- idempotencyKey: modelCallIdempotencyKey(mutableScope, 0),+ idempotencyKey: modelCallIdempotencyKey(mutableScope, 0, 0),input: { messages: [{ content: "private", role: "user" }] },model: { modelId: "model", provider: "test" },scope: mutableScope,@@ -639,7 +642,7 @@ describe("provider dispatch groups", () => {await contextStorage.run(new ContextContainer(), async () => {await hooks.publish({- idempotencyKey: modelCallIdempotencyKey(sharedScope, 0),+ idempotencyKey: modelCallIdempotencyKey(sharedScope, 0, 0),input: { messages: [] },model: { modelId: "model", provider: "test" },scope: sharedScope,@@ -648,7 +651,7 @@ describe("provider dispatch groups", () => {await hooks.publish({content: [],finishReason: "stop",- idempotencyKey: modelCallIdempotencyKey(sharedScope, 0),+ idempotencyKey: modelCallIdempotencyKey(sharedScope, 0, 0),scope: sharedScope,type: "model.call.completed",usage: {},diff --git a/packages/eve/src/instrumentation/lifecycle.ts b/packages/eve/src/instrumentation/lifecycle.tsindex 3de2d6cab..2c2f31ff0 100644--- a/packages/eve/src/instrumentation/lifecycle.ts+++ b/packages/eve/src/instrumentation/lifecycle.ts@@ -121,12 +121,13 @@ export function attemptIdempotencyKey(scope: InstrumentationAttemptScope): strinreturn `step:${scope.attemptId}`;}-/** One model call occurs per AI SDK step within an eve attempt. */+/** Identifies one provider call, including retries, within an AI SDK step. */export function modelCallIdempotencyKey(scope: InstrumentationAttemptScope,stepNumber: number,+ callIndex: number,): string {- return `model:${scope.attemptId}:${String(stepNumber)}`;+ return `model:${scope.attemptId}:${String(stepNumber)}:${String(callIndex)}`;}export function toolCallIdempotencyKey(diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex 7395c7d46..2dda5c680 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -3131,7 +3131,7 @@ describe("createAgentOtelInstrumentation", () => {scope,type: "step.attempt.started",});- const idempotencyKey = modelCallIdempotencyKey(scope, 0);+ const idempotencyKey = modelCallIdempotencyKey(scope, 0, 0);await runtime.hooks.publish({idempotencyKey,input: { messages: [] },diff --git a/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts b/packages/eve/src/tracing/agent-telemetry-contract.integration.test.tsindex f52d1a93c..19ab071b4 100644--- a/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts+++ b/packages/eve/src/tracing/agent-telemetry-contract.integration.test.ts@@ -417,7 +417,7 @@ describe("exported agent telemetry contract", () => {scope: childScope,type: "step.attempt.started",});- const modelKey = modelCallIdempotencyKey(childScope, 0);+ const modelKey = modelCallIdempotencyKey(childScope, 0, 0);await childHooks.publish({idempotencyKey: modelKey,model: { modelId: "test", provider: "test" },