The Open Framework for Building Agents
feat(eve): persist provider state and bound instrumentation handlers (#1860)
Chad Hietala1528fda2da4a
diff --git a/.changeset/provider-state-deadlines.md b/.changeset/provider-state-deadlines.mdnew file mode 100644index 000000000..92fb99d2b--- /dev/null+++ b/.changeset/provider-state-deadlines.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Add durable operation-scoped state to instrumentation handlers, isolated by provider and automatically released at terminal boundaries. Bound each handler and persist start-handler abandonment so a stalled provider cannot block the bus or later receive a mismatched terminal.diff --git a/packages/eve/src/execution/workflow-steps.ts b/packages/eve/src/execution/workflow-steps.tsindex 95509d969..a553d0483 100644--- a/packages/eve/src/execution/workflow-steps.ts+++ b/packages/eve/src/execution/workflow-steps.ts@@ -29,6 +29,7 @@ import {isHarnessBetweenTurns,setHarnessEmissionState,} from "#harness/emission.js";+import { preserveSerializedInstrumentationState } from "#harness/instrumentation-state.js";import { preserveSerializedAgentTraceState } from "#tracing/agent-trace-context-store.js";import { matchAuthorizationCallbacks } from "#execution/authorization-callback-match.js";import { readTurnSleepDurationMs } from "#harness/turn-sleep.js";@@ -447,11 +448,15 @@ export async function turnStep(rawInput: TurnStepInput): Promise<DurableStepResu} catch (error) {if (!isTurnCancellation(error)) throw error;writer.releaseLock();+ // Both carve-outs exist because the cancellation epilogue still publishes+ // `turn.cancelled` downstream, and the open spans and provider state it+ // needs to close were written by the step being discarded.+ const interrupted = serializeContext(ctx);return {action: "cancelled",- serializedContext: preserveSerializedAgentTraceState(- input.serializedContext,- serializeContext(ctx),+ serializedContext: preserveSerializedInstrumentationState(+ preserveSerializedAgentTraceState(input.serializedContext, interrupted),+ interrupted,),sessionState: input.sessionState,};diff --git a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts b/packages/eve/src/harness/ai-sdk-hook-bridge.test.tsindex fa647500f..0a51032d6 100644--- a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts+++ b/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts@@ -38,6 +38,7 @@ describe("createAiSdkHookBridge", () => {);},},+ name,};};const hooks = createInstrumentationHooks([provider("a"), provider("b")]);@@ -221,6 +222,7 @@ describe("createAiSdkHookBridge", () => {expect(after).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ error, type: "model.call.failed" }),+ expect.anything(),);});@@ -236,6 +238,7 @@ describe("createAiSdkHookBridge", () => {expect(after).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ error: reason, type: "tool.call.failed" }),+ expect.anything(),);});@@ -265,8 +268,8 @@ describe("createAiSdkHookBridge", () => {scope,type: "step.attempt.started",};- expect(mutator).toHaveBeenCalledExactlyOnceWith(expected);- expect(started).toHaveBeenCalledExactlyOnceWith(expected);+ expect(mutator).toHaveBeenCalledExactlyOnceWith(expected, expect.anything());+ expect(started).toHaveBeenCalledExactlyOnceWith(expected, expect.anything());});it("projects the model call callbacks onto eve fields only", async () => {@@ -333,45 +336,51 @@ describe("createAiSdkHookBridge", () => {},]);- expect(before).toHaveBeenCalledExactlyOnceWith({- idempotencyKey: `model:${scope.attemptId}:0`,- input: { instructions: "be brief", messages: [{ content: "hi", role: "user" }] },- model: { modelId: "model", provider: "test" },- scope,- type: "model.call.started",- });+ expect(before).toHaveBeenCalledExactlyOnceWith(+ {+ idempotencyKey: `model:${scope.attemptId}:0`,+ input: { instructions: "be brief", messages: [{ content: "hi", role: "user" }] },+ model: { modelId: "model", provider: "test" },+ scope,+ type: "model.call.started",+ },+ expect.anything(),+ );// An unrecognized part kind is dropped rather than forwarded, so widening// InstrumentationContentPart is what makes a new kind reachable.- expect(after).toHaveBeenCalledExactlyOnceWith({- content: [- { text: "thinking", type: "reasoning" },- { text: "hello", type: "text" },- { callId: "tool-1", input: { a: 1 }, toolName: "search", type: "tool-call" },- {- callId: "tool-1",- input: { a: 1 },- output: "ok",- toolName: "search",- type: "tool-result",- },- {- callId: "tool-2",- error: "boom",- input: { a: 2 },- toolName: "search",- type: "tool-error",+ expect(after).toHaveBeenCalledExactlyOnceWith(+ {+ content: [+ { text: "thinking", type: "reasoning" },+ { text: "hello", type: "text" },+ { callId: "tool-1", input: { a: 1 }, toolName: "search", type: "tool-call" },+ {+ callId: "tool-1",+ input: { a: 1 },+ output: "ok",+ toolName: "search",+ type: "tool-result",+ },+ {+ callId: "tool-2",+ error: "boom",+ input: { a: 2 },+ toolName: "search",+ type: "tool-error",+ },+ ],+ finishReason: "tool-calls",+ idempotencyKey: `model:${scope.attemptId}:0`,+ scope,+ type: "model.call.completed",+ usage: {+ inputTokenDetails: { cacheReadTokens: 3, cacheWriteTokens: 4 },+ inputTokens: 1,+ outputTokens: 2,},- ],- finishReason: "tool-calls",- idempotencyKey: `model:${scope.attemptId}:0`,- scope,- type: "model.call.completed",- usage: {- inputTokenDetails: { cacheReadTokens: 3, cacheWriteTokens: 4 },- inputTokens: 1,- outputTokens: 2,},- });+ expect.anything(),+ );});it.each([@@ -405,20 +414,26 @@ describe("createAiSdkHookBridge", () => {{ callId: "call-1", toolCall, toolExecutionMs: 1, toolOutput },]);- expect(before).toHaveBeenCalledExactlyOnceWith({- callId: "tool-1",- idempotencyKey: `tool:${scope.attemptId}:tool-1:0`,- input: { q: "eve" },- scope,- toolName: "search",- type: "tool.call.started",- });- expect(after).toHaveBeenCalledExactlyOnceWith({- idempotencyKey: `tool:${scope.attemptId}:tool-1:0`,- output: expected,- scope,- type: "tool.call.completed",- });+ expect(before).toHaveBeenCalledExactlyOnceWith(+ {+ callId: "tool-1",+ idempotencyKey: `tool:${scope.attemptId}:tool-1:0`,+ input: { q: "eve" },+ scope,+ toolName: "search",+ type: "tool.call.started",+ },+ expect.anything(),+ );+ expect(after).toHaveBeenCalledExactlyOnceWith(+ {+ idempotencyKey: `tool:${scope.attemptId}:tool-1:0`,+ output: expected,+ scope,+ type: "tool.call.completed",+ },+ expect.anything(),+ );},);@@ -433,6 +448,7 @@ describe("createAiSdkHookBridge", () => {},"model.call.started": (event) => void own.set(event.idempotencyKey, `${name}-state`),},+ name,};};const hooks = createInstrumentationHooks([provider("a"), provider("b")]);diff --git a/packages/eve/src/harness/instrumentation-dispatch.ts b/packages/eve/src/harness/instrumentation-dispatch.tsnew file mode 100644index 000000000..67d43f4aa--- /dev/null+++ b/packages/eve/src/harness/instrumentation-dispatch.ts@@ -0,0 +1,207 @@+import {+ abandonInstrumentationState,+ instrumentationStateSlot,+ isInstrumentationStateAbandoned,+ releaseAllInstrumentationAttemptState,+ releaseAllInstrumentationState,+} from "#harness/instrumentation-state.js";+import { createLogger, formatError } from "#internal/logging.js";++import type {+ CreateInstrumentationHooksOptions,+ InstrumentationDispatchGroups,+ InstrumentationEvent,+ InstrumentationEventHandler,+ InstrumentationHooks,+ InstrumentationHooksInput,+ InstrumentationProviderDefinition,+} from "#harness/instrumentation-lifecycle.js";++const log = createLogger("harness.instrumentation-dispatch");+const DEFAULT_HANDLER_TIMEOUT_MS = 5_000;++export function createInstrumentationDispatcher(+ input: InstrumentationHooksInput,+ options: CreateInstrumentationHooksOptions,+): InstrumentationHooks {+ const handlerTimeoutMs = options.handlerTimeoutMs ?? DEFAULT_HANDLER_TIMEOUT_MS;+ const groups = normalizeDispatchGroups(input);+ const snapshots = new WeakMap<object, unknown>();++ const publish = async (event: InstrumentationEvent): Promise<void> => {+ const snapshot = snapshotInstrumentationEvent(event, snapshots);+ try {+ try {+ for (const provider of groups.serialBefore) {+ await dispatchToProvider(provider, snapshot, handlerTimeoutMs);+ }++ if (groups.parallel.length === 1) {+ await dispatchToProvider(groups.parallel[0]!, snapshot, handlerTimeoutMs);+ } else if (groups.parallel.length > 1) {+ const results = await Promise.allSettled(+ groups.parallel.map((provider) =>+ dispatchToProvider(provider, snapshot, handlerTimeoutMs),+ ),+ );+ const rejected = results.find(+ (result): result is PromiseRejectedResult => result.status === "rejected",+ );+ if (rejected !== undefined) throw rejected.reason;+ }+ } finally {+ for (const provider of groups.serialAfter) {+ await dispatchToProvider(provider, snapshot, handlerTimeoutMs);+ }+ }+ } finally {+ releaseTerminalState(snapshot);+ }+ };++ return { publish };+}++function snapshotInstrumentationEvent(+ event: InstrumentationEvent,+ snapshots: WeakMap<object, unknown>,+): InstrumentationEvent {+ return snapshotPlainValue(event, snapshots) as InstrumentationEvent;+}++function snapshotPlainValue(value: unknown, seen: WeakMap<object, unknown>): unknown {+ if (Array.isArray(value)) {+ const existing = seen.get(value);+ if (existing !== undefined) return existing;+ const copy: unknown[] = [];+ seen.set(value, copy);+ for (const entry of value) copy.push(snapshotPlainValue(entry, seen));+ return Object.freeze(copy);+ }++ if (typeof value !== "object" || value === null) return value;+ let prototype: object | null;+ try {+ prototype = Object.getPrototypeOf(value) as object | null;+ } catch {+ return value;+ }+ if (prototype !== Object.prototype && prototype !== null) return value;++ const existing = seen.get(value);+ if (existing !== undefined) return existing;+ const copy: Record<string, unknown> = {};+ seen.set(value, copy);+ for (const [key, entry] of Object.entries(value)) {+ copy[key] = snapshotPlainValue(entry, seen);+ }+ return Object.freeze(copy);+}++function normalizeDispatchGroups(+ input: InstrumentationHooksInput,+): Required<InstrumentationDispatchGroups> {+ if (!Array.isArray(input)) {+ const groups = input as InstrumentationDispatchGroups;+ return {+ parallel: groups.parallel ?? [],+ serialAfter: groups.serialAfter ?? [],+ serialBefore: groups.serialBefore ?? [],+ };+ }++ return {+ parallel: [],+ serialAfter: [],+ serialBefore: input.map((provider, index) => ({+ ...provider,+ name: provider.name ?? `provider-${String(index)}`,+ })),+ };+}++async function dispatchToProvider(+ provider: InstrumentationProviderDefinition,+ event: InstrumentationEvent,+ handlerTimeoutMs: number,+): Promise<void> {+ const startedBoundary = event.type.endsWith(".started") || event.type === "input.requested";+ const attemptId = stateAttemptId(event);+ const providerName = provider.name;+ if (isInstrumentationStateAbandoned(providerName, event.idempotencyKey)) return;+ const handler = provider.events?.[event.type];+ if (handler === undefined) return;+ const state = instrumentationStateSlot(providerName, event.idempotencyKey, attemptId);+ try {+ const settled = await withTimeout(+ () => (handler as InstrumentationEventHandler<InstrumentationEvent>)(event, { state }),+ handlerTimeoutMs,+ () => {+ state.revoke();+ if (startedBoundary)+ abandonInstrumentationState(providerName, event.idempotencyKey, attemptId);+ },+ );+ if (!settled) {+ log.warn("instrumentation provider timed out", {+ boundary: event.type,+ provider: providerName,+ timeoutMs: handlerTimeoutMs,+ });+ }+ } catch (error) {+ log.warn("instrumentation provider failed", {+ boundary: event.type,+ error: formatError(error),+ provider: providerName,+ });+ } finally {+ state.revoke();+ }+}++async function withTimeout(+ run: () => void | PromiseLike<void>,+ timeoutMs: number,+ onTimeout: () => void,+): Promise<boolean> {+ let timer: ReturnType<typeof setTimeout> | undefined;+ try {+ return await Promise.race([+ Promise.resolve(run()).then(() => true),+ new Promise<boolean>((resolve) => {+ timer = setTimeout(() => {+ onTimeout();+ resolve(false);+ }, timeoutMs);+ }),+ ]);+ } finally {+ clearTimeout(timer);+ }+}++function stateAttemptId(event: InstrumentationEvent): string | undefined {+ if (!("scope" in event)) return undefined;+ return event.type.startsWith("model.call.") ||+ event.type.startsWith("tool.call.") ||+ event.type.startsWith("step.attempt.")+ ? event.scope.attemptId+ : undefined;+}++function releaseTerminalState(event: InstrumentationEvent): void {+ if (isTerminal(event.type)) releaseAllInstrumentationState(event.idempotencyKey);+ if (event.type === "step.attempt.completed" || event.type === "step.attempt.failed") {+ releaseAllInstrumentationAttemptState(event.scope.attemptId);+ }+}++function isTerminal(type: InstrumentationEvent["type"]): boolean {+ return (+ type.endsWith(".completed") ||+ type.endsWith(".failed") ||+ type.endsWith(".cancelled") ||+ type === "input.resolved"+ );+}diff --git a/packages/eve/src/harness/instrumentation-lifecycle.test.ts b/packages/eve/src/harness/instrumentation-lifecycle.test.tsindex 59af53480..5e0917e88 100644--- a/packages/eve/src/harness/instrumentation-lifecycle.test.ts+++ b/packages/eve/src/harness/instrumentation-lifecycle.test.ts@@ -1,14 +1,26 @@-import { describe, expect, it } from "vitest";+import { describe, expect, it, vi } from "vitest";+import { ContextContainer, contextStorage } from "#context/container.js";+import { deserializeContext, serializeContext } from "#context/serialize.js";import {attemptIdempotencyKey,+ createInstrumentationHooks,inputIdempotencyKey,modelCallIdempotencyKey,sessionIdempotencyKey,toolCallIdempotencyKey,turnIdempotencyKey,type InstrumentationAttemptScope,+ type InstrumentationModelCallStartedEvent,+ type InstrumentationProviderDefinition,} from "#harness/instrumentation-lifecycle.js";+import { instrumentationStateSlot } from "#harness/instrumentation-state.js";++const { logWarn } = vi.hoisted(() => ({ logWarn: vi.fn() }));+vi.mock("#internal/logging.js", () => ({+ createLogger: () => ({ warn: logWarn }),+ formatError: (error: unknown) => error,+}));const scope: InstrumentationAttemptScope = {attemptId: "session-1:turn-1:0:0",@@ -18,6 +30,14 @@ const scope: InstrumentationAttemptScope = {turnId: "turn-1",};+function deferred(): { readonly promise: Promise<void>; readonly resolve: () => void } {+ let resolve!: () => void;+ const promise = new Promise<void>((complete) => {+ resolve = complete;+ });+ return { promise, resolve };+}+describe("instrumentation idempotency keys", () => {it("separates classes that share an identifier", () => {expect(sessionIdempotencyKey("shared")).not.toBe(turnIdempotencyKey("shared", "shared"));@@ -48,3 +68,682 @@ describe("instrumentation idempotency keys", () => {expect(modelCallIdempotencyKey(scope, 0)).not.toBe(toolCallIdempotencyKey(scope, "call-1", 0));});});++describe("provider state lifecycle", () => {+ const started = {+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),+ rootSessionId: "session-1",+ sequence: 0,+ sessionId: "session-1",+ turnId: "turn-1",+ type: "turn.started" as const,+ };+ const completed = {+ idempotencyKey: started.idempotencyKey,+ sessionId: "session-1",+ turnId: "turn-1",+ type: "turn.completed" as const,+ };++ it("carries a value from a start to its terminal and releases it", async () => {+ let observed: unknown;+ const hooks = createInstrumentationHooks([+ {+ events: {+ "turn.completed": (_event, ctx) => {+ observed = ctx.state.get();+ },+ "turn.started": (_event, ctx) => ctx.state.set({ rowId: "row-1" }),+ },+ name: "sink",+ },+ ]);++ await contextStorage.run(new ContextContainer(), async () => {+ await hooks.publish(started);+ await hooks.publish(completed);+ expect(instrumentationStateSlot("sink", started.idempotencyKey).get()).toBeUndefined();+ });+ expect(observed).toEqual({ rowId: "row-1" });+ });++ it("releases state when no terminal handler exists", async () => {+ const hooks = createInstrumentationHooks([+ { events: { "turn.started": (_event, ctx) => ctx.state.set("open") }, name: "sink" },+ ]);+ await contextStorage.run(new ContextContainer(), async () => {+ await hooks.publish(started);+ await hooks.publish(completed);+ expect(instrumentationStateSlot("sink", started.idempotencyKey).get()).toBeUndefined();+ });+ });++ it("releases terminal state after its provider is removed", async () => {+ const starts = createInstrumentationHooks([+ { events: { "turn.started": (_event, ctx) => ctx.state.set("open") }, name: "removed" },+ ]);+ const terminals = createInstrumentationHooks([]);+ await contextStorage.run(new ContextContainer(), async () => {+ await starts.publish(started);+ await terminals.publish(completed);+ expect(instrumentationStateSlot("removed", started.idempotencyKey).get()).toBeUndefined();+ });+ });++ it("keeps input state past its attempt and restores it for resolution", async () => {+ const key = inputIdempotencyKey(scope.sessionId, scope.turnId, "request-1");+ const context = new ContextContainer();+ const starts = createInstrumentationHooks([+ {+ events: { "input.requested": (_event, ctx) => ctx.state.set({ rowId: "input-1" }) },+ name: "sink",+ },+ ]);+ await contextStorage.run(context, async () => {+ await starts.publish({+ action: { callId: "call-1", name: "weather" },+ idempotencyKey: key,+ kind: "tool-approval",+ request: { prompt: "Approve weather?" },+ requestId: "request-1",+ scope,+ type: "input.requested",+ });+ await starts.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ type: "step.attempt.completed",+ });+ });++ const observed: unknown[] = [];+ const restored = await deserializeContext(await serializeContext(context));+ const terminals = createInstrumentationHooks([+ {+ events: { "input.resolved": (_event, ctx) => void observed.push(ctx.state.get()) },+ name: "sink",+ },+ ]);+ await contextStorage.run(restored, async () => {+ await terminals.publish({+ idempotencyKey: key,+ kind: "tool-approval",+ outcome: "approved",+ requestId: "request-1",+ response: { optionId: "approve" },+ scope,+ type: "input.resolved",+ });+ expect(instrumentationStateSlot("sink", key).get()).toBeUndefined();+ });+ expect(observed).toEqual([{ rowId: "input-1" }]);+ });++ it("releases unterminated model state when its attempt ends", async () => {+ const modelKey = modelCallIdempotencyKey(scope, 0);+ const hooks = createInstrumentationHooks([+ {+ events: { "model.call.started": (_event, ctx) => ctx.state.set("open") },+ name: "sink",+ },+ ]);+ await contextStorage.run(new ContextContainer(), async () => {+ await hooks.publish({+ idempotencyKey: modelKey,+ input: { messages: [] },+ model: { modelId: "model", provider: "test" },+ scope,+ type: "model.call.started",+ });+ await hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ type: "step.attempt.completed",+ });+ expect(instrumentationStateSlot("sink", modelKey).get()).toBeUndefined();+ });+ });++ it("releases attempt-owned state after its provider is removed", async () => {+ const modelKey = modelCallIdempotencyKey(scope, 0);+ const starts = createInstrumentationHooks([+ {+ events: { "model.call.started": (_event, ctx) => ctx.state.set("open") },+ name: "removed",+ },+ ]);+ const terminals = createInstrumentationHooks([]);+ await contextStorage.run(new ContextContainer(), async () => {+ await starts.publish({+ idempotencyKey: modelKey,+ input: { messages: [] },+ model: { modelId: "model", provider: "test" },+ scope,+ type: "model.call.started",+ });+ await terminals.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ type: "step.attempt.completed",+ });+ expect(instrumentationStateSlot("removed", modelKey).get()).toBeUndefined();+ });+ });+});++describe("provider handler deadlines", () => {+ const key = turnIdempotencyKey("session-1", "turn-1");+ const started = {+ idempotencyKey: key,+ rootSessionId: "session-1",+ sequence: 0,+ sessionId: "session-1",+ turnId: "turn-1",+ type: "turn.started" as const,+ };+ const completed = {+ idempotencyKey: key,+ sessionId: "session-1",+ turnId: "turn-1",+ type: "turn.completed" as const,+ };+ const hang = () => new Promise<void>(() => {});++ it("lets providers behind a hanging handler run", async () => {+ const after = vi.fn();+ const hooks = createInstrumentationHooks(+ [+ { events: { "turn.started": hang }, name: "hangs" },+ { events: { "turn.started": after }, name: "after" },+ ],+ { handlerTimeoutMs: 1 },+ );+ await contextStorage.run(new ContextContainer(), () => hooks.publish(started));+ expect(after).toHaveBeenCalledOnce();+ });++ it("persists abandonment across workers and skips the terminal", async () => {+ const terminal = vi.fn();+ const context = new ContextContainer();+ const starts = createInstrumentationHooks(+ [{ events: { "turn.started": hang }, name: "sink" }],+ { handlerTimeoutMs: 1 },+ );+ await contextStorage.run(context, () => starts.publish(started));++ const restored = await deserializeContext(await serializeContext(context));+ const terminals = createInstrumentationHooks(+ [{ events: { "turn.completed": terminal }, name: "sink" }],+ { handlerTimeoutMs: 1 },+ );+ await contextStorage.run(restored, () => terminals.publish(completed));+ expect(terminal).not.toHaveBeenCalled();+ });++ it("abandons a timed-out input request across workers", async () => {+ const key = inputIdempotencyKey(scope.sessionId, scope.turnId, "request-1");+ const context = new ContextContainer();+ const starts = createInstrumentationHooks(+ [{ events: { "input.requested": hang }, name: "sink" }],+ { handlerTimeoutMs: 1 },+ );+ await contextStorage.run(context, () =>+ starts.publish({+ action: { callId: "call-1", name: "weather" },+ idempotencyKey: key,+ kind: "tool-approval",+ request: { prompt: "Approve weather?" },+ requestId: "request-1",+ scope,+ type: "input.requested",+ }),+ );++ const terminal = vi.fn();+ const restored = await deserializeContext(await serializeContext(context));+ const terminals = createInstrumentationHooks([+ { events: { "input.resolved": terminal }, name: "sink" },+ ]);+ await contextStorage.run(restored, () =>+ terminals.publish({+ idempotencyKey: key,+ kind: "tool-approval",+ outcome: "approved",+ requestId: "request-1",+ scope,+ type: "input.resolved",+ }),+ );+ expect(terminal).not.toHaveBeenCalled();+ });++ it("ignores a timed-out handler's late state write", async () => {+ let resume!: () => void;+ const continued = new Promise<void>((resolve) => {+ resume = resolve;+ });+ const hooks = createInstrumentationHooks(+ [+ {+ events: {+ "turn.started": async (_event, ctx) => {+ await continued;+ ctx.state.set("late");+ },+ },+ name: "slow",+ },+ ],+ { handlerTimeoutMs: 1 },+ );+ await contextStorage.run(new ContextContainer(), async () => {+ await hooks.publish(started);+ await hooks.publish(completed);+ resume();+ await continued;+ await Promise.resolve();+ expect(instrumentationStateSlot("slow", key).get()).toBeUndefined();+ });+ });++ it("does not abandon an operation for a point-event timeout", async () => {+ const terminal = vi.fn();+ const hooks = createInstrumentationHooks(+ [+ {+ events: {+ "step.attempt.completed": terminal,+ "step.attempt.metadata": hang,+ },+ name: "slow-metadata",+ },+ ],+ { handlerTimeoutMs: 1 },+ );+ await contextStorage.run(new ContextContainer(), async () => {+ await hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ providerMetadata: {},+ scope,+ type: "step.attempt.metadata",+ });+ await hooks.publish({+ idempotencyKey: attemptIdempotencyKey(scope),+ scope,+ type: "step.attempt.completed",+ });+ });+ expect(terminal).toHaveBeenCalledOnce();+ expect(logWarn).toHaveBeenCalledWith(+ "instrumentation provider timed out",+ expect.objectContaining({ boundary: "step.attempt.metadata", provider: "slow-metadata" }),+ );+ });+});++describe("provider dispatch groups", () => {+ const started = {+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),+ rootSessionId: "session-1",+ sequence: 0,+ sessionId: "session-1",+ turnId: "turn-1",+ type: "turn.started" as const,+ };+ const completed = {+ idempotencyKey: started.idempotencyKey,+ sessionId: "session-1",+ turnId: "turn-1",+ type: "turn.completed" as const,+ };++ it("starts every parallel handler before either is released", async () => {+ const first = deferred();+ const second = deferred();+ const order: string[] = [];+ const hooks = createInstrumentationHooks({+ parallel: [+ {+ events: {+ "turn.started": async () => {+ order.push("first:start");+ await first.promise;+ order.push("first:end");+ },+ },+ name: "first",+ },+ {+ events: {+ "turn.started": async () => {+ order.push("second:start");+ await second.promise;+ order.push("second:end");+ },+ },+ name: "second",+ },+ ],+ });++ await contextStorage.run(new ContextContainer(), async () => {+ const publication = hooks.publish(started);+ expect(order).toEqual(["first:start", "second:start"]);+ first.resolve();+ second.resolve();+ await publication;+ });+ });++ it("publishes one immutable snapshot to every dispatch phase", async () => {+ const observed: unknown[] = [];+ const mutableScope = { ...scope };+ const event: InstrumentationModelCallStartedEvent = {+ idempotencyKey: modelCallIdempotencyKey(mutableScope, 0),+ input: { messages: [{ content: "private", role: "user" }] },+ model: { modelId: "model", provider: "test" },+ scope: mutableScope,+ type: "model.call.started" as const,+ };+ const mutator = (snapshot: InstrumentationModelCallStartedEvent): void => {+ expect(Object.isFrozen(snapshot)).toBe(true);+ expect(Object.isFrozen(snapshot.scope)).toBe(true);+ expect(Object.isFrozen(snapshot.input)).toBe(true);+ expect(Reflect.set(snapshot, "idempotencyKey", "corrupted")).toBe(false);+ expect(Reflect.set(snapshot.scope, "turnId", "corrupted")).toBe(false);+ };+ const observer = (snapshot: InstrumentationModelCallStartedEvent): void => {+ observed.push(snapshot.idempotencyKey, snapshot.scope.turnId, snapshot.type);+ };+ const hooks = createInstrumentationHooks({+ parallel: [+ { events: { "model.call.started": mutator }, name: "mutator" },+ { events: { "model.call.started": observer }, name: "observer" },+ ],+ serialAfter: [{ events: { "model.call.started": observer }, name: "cleanup" }],+ serialBefore: [{ events: { "model.call.started": observer }, name: "framework" }],+ });++ await contextStorage.run(new ContextContainer(), () => hooks.publish(event));++ expect(observed).toEqual([+ event.idempotencyKey,+ scope.turnId,+ event.type,+ event.idempotencyKey,+ scope.turnId,+ event.type,+ event.idempotencyKey,+ scope.turnId,+ event.type,+ ]);+ expect(Object.isFrozen(event)).toBe(false);+ expect(mutableScope.turnId).toBe(scope.turnId);+ });++ it("reuses the frozen scope snapshot across publications", async () => {+ const observedScopes: InstrumentationAttemptScope[] = [];+ const hooks = createInstrumentationHooks([+ {+ events: {+ "model.call.completed": (event) => void observedScopes.push(event.scope),+ "model.call.started": (event) => void observedScopes.push(event.scope),+ },+ name: "observer",+ },+ ]);+ const sharedScope = { ...scope };++ await contextStorage.run(new ContextContainer(), async () => {+ await hooks.publish({+ idempotencyKey: modelCallIdempotencyKey(sharedScope, 0),+ input: { messages: [] },+ model: { modelId: "model", provider: "test" },+ scope: sharedScope,+ type: "model.call.started",+ });+ await hooks.publish({+ content: [],+ finishReason: "stop",+ idempotencyKey: modelCallIdempotencyKey(sharedScope, 0),+ scope: sharedScope,+ type: "model.call.completed",+ usage: {},+ });+ });++ expect(observedScopes).toHaveLength(2);+ expect(observedScopes[0]).toBe(observedScopes[1]);+ expect(observedScopes[0]).not.toBe(sharedScope);+ expect(Object.isFrozen(observedScopes[0])).toBe(true);+ });++ it("uses one timeout window for concurrent handlers", async () => {+ vi.useFakeTimers();+ const fast = vi.fn();+ const hooks = createInstrumentationHooks(+ {+ parallel: [+ { events: { "turn.started": () => new Promise<void>(() => {}) }, name: "hangs-one" },+ { events: { "turn.started": fast }, name: "fast" },+ { events: { "turn.started": () => new Promise<void>(() => {}) }, name: "hangs-two" },+ ],+ },+ { handlerTimeoutMs: 10 },+ );++ try {+ await contextStorage.run(new ContextContainer(), async () => {+ const publication = hooks.publish(started);+ let settled = false;+ void publication.then(() => {+ settled = true;+ });+ expect(fast).toHaveBeenCalledOnce();+ await vi.advanceTimersByTimeAsync(9);+ expect(settled).toBe(false);+ await vi.advanceTimersByTimeAsync(1);+ await publication;+ expect(settled).toBe(true);+ });+ } finally {+ vi.useRealTimers();+ }+ });++ it("continues parallel delivery when another handler throws", async () => {+ const received = vi.fn();+ const hooks = createInstrumentationHooks({+ parallel: [+ {+ events: {+ "turn.started": () => {+ throw new Error("expected provider failure");+ },+ },+ name: "throws",+ },+ { events: { "turn.started": received }, name: "receives" },+ ],+ });++ await contextStorage.run(new ContextContainer(), () => hooks.publish(started));+ expect(received).toHaveBeenCalledOnce();+ });++ it("runs serial phases around the parallel publication", async () => {+ vi.useFakeTimers();+ const before = deferred();+ const order: string[] = [];+ const hooks = createInstrumentationHooks(+ {+ parallel: [+ {+ events: {+ "turn.started": () => {+ order.push("parallel");+ return new Promise<void>(() => {});+ },+ },+ name: "parallel",+ },+ ],+ serialAfter: [+ { events: { "turn.started": () => void order.push("after") }, name: "after" },+ ],+ serialBefore: [+ {+ events: {+ "turn.started": async () => {+ order.push("before:start");+ await before.promise;+ order.push("before:end");+ },+ },+ name: "before",+ },+ ],+ },+ { handlerTimeoutMs: 10 },+ );++ try {+ await contextStorage.run(new ContextContainer(), async () => {+ const publication = hooks.publish(started);+ expect(order).toEqual(["before:start"]);+ before.resolve();+ await vi.advanceTimersByTimeAsync(0);+ expect(order).toEqual(["before:start", "before:end", "parallel"]);+ await vi.advanceTimersByTimeAsync(10);+ await publication;+ });+ expect(order).toEqual(["before:start", "before:end", "parallel", "after"]);+ } finally {+ vi.useRealTimers();+ }+ });++ it("runs serial cleanup before rethrowing an unexpected parallel rejection", async () => {+ const failure = new Error("framework state failed");+ const cleanup = vi.fn();+ const broken = Object.defineProperty({ name: "broken" }, "events", {+ get: () => {+ throw failure;+ },+ }) as InstrumentationProviderDefinition;+ const hooks = createInstrumentationHooks({+ parallel: [broken],+ serialAfter: [{ events: { "turn.started": cleanup }, name: "cleanup" }],+ });++ await contextStorage.run(new ContextContainer(), async () => {+ await expect(hooks.publish(started)).rejects.toBe(failure);+ });+ expect(cleanup).toHaveBeenCalledOnce();+ });++ it("keeps durable state isolated between parallel providers", async () => {+ const observed = new Map<string, unknown>();+ const hooks = createInstrumentationHooks({+ parallel: [+ {+ events: {+ "turn.completed": (_event, ctx) => {+ observed.set("first", ctx.state.get());+ },+ "turn.started": (_event, ctx) => ctx.state.set("one"),+ },+ name: "first",+ },+ {+ events: {+ "turn.completed": (_event, ctx) => {+ observed.set("second", ctx.state.get());+ },+ "turn.started": (_event, ctx) => ctx.state.set("two"),+ },+ name: "second",+ },+ ],+ });++ await contextStorage.run(new ContextContainer(), async () => {+ await hooks.publish(started);+ await hooks.publish(completed);+ });+ expect(observed).toEqual(+ new Map([+ ["first", "one"],+ ["second", "two"],+ ]),+ );+ });++ it("suppresses only a timed-out provider terminal and revokes its late write", async () => {+ vi.useFakeTimers();+ const continuation = deferred();+ const unaffectedTerminal = vi.fn();+ const timedOutTerminal = vi.fn();+ const hooks = createInstrumentationHooks(+ {+ parallel: [+ {+ events: {+ "turn.completed": timedOutTerminal,+ "turn.started": async (_event, ctx) => {+ await continuation.promise;+ ctx.state.set("late");+ },+ },+ name: "times-out",+ },+ { events: { "turn.completed": unaffectedTerminal }, name: "unaffected" },+ ],+ },+ { handlerTimeoutMs: 10 },+ );++ try {+ await contextStorage.run(new ContextContainer(), async () => {+ const publication = hooks.publish(started);+ await vi.advanceTimersByTimeAsync(10);+ await publication;+ await hooks.publish(completed);+ continuation.resolve();+ await continuation.promise;+ await Promise.resolve();+ expect(instrumentationStateSlot("times-out", started.idempotencyKey).get()).toBeUndefined();+ });+ } finally {+ vi.useRealTimers();+ }+ expect(timedOutTerminal).not.toHaveBeenCalled();+ expect(unaffectedTerminal).toHaveBeenCalledOnce();+ });++ it("keeps array input sequential", async () => {+ const first = deferred();+ const order: string[] = [];+ const hooks = createInstrumentationHooks([+ {+ events: {+ "turn.started": async () => {+ order.push("first:start");+ await first.promise;+ order.push("first:end");+ },+ },+ name: "first",+ },+ { events: { "turn.started": () => void order.push("second") }, name: "second" },+ ]);++ await contextStorage.run(new ContextContainer(), async () => {+ const publication = hooks.publish(started);+ expect(order).toEqual(["first:start"]);+ first.resolve();+ await publication;+ });+ expect(order).toEqual(["first:start", "first:end", "second"]);+ });+});diff --git a/packages/eve/src/harness/instrumentation-lifecycle.ts b/packages/eve/src/harness/instrumentation-lifecycle.tsindex 5f78e03e2..bb7df010a 100644--- a/packages/eve/src/harness/instrumentation-lifecycle.ts+++ b/packages/eve/src/harness/instrumentation-lifecycle.ts@@ -1,4 +1,5 @@-import { createLogger, formatError } from "#internal/logging.js";+import { createInstrumentationDispatcher } from "#harness/instrumentation-dispatch.js";+import type { InstrumentationStateSlot } from "#harness/instrumentation-state.js";/*** Stable eve identity for one actual model attempt.@@ -345,16 +346,23 @@ export type InstrumentationToolCallTerminalEvent =| InstrumentationToolCallCompletedEvent| InstrumentationToolCallFailedEvent;+export interface InstrumentationHandlerContext {+ readonly state: InstrumentationStateSlot;+}+/*** The AI SDK can omit a model terminal when an incomplete stream closes. A* provider that correlates starts with terminals must scope that state to the* attempt and release anything still open when the step attempt terminates.*/-export type InstrumentationEventHandler<TEvent> = (event: TEvent) => void | PromiseLike<void>;+export type InstrumentationEventHandler<TEvent> = (+ event: TEvent,+ ctx: InstrumentationHandlerContext,+) => void | PromiseLike<void>;/** Internal provider shape mirrored by the future public hook contract. */export interface InstrumentationProviderDefinition {- readonly name?: string;+ readonly name: string;readonly events?: {readonly "step.attempt.started"?: InstrumentationEventHandler<InstrumentationStepAttemptStartedEvent>;readonly "step.attempt.completed"?: InstrumentationEventHandler<InstrumentationStepAttemptCompletedEvent>;@@ -381,6 +389,20 @@ export interface InstrumentationProviderDefinition {readonly shutdown?: () => void | PromiseLike<void>;}+type InstrumentationProviderInput = Omit<InstrumentationProviderDefinition, "name"> & {+ readonly name?: string;+};++export interface InstrumentationDispatchGroups {+ readonly serialBefore?: readonly InstrumentationProviderDefinition[];+ readonly parallel?: readonly InstrumentationProviderDefinition[];+ readonly serialAfter?: readonly InstrumentationProviderDefinition[];+}++export type InstrumentationHooksInput =+ | readonly InstrumentationProviderInput[]+ | InstrumentationDispatchGroups;+/** Events that pair a start with its terminal under one `idempotencyKey`. */export type InstrumentationCorrelatedEvent =| InstrumentationInputRequestedEvent@@ -425,26 +447,14 @@ export interface InstrumentationHooks {publish(event: InstrumentationEvent): Promise<void>;}-const log = createLogger("harness.instrumentation-lifecycle");+export interface CreateInstrumentationHooksOptions {+ readonly handlerTimeoutMs?: number;+}-/** Creates failure-isolated hooks backed by an ordered provider list. */+/** Creates failure-isolated hooks backed by normalized dispatch groups. */export function createInstrumentationHooks(- providers: readonly InstrumentationProviderDefinition[],+ input: InstrumentationHooksInput,+ options: CreateInstrumentationHooksOptions = {},): InstrumentationHooks {- const publish = async (event: InstrumentationEvent): Promise<void> => {- for (const provider of providers) {- const handler = provider.events?.[event.type];- if (handler === undefined) continue;- try {- await (handler as InstrumentationEventHandler<InstrumentationEvent>)(event);- } catch (error) {- log.warn("instrumentation provider failed", {- boundary: event.type,- error: formatError(error),- });- }- }- };-- return { publish };+ return createInstrumentationDispatcher(input, options);}diff --git a/packages/eve/src/harness/instrumentation-state.test.ts b/packages/eve/src/harness/instrumentation-state.test.tsnew file mode 100644index 000000000..5659fbd5b--- /dev/null+++ b/packages/eve/src/harness/instrumentation-state.test.ts@@ -0,0 +1,106 @@+import { describe, expect, it } from "vitest";++import { ContextContainer, contextStorage } from "#context/container.js";+import { deserializeContext, serializeContext } from "#context/serialize.js";+import {+ abandonInstrumentationState,+ instrumentationStateSlot,+ isInstrumentationStateAbandoned,+ preserveSerializedInstrumentationState,+ releaseAllInstrumentationAttemptState,+ releaseAllInstrumentationState,+} from "#harness/instrumentation-state.js";++describe("instrumentation state", () => {+ it("survives a serialized step boundary", async () => {+ const context = new ContextContainer();+ contextStorage.run(context, () => {+ instrumentationStateSlot("sink", "model:1", "attempt-1").set({ rowId: "row-1" });+ });+ const restored = await deserializeContext(await serializeContext(context));+ contextStorage.run(restored, () => {+ expect(instrumentationStateSlot("sink", "model:1").get()).toEqual({ rowId: "row-1" });+ });+ });++ it("isolates providers and operations", () => {+ contextStorage.run(new ContextContainer(), () => {+ instrumentationStateSlot("a", "turn:1").set("a-1");+ instrumentationStateSlot("b", "turn:1").set("b-1");+ instrumentationStateSlot("a", "turn:2").set("a-2");+ expect(instrumentationStateSlot("a", "turn:1").get()).toBe("a-1");+ expect(instrumentationStateSlot("b", "turn:1").get()).toBe("b-1");+ expect(instrumentationStateSlot("a", "turn:2").get()).toBe("a-2");+ });+ });++ it("rejects values that cannot survive serialization", () => {+ contextStorage.run(new ContextContainer(), () => {+ expect(() => instrumentationStateSlot("sink", "turn:1").set(new Date() as never)).toThrow(+ TypeError,+ );+ });+ });++ it("revokes late reads and writes", () => {+ contextStorage.run(new ContextContainer(), () => {+ const lease = instrumentationStateSlot("sink", "turn:1");+ lease.set("before");+ lease.revoke();+ lease.set("after");+ expect(lease.get()).toBeUndefined();+ expect(instrumentationStateSlot("sink", "turn:1").get()).toBe("before");+ });+ });++ it("does not expose mutable durable state through a retained read", () => {+ contextStorage.run(new ContextContainer(), () => {+ const original = { nested: { count: 1 } };+ const lease = instrumentationStateSlot("sink", "turn:1");+ lease.set(original);+ original.nested.count = 2;++ const retained = lease.get() as { nested: { count: number } };+ lease.revoke();++ expect(Object.isFrozen(retained)).toBe(true);+ expect(Object.isFrozen(retained.nested)).toBe(true);+ expect(Reflect.set(retained.nested, "count", 3)).toBe(false);+ expect(instrumentationStateSlot("sink", "turn:1").get()).toEqual({ nested: { count: 1 } });+ });+ });++ it("releases exact and attempt-owned state", () => {+ contextStorage.run(new ContextContainer(), () => {+ instrumentationStateSlot("sink", "model:1", "attempt-1").set("one");+ instrumentationStateSlot("sink", "model:2", "attempt-2").set("two");+ releaseAllInstrumentationState("model:2");+ releaseAllInstrumentationAttemptState("attempt-1");+ expect(instrumentationStateSlot("sink", "model:1").get()).toBeUndefined();+ expect(instrumentationStateSlot("sink", "model:2").get()).toBeUndefined();+ });+ });++ it("preserves state from a discarded step", () => {+ expect(+ preserveSerializedInstrumentationState(+ { authored: "original" },+ { "eve.harness.instrumentationState": { state: true } },+ ),+ ).toEqual({+ authored: "original",+ "eve.harness.instrumentationState": { state: true },+ });+ });++ it("persists abandonment across serialization", async () => {+ const context = new ContextContainer();+ contextStorage.run(context, () => {+ abandonInstrumentationState("sink", "model:1", "attempt-1");+ });+ const restored = await deserializeContext(await serializeContext(context));+ contextStorage.run(restored, () => {+ expect(isInstrumentationStateAbandoned("sink", "model:1")).toBe(true);+ });+ });+});diff --git a/packages/eve/src/harness/instrumentation-state.ts b/packages/eve/src/harness/instrumentation-state.tsnew file mode 100644index 000000000..18ee462f3--- /dev/null+++ b/packages/eve/src/harness/instrumentation-state.ts@@ -0,0 +1,176 @@+import { contextStorage, loadContext } from "#context/container.js";+import { ContextKey } from "#context/key.js";+import { type JsonValue, parseJsonValue } from "#shared/json.js";++interface InstrumentationStateRecord {+ abandoned?: true;+ attemptId?: string;+ value?: JsonValue;+}++type InstrumentationStateMap = Readonly<Record<string, InstrumentationStateRecord>>;++const InstrumentationStateKey = new ContextKey<InstrumentationStateMap>(+ "eve.harness.instrumentationState",+ {+ codec: {+ deserialize: deserializeState,+ serialize: (state) => state,+ },+ },+);++/** Keeps provider state from an interrupted step's discarded context changes. */+export function preserveSerializedInstrumentationState(+ original: Record<string, unknown>,+ interrupted: Record<string, unknown>,+): Record<string, unknown> {+ const state = interrupted[InstrumentationStateKey.name];+ return state === undefined ? original : { ...original, [InstrumentationStateKey.name]: state };+}++export interface InstrumentationStateSlot {+ get(): JsonValue | undefined;+ set(value: JsonValue | undefined): void;+}++export interface InstrumentationStateLease extends InstrumentationStateSlot {+ revoke(): void;+}++/** One provider's durable state for one operation. */+export function instrumentationStateSlot(+ provider: string,+ idempotencyKey: string,+ attemptId?: string,+): InstrumentationStateLease {+ const key = stateKey(provider, idempotencyKey);+ let active = true;+ return {+ get: () => {+ if (!active) return undefined;+ const value = contextStorage.getStore()?.get(InstrumentationStateKey)?.[key]?.value;+ return value === undefined ? undefined : cloneAndFreezeJson(value);+ },+ revoke: () => {+ active = false;+ },+ set: (value) => {+ if (!active || contextStorage.getStore() === undefined) return;+ if (value === undefined) {+ writeState((state) => writeSlot(state, key, undefined));+ return;+ }+ const record: InstrumentationStateRecord = {+ value: cloneAndFreezeJson(parseJsonValue(value)),+ };+ const current = contextStorage.getStore()?.get(InstrumentationStateKey)?.[key];+ if (current?.abandoned === true) record.abandoned = true;+ if (attemptId !== undefined) record.attemptId = attemptId;+ writeState((state) => writeSlot(state, key, record));+ },+ };+}++export function abandonInstrumentationState(+ provider: string,+ idempotencyKey: string,+ attemptId?: string,+): void {+ if (contextStorage.getStore() === undefined) return;+ const key = stateKey(provider, idempotencyKey);+ writeState((state) => {+ const current = state[key];+ const record: InstrumentationStateRecord = { abandoned: true };+ const owner = attemptId ?? current?.attemptId;+ if (owner !== undefined) record.attemptId = owner;+ if (current?.value !== undefined) record.value = current.value;+ return writeSlot(state, key, record);+ });+}++export function isInstrumentationStateAbandoned(provider: string, idempotencyKey: string): boolean {+ return (+ contextStorage.getStore()?.get(InstrumentationStateKey)?.[stateKey(provider, idempotencyKey)]+ ?.abandoned === true+ );+}++/** Releases one operation across namespaces, including providers no longer registered. */+export function releaseAllInstrumentationState(idempotencyKey: string): void {+ const suffix = `\0${idempotencyKey}`;+ releaseMatchingInstrumentationState((key) => key.endsWith(suffix));+}++/** Releases attempt-owned children across namespaces when their terminals are omitted. */+export function releaseAllInstrumentationAttemptState(attemptId: string): void {+ releaseMatchingInstrumentationState((_key, record) => record.attemptId === attemptId);+}++function releaseMatchingInstrumentationState(+ matches: (key: string, record: InstrumentationStateRecord) => boolean,+): void {+ const current = contextStorage.getStore()?.get(InstrumentationStateKey);+ if (+ current === undefined ||+ !Object.entries(current).some(([key, record]) => matches(key, record))+ ) {+ return;+ }+ writeState((state) => {+ const next = { ...state };+ for (const [key, record] of Object.entries(state)) {+ if (matches(key, record)) delete next[key];+ }+ return next;+ });+}++function writeState(update: (state: InstrumentationStateMap) => InstrumentationStateMap): void {+ loadContext().set(InstrumentationStateKey, (state) => update(state ?? {}));+}++function writeSlot(+ state: InstrumentationStateMap,+ key: string,+ value: InstrumentationStateRecord | undefined,+): InstrumentationStateMap {+ const next = { ...state };+ if (value === undefined) delete next[key];+ else next[key] = value;+ return next;+}++function stateKey(provider: string, idempotencyKey: string): string {+ return `${provider}\0${idempotencyKey}`;+}++function deserializeState(data: unknown): InstrumentationStateMap {+ if (typeof data !== "object" || data === null || Array.isArray(data)) return {};+ const state: Record<string, InstrumentationStateRecord> = {};+ for (const [key, value] of Object.entries(data)) {+ if (typeof value !== "object" || value === null || Array.isArray(value)) continue;+ const record = value as Record<string, unknown>;+ const parsed: InstrumentationStateRecord = {};+ if (record["abandoned"] === true) parsed.abandoned = true;+ if (typeof record["attemptId"] === "string") parsed.attemptId = record["attemptId"];+ if (record["value"] !== undefined) {+ parsed.value = cloneAndFreezeJson(record["value"] as JsonValue);+ }+ if (parsed.abandoned === true || parsed.value !== undefined) state[key] = parsed;+ }+ return state;+}++function cloneAndFreezeJson(value: JsonValue): JsonValue {+ if (Array.isArray(value)) {+ return Object.freeze(value.map((entry) => cloneAndFreezeJson(entry)));+ }+ if (typeof value !== "object" || value === null) return value;++ const copy: Record<string, JsonValue> = {};+ for (const [key, entry] of Object.entries(value)) {+ copy[key] = cloneAndFreezeJson(entry);+ }+ return Object.freeze(copy);+}diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex 90f3ebb6e..a780b2cd1 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -10242,6 +10242,7 @@ describe("createToolLoopHarness", () => {scope: expect.objectContaining({ attemptIndex: 0 }),type: "step.attempt.completed",}),+ expect.anything(),);});diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.tsindex 2f617df9d..a5b920303 100644--- a/packages/eve/src/tracing/agent-otel-provider.test.ts+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts@@ -1,12 +1,13 @@-import { SpanStatusCode } from "@opentelemetry/api";+import { SpanStatusCode, trace as apiTrace } from "@opentelemetry/api";import {BasicTracerProvider,InMemorySpanExporter,SimpleSpanProcessor,type ReadableSpan,} from "@opentelemetry/sdk-trace-base";-import { describe, expect, it } from "vitest";+import { describe, expect, it, vi } from "vitest";+import { context } from "#compiled/@opentelemetry/api/index.js";import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js";import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";@@ -263,6 +264,7 @@ function nanos(hrTime: readonly [number, number]): bigint {describe("createAgentOtelInstrumentation", () => {it("emits the agent hierarchy in one session trace", async () => {const runtime = createRuntime();+ const contextWith = vi.spyOn(context, "with");await emitAttempt({hooks: runtime.hooks,runInContext: runtime.runInContext,@@ -270,6 +272,8 @@ describe("createAgentOtelInstrumentation", () => {turnId: "turn-1",turnSequence: 0,});+ const executionParents = contextWith.mock.calls.map(([parent]) => parent);+ contextWith.mockRestore();await runtime.provider.forceFlush();const spans = runtime.exporter.getFinishedSpans();@@ -292,6 +296,12 @@ describe("createAgentOtelInstrumentation", () => {expect(step.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);expect(operation.parentSpanContext?.spanId).toBe(step.spanContext().spanId);expect(model.parentSpanContext?.spanId).toBe(operation.spanContext().spanId);+ expect(+ executionParents.some(+ (parent) =>+ apiTrace.getSpan(parent as never)?.spanContext().spanId === model.spanContext().spanId,+ ),+ ).toBe(true);expect(action.parentSpanContext?.spanId).toBe(step.spanContext().spanId);expect(tool.parentSpanContext?.spanId).toBe(action.spanContext().spanId);expect(new Set(spans.map((span) => span.spanContext().traceId))).toHaveLength(1);diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.tsindex cf711b8b5..245f576a7 100644--- a/packages/eve/src/tracing/agent-otel-provider.ts+++ b/packages/eve/src/tracing/agent-otel-provider.ts@@ -84,6 +84,7 @@ export function createAgentOtelInstrumentation(InstrumentationAttemptScope,{ readonly models: Map<string, Context>; readonly tools: Map<string, Context> }>();+ const attemptScopes = new Map<string, InstrumentationAttemptScope>();// A serverless turn runs inside one `turnStep` "use step" invocation. If// that worker is lost, Workflow retries the whole step from entry rather// than resuming this callback sequence in a replacement process.@@ -173,12 +174,15 @@ export function createAgentOtelInstrumentation(},step: { context: stepContext, span: stepSpan },});+ attemptScopes.set(event.scope.attemptId, event.scope);};const onStepTerminal = (event: InstrumentationStepAttemptTerminalEvent): void => {- executionContexts.delete(event.scope);- drainOpenSpans(event);- const attempt = steps.get(event.scope);+ const scope = attemptScopes.get(event.scope.attemptId) ?? event.scope;+ executionContexts.delete(scope);+ drainOpenSpans({ ...event, scope });+ attemptScopes.delete(event.scope.attemptId);+ const attempt = steps.get(scope);if (attempt === undefined) return;// The span event drops the `attempt` segment: this span *is* one attempt,// and `agent.step.attempt` on it already says which.@@ -191,7 +195,7 @@ export function createAgentOtelInstrumentation(}attempt.operation.span.end();attempt.step.span.end();- steps.delete(event.scope);+ steps.delete(scope);};const onTurnTerminal = async (event: InstrumentationTurnTerminalEvent): Promise<void> => {@@ -526,7 +530,8 @@ export function createAgentOtelInstrumentation(},},runInContext(operation, execute) {- const contexts = executionContexts.get(operation.scope);+ const scope = attemptScopes.get(operation.scope.attemptId) ?? operation.scope;+ const contexts = executionContexts.get(scope);const parent =operation.type === "model.call"? contexts?.models.get(operation.idempotencyKey)diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts b/packages/eve/src/tracing/install-instrumentation-runtime.test.tsindex bd0c07fb8..4d65b92a0 100644--- a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts+++ b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts@@ -45,7 +45,7 @@ describe("installInstrumentationRuntime", () => {const runtime = installInstrumentationRuntime({collected: collectOtelPipeline([otelIntegration()]),frameworkVersion: "test",- providers: [{ flush: providerFlush, shutdown: providerShutdown }],+ providers: [{ flush: providerFlush, name: "test", shutdown: providerShutdown }],serviceName: "weather",});