Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(eve): persist provider state and bound instrumentation handlers (#1860)

Chad Hietala1528fda2da4a
diff --git a/.changeset/provider-state-deadlines.md b/.changeset/provider-state-deadlines.md
new file mode 100644
index 000000000..92fb99d2b
--- /dev/null
+++ b/.changeset/provider-state-deadlines.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Add durable operation-scoped state to instrumentation handlers, isolated by provider and automatically released at terminal boundaries. Bound each handler and persist start-handler abandonment so a stalled provider cannot block the bus or later receive a mismatched terminal.
diff --git a/packages/eve/src/execution/workflow-steps.ts b/packages/eve/src/execution/workflow-steps.ts
index 95509d969..a553d0483 100644
--- a/packages/eve/src/execution/workflow-steps.ts
+++ b/packages/eve/src/execution/workflow-steps.ts
@@ -29,6 +29,7 @@ import {
isHarnessBetweenTurns,
setHarnessEmissionState,
} from "#harness/emission.js";
+import { preserveSerializedInstrumentationState } from "#harness/instrumentation-state.js";
import { preserveSerializedAgentTraceState } from "#tracing/agent-trace-context-store.js";
import { matchAuthorizationCallbacks } from "#execution/authorization-callback-match.js";
import { readTurnSleepDurationMs } from "#harness/turn-sleep.js";
@@ -447,11 +448,15 @@ export async function turnStep(rawInput: TurnStepInput): Promise<DurableStepResu
} catch (error) {
if (!isTurnCancellation(error)) throw error;
writer.releaseLock();
+ // Both carve-outs exist because the cancellation epilogue still publishes
+ // `turn.cancelled` downstream, and the open spans and provider state it
+ // needs to close were written by the step being discarded.
+ const interrupted = serializeContext(ctx);
return {
action: "cancelled",
- serializedContext: preserveSerializedAgentTraceState(
- input.serializedContext,
- serializeContext(ctx),
+ serializedContext: preserveSerializedInstrumentationState(
+ preserveSerializedAgentTraceState(input.serializedContext, interrupted),
+ interrupted,
),
sessionState: input.sessionState,
};
diff --git a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts b/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts
index fa647500f..0a51032d6 100644
--- a/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts
+++ b/packages/eve/src/harness/ai-sdk-hook-bridge.test.ts
@@ -38,6 +38,7 @@ describe("createAiSdkHookBridge", () => {
);
},
},
+ name,
};
};
const hooks = createInstrumentationHooks([provider("a"), provider("b")]);
@@ -221,6 +222,7 @@ describe("createAiSdkHookBridge", () => {
expect(after).toHaveBeenCalledExactlyOnceWith(
expect.objectContaining({ error, type: "model.call.failed" }),
+ expect.anything(),
);
});
@@ -236,6 +238,7 @@ describe("createAiSdkHookBridge", () => {
expect(after).toHaveBeenCalledExactlyOnceWith(
expect.objectContaining({ error: reason, type: "tool.call.failed" }),
+ expect.anything(),
);
});
@@ -265,8 +268,8 @@ describe("createAiSdkHookBridge", () => {
scope,
type: "step.attempt.started",
};
- expect(mutator).toHaveBeenCalledExactlyOnceWith(expected);
- expect(started).toHaveBeenCalledExactlyOnceWith(expected);
+ expect(mutator).toHaveBeenCalledExactlyOnceWith(expected, expect.anything());
+ expect(started).toHaveBeenCalledExactlyOnceWith(expected, expect.anything());
});
it("projects the model call callbacks onto eve fields only", async () => {
@@ -333,45 +336,51 @@ describe("createAiSdkHookBridge", () => {
},
]);
- expect(before).toHaveBeenCalledExactlyOnceWith({
- idempotencyKey: `model:${scope.attemptId}:0`,
- input: { instructions: "be brief", messages: [{ content: "hi", role: "user" }] },
- model: { modelId: "model", provider: "test" },
- scope,
- type: "model.call.started",
- });
+ expect(before).toHaveBeenCalledExactlyOnceWith(
+ {
+ idempotencyKey: `model:${scope.attemptId}:0`,
+ input: { instructions: "be brief", messages: [{ content: "hi", role: "user" }] },
+ model: { modelId: "model", provider: "test" },
+ scope,
+ type: "model.call.started",
+ },
+ expect.anything(),
+ );
// An unrecognized part kind is dropped rather than forwarded, so widening
// InstrumentationContentPart is what makes a new kind reachable.
- expect(after).toHaveBeenCalledExactlyOnceWith({
- content: [
- { text: "thinking", type: "reasoning" },
- { text: "hello", type: "text" },
- { callId: "tool-1", input: { a: 1 }, toolName: "search", type: "tool-call" },
- {
- callId: "tool-1",
- input: { a: 1 },
- output: "ok",
- toolName: "search",
- type: "tool-result",
- },
- {
- callId: "tool-2",
- error: "boom",
- input: { a: 2 },
- toolName: "search",
- type: "tool-error",
+ expect(after).toHaveBeenCalledExactlyOnceWith(
+ {
+ content: [
+ { text: "thinking", type: "reasoning" },
+ { text: "hello", type: "text" },
+ { callId: "tool-1", input: { a: 1 }, toolName: "search", type: "tool-call" },
+ {
+ callId: "tool-1",
+ input: { a: 1 },
+ output: "ok",
+ toolName: "search",
+ type: "tool-result",
+ },
+ {
+ callId: "tool-2",
+ error: "boom",
+ input: { a: 2 },
+ toolName: "search",
+ type: "tool-error",
+ },
+ ],
+ finishReason: "tool-calls",
+ idempotencyKey: `model:${scope.attemptId}:0`,
+ scope,
+ type: "model.call.completed",
+ usage: {
+ inputTokenDetails: { cacheReadTokens: 3, cacheWriteTokens: 4 },
+ inputTokens: 1,
+ outputTokens: 2,
},
- ],
- finishReason: "tool-calls",
- idempotencyKey: `model:${scope.attemptId}:0`,
- scope,
- type: "model.call.completed",
- usage: {
- inputTokenDetails: { cacheReadTokens: 3, cacheWriteTokens: 4 },
- inputTokens: 1,
- outputTokens: 2,
},
- });
+ expect.anything(),
+ );
});
it.each([
@@ -405,20 +414,26 @@ describe("createAiSdkHookBridge", () => {
{ callId: "call-1", toolCall, toolExecutionMs: 1, toolOutput },
]);
- expect(before).toHaveBeenCalledExactlyOnceWith({
- callId: "tool-1",
- idempotencyKey: `tool:${scope.attemptId}:tool-1:0`,
- input: { q: "eve" },
- scope,
- toolName: "search",
- type: "tool.call.started",
- });
- expect(after).toHaveBeenCalledExactlyOnceWith({
- idempotencyKey: `tool:${scope.attemptId}:tool-1:0`,
- output: expected,
- scope,
- type: "tool.call.completed",
- });
+ expect(before).toHaveBeenCalledExactlyOnceWith(
+ {
+ callId: "tool-1",
+ idempotencyKey: `tool:${scope.attemptId}:tool-1:0`,
+ input: { q: "eve" },
+ scope,
+ toolName: "search",
+ type: "tool.call.started",
+ },
+ expect.anything(),
+ );
+ expect(after).toHaveBeenCalledExactlyOnceWith(
+ {
+ idempotencyKey: `tool:${scope.attemptId}:tool-1:0`,
+ output: expected,
+ scope,
+ type: "tool.call.completed",
+ },
+ expect.anything(),
+ );
},
);
@@ -433,6 +448,7 @@ describe("createAiSdkHookBridge", () => {
},
"model.call.started": (event) => void own.set(event.idempotencyKey, `${name}-state`),
},
+ name,
};
};
const hooks = createInstrumentationHooks([provider("a"), provider("b")]);
diff --git a/packages/eve/src/harness/instrumentation-dispatch.ts b/packages/eve/src/harness/instrumentation-dispatch.ts
new file mode 100644
index 000000000..67d43f4aa
--- /dev/null
+++ b/packages/eve/src/harness/instrumentation-dispatch.ts
@@ -0,0 +1,207 @@
+import {
+ abandonInstrumentationState,
+ instrumentationStateSlot,
+ isInstrumentationStateAbandoned,
+ releaseAllInstrumentationAttemptState,
+ releaseAllInstrumentationState,
+} from "#harness/instrumentation-state.js";
+import { createLogger, formatError } from "#internal/logging.js";
+
+import type {
+ CreateInstrumentationHooksOptions,
+ InstrumentationDispatchGroups,
+ InstrumentationEvent,
+ InstrumentationEventHandler,
+ InstrumentationHooks,
+ InstrumentationHooksInput,
+ InstrumentationProviderDefinition,
+} from "#harness/instrumentation-lifecycle.js";
+
+const log = createLogger("harness.instrumentation-dispatch");
+const DEFAULT_HANDLER_TIMEOUT_MS = 5_000;
+
+export function createInstrumentationDispatcher(
+ input: InstrumentationHooksInput,
+ options: CreateInstrumentationHooksOptions,
+): InstrumentationHooks {
+ const handlerTimeoutMs = options.handlerTimeoutMs ?? DEFAULT_HANDLER_TIMEOUT_MS;
+ const groups = normalizeDispatchGroups(input);
+ const snapshots = new WeakMap<object, unknown>();
+
+ const publish = async (event: InstrumentationEvent): Promise<void> => {
+ const snapshot = snapshotInstrumentationEvent(event, snapshots);
+ try {
+ try {
+ for (const provider of groups.serialBefore) {
+ await dispatchToProvider(provider, snapshot, handlerTimeoutMs);
+ }
+
+ if (groups.parallel.length === 1) {
+ await dispatchToProvider(groups.parallel[0]!, snapshot, handlerTimeoutMs);
+ } else if (groups.parallel.length > 1) {
+ const results = await Promise.allSettled(
+ groups.parallel.map((provider) =>
+ dispatchToProvider(provider, snapshot, handlerTimeoutMs),
+ ),
+ );
+ const rejected = results.find(
+ (result): result is PromiseRejectedResult => result.status === "rejected",
+ );
+ if (rejected !== undefined) throw rejected.reason;
+ }
+ } finally {
+ for (const provider of groups.serialAfter) {
+ await dispatchToProvider(provider, snapshot, handlerTimeoutMs);
+ }
+ }
+ } finally {
+ releaseTerminalState(snapshot);
+ }
+ };
+
+ return { publish };
+}
+
+function snapshotInstrumentationEvent(
+ event: InstrumentationEvent,
+ snapshots: WeakMap<object, unknown>,
+): InstrumentationEvent {
+ return snapshotPlainValue(event, snapshots) as InstrumentationEvent;
+}
+
+function snapshotPlainValue(value: unknown, seen: WeakMap<object, unknown>): unknown {
+ if (Array.isArray(value)) {
+ const existing = seen.get(value);
+ if (existing !== undefined) return existing;
+ const copy: unknown[] = [];
+ seen.set(value, copy);
+ for (const entry of value) copy.push(snapshotPlainValue(entry, seen));
+ return Object.freeze(copy);
+ }
+
+ if (typeof value !== "object" || value === null) return value;
+ let prototype: object | null;
+ try {
+ prototype = Object.getPrototypeOf(value) as object | null;
+ } catch {
+ return value;
+ }
+ if (prototype !== Object.prototype && prototype !== null) return value;
+
+ const existing = seen.get(value);
+ if (existing !== undefined) return existing;
+ const copy: Record<string, unknown> = {};
+ seen.set(value, copy);
+ for (const [key, entry] of Object.entries(value)) {
+ copy[key] = snapshotPlainValue(entry, seen);
+ }
+ return Object.freeze(copy);
+}
+
+function normalizeDispatchGroups(
+ input: InstrumentationHooksInput,
+): Required<InstrumentationDispatchGroups> {
+ if (!Array.isArray(input)) {
+ const groups = input as InstrumentationDispatchGroups;
+ return {
+ parallel: groups.parallel ?? [],
+ serialAfter: groups.serialAfter ?? [],
+ serialBefore: groups.serialBefore ?? [],
+ };
+ }
+
+ return {
+ parallel: [],
+ serialAfter: [],
+ serialBefore: input.map((provider, index) => ({
+ ...provider,
+ name: provider.name ?? `provider-${String(index)}`,
+ })),
+ };
+}
+
+async function dispatchToProvider(
+ provider: InstrumentationProviderDefinition,
+ event: InstrumentationEvent,
+ handlerTimeoutMs: number,
+): Promise<void> {
+ const startedBoundary = event.type.endsWith(".started") || event.type === "input.requested";
+ const attemptId = stateAttemptId(event);
+ const providerName = provider.name;
+ if (isInstrumentationStateAbandoned(providerName, event.idempotencyKey)) return;
+ const handler = provider.events?.[event.type];
+ if (handler === undefined) return;
+ const state = instrumentationStateSlot(providerName, event.idempotencyKey, attemptId);
+ try {
+ const settled = await withTimeout(
+ () => (handler as InstrumentationEventHandler<InstrumentationEvent>)(event, { state }),
+ handlerTimeoutMs,
+ () => {
+ state.revoke();
+ if (startedBoundary)
+ abandonInstrumentationState(providerName, event.idempotencyKey, attemptId);
+ },
+ );
+ if (!settled) {
+ log.warn("instrumentation provider timed out", {
+ boundary: event.type,
+ provider: providerName,
+ timeoutMs: handlerTimeoutMs,
+ });
+ }
+ } catch (error) {
+ log.warn("instrumentation provider failed", {
+ boundary: event.type,
+ error: formatError(error),
+ provider: providerName,
+ });
+ } finally {
+ state.revoke();
+ }
+}
+
+async function withTimeout(
+ run: () => void | PromiseLike<void>,
+ timeoutMs: number,
+ onTimeout: () => void,
+): Promise<boolean> {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ try {
+ return await Promise.race([
+ Promise.resolve(run()).then(() => true),
+ new Promise<boolean>((resolve) => {
+ timer = setTimeout(() => {
+ onTimeout();
+ resolve(false);
+ }, timeoutMs);
+ }),
+ ]);
+ } finally {
+ clearTimeout(timer);
+ }
+}
+
+function stateAttemptId(event: InstrumentationEvent): string | undefined {
+ if (!("scope" in event)) return undefined;
+ return event.type.startsWith("model.call.") ||
+ event.type.startsWith("tool.call.") ||
+ event.type.startsWith("step.attempt.")
+ ? event.scope.attemptId
+ : undefined;
+}
+
+function releaseTerminalState(event: InstrumentationEvent): void {
+ if (isTerminal(event.type)) releaseAllInstrumentationState(event.idempotencyKey);
+ if (event.type === "step.attempt.completed" || event.type === "step.attempt.failed") {
+ releaseAllInstrumentationAttemptState(event.scope.attemptId);
+ }
+}
+
+function isTerminal(type: InstrumentationEvent["type"]): boolean {
+ return (
+ type.endsWith(".completed") ||
+ type.endsWith(".failed") ||
+ type.endsWith(".cancelled") ||
+ type === "input.resolved"
+ );
+}
diff --git a/packages/eve/src/harness/instrumentation-lifecycle.test.ts b/packages/eve/src/harness/instrumentation-lifecycle.test.ts
index 59af53480..5e0917e88 100644
--- a/packages/eve/src/harness/instrumentation-lifecycle.test.ts
+++ b/packages/eve/src/harness/instrumentation-lifecycle.test.ts
@@ -1,14 +1,26 @@
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
+import { ContextContainer, contextStorage } from "#context/container.js";
+import { deserializeContext, serializeContext } from "#context/serialize.js";
import {
attemptIdempotencyKey,
+ createInstrumentationHooks,
inputIdempotencyKey,
modelCallIdempotencyKey,
sessionIdempotencyKey,
toolCallIdempotencyKey,
turnIdempotencyKey,
type InstrumentationAttemptScope,
+ type InstrumentationModelCallStartedEvent,
+ type InstrumentationProviderDefinition,
} from "#harness/instrumentation-lifecycle.js";
+import { instrumentationStateSlot } from "#harness/instrumentation-state.js";
+
+const { logWarn } = vi.hoisted(() => ({ logWarn: vi.fn() }));
+vi.mock("#internal/logging.js", () => ({
+ createLogger: () => ({ warn: logWarn }),
+ formatError: (error: unknown) => error,
+}));
const scope: InstrumentationAttemptScope = {
attemptId: "session-1:turn-1:0:0",
@@ -18,6 +30,14 @@ const scope: InstrumentationAttemptScope = {
turnId: "turn-1",
};
+function deferred(): { readonly promise: Promise<void>; readonly resolve: () => void } {
+ let resolve!: () => void;
+ const promise = new Promise<void>((complete) => {
+ resolve = complete;
+ });
+ return { promise, resolve };
+}
+
describe("instrumentation idempotency keys", () => {
it("separates classes that share an identifier", () => {
expect(sessionIdempotencyKey("shared")).not.toBe(turnIdempotencyKey("shared", "shared"));
@@ -48,3 +68,682 @@ describe("instrumentation idempotency keys", () => {
expect(modelCallIdempotencyKey(scope, 0)).not.toBe(toolCallIdempotencyKey(scope, "call-1", 0));
});
});
+
+describe("provider state lifecycle", () => {
+ const started = {
+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),
+ rootSessionId: "session-1",
+ sequence: 0,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "turn.started" as const,
+ };
+ const completed = {
+ idempotencyKey: started.idempotencyKey,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "turn.completed" as const,
+ };
+
+ it("carries a value from a start to its terminal and releases it", async () => {
+ let observed: unknown;
+ const hooks = createInstrumentationHooks([
+ {
+ events: {
+ "turn.completed": (_event, ctx) => {
+ observed = ctx.state.get();
+ },
+ "turn.started": (_event, ctx) => ctx.state.set({ rowId: "row-1" }),
+ },
+ name: "sink",
+ },
+ ]);
+
+ await contextStorage.run(new ContextContainer(), async () => {
+ await hooks.publish(started);
+ await hooks.publish(completed);
+ expect(instrumentationStateSlot("sink", started.idempotencyKey).get()).toBeUndefined();
+ });
+ expect(observed).toEqual({ rowId: "row-1" });
+ });
+
+ it("releases state when no terminal handler exists", async () => {
+ const hooks = createInstrumentationHooks([
+ { events: { "turn.started": (_event, ctx) => ctx.state.set("open") }, name: "sink" },
+ ]);
+ await contextStorage.run(new ContextContainer(), async () => {
+ await hooks.publish(started);
+ await hooks.publish(completed);
+ expect(instrumentationStateSlot("sink", started.idempotencyKey).get()).toBeUndefined();
+ });
+ });
+
+ it("releases terminal state after its provider is removed", async () => {
+ const starts = createInstrumentationHooks([
+ { events: { "turn.started": (_event, ctx) => ctx.state.set("open") }, name: "removed" },
+ ]);
+ const terminals = createInstrumentationHooks([]);
+ await contextStorage.run(new ContextContainer(), async () => {
+ await starts.publish(started);
+ await terminals.publish(completed);
+ expect(instrumentationStateSlot("removed", started.idempotencyKey).get()).toBeUndefined();
+ });
+ });
+
+ it("keeps input state past its attempt and restores it for resolution", async () => {
+ const key = inputIdempotencyKey(scope.sessionId, scope.turnId, "request-1");
+ const context = new ContextContainer();
+ const starts = createInstrumentationHooks([
+ {
+ events: { "input.requested": (_event, ctx) => ctx.state.set({ rowId: "input-1" }) },
+ name: "sink",
+ },
+ ]);
+ await contextStorage.run(context, async () => {
+ await starts.publish({
+ action: { callId: "call-1", name: "weather" },
+ idempotencyKey: key,
+ kind: "tool-approval",
+ request: { prompt: "Approve weather?" },
+ requestId: "request-1",
+ scope,
+ type: "input.requested",
+ });
+ await starts.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ scope,
+ type: "step.attempt.completed",
+ });
+ });
+
+ const observed: unknown[] = [];
+ const restored = await deserializeContext(await serializeContext(context));
+ const terminals = createInstrumentationHooks([
+ {
+ events: { "input.resolved": (_event, ctx) => void observed.push(ctx.state.get()) },
+ name: "sink",
+ },
+ ]);
+ await contextStorage.run(restored, async () => {
+ await terminals.publish({
+ idempotencyKey: key,
+ kind: "tool-approval",
+ outcome: "approved",
+ requestId: "request-1",
+ response: { optionId: "approve" },
+ scope,
+ type: "input.resolved",
+ });
+ expect(instrumentationStateSlot("sink", key).get()).toBeUndefined();
+ });
+ expect(observed).toEqual([{ rowId: "input-1" }]);
+ });
+
+ it("releases unterminated model state when its attempt ends", async () => {
+ const modelKey = modelCallIdempotencyKey(scope, 0);
+ const hooks = createInstrumentationHooks([
+ {
+ events: { "model.call.started": (_event, ctx) => ctx.state.set("open") },
+ name: "sink",
+ },
+ ]);
+ await contextStorage.run(new ContextContainer(), async () => {
+ await hooks.publish({
+ idempotencyKey: modelKey,
+ input: { messages: [] },
+ model: { modelId: "model", provider: "test" },
+ scope,
+ type: "model.call.started",
+ });
+ await hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ scope,
+ type: "step.attempt.completed",
+ });
+ expect(instrumentationStateSlot("sink", modelKey).get()).toBeUndefined();
+ });
+ });
+
+ it("releases attempt-owned state after its provider is removed", async () => {
+ const modelKey = modelCallIdempotencyKey(scope, 0);
+ const starts = createInstrumentationHooks([
+ {
+ events: { "model.call.started": (_event, ctx) => ctx.state.set("open") },
+ name: "removed",
+ },
+ ]);
+ const terminals = createInstrumentationHooks([]);
+ await contextStorage.run(new ContextContainer(), async () => {
+ await starts.publish({
+ idempotencyKey: modelKey,
+ input: { messages: [] },
+ model: { modelId: "model", provider: "test" },
+ scope,
+ type: "model.call.started",
+ });
+ await terminals.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ scope,
+ type: "step.attempt.completed",
+ });
+ expect(instrumentationStateSlot("removed", modelKey).get()).toBeUndefined();
+ });
+ });
+});
+
+describe("provider handler deadlines", () => {
+ const key = turnIdempotencyKey("session-1", "turn-1");
+ const started = {
+ idempotencyKey: key,
+ rootSessionId: "session-1",
+ sequence: 0,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "turn.started" as const,
+ };
+ const completed = {
+ idempotencyKey: key,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "turn.completed" as const,
+ };
+ const hang = () => new Promise<void>(() => {});
+
+ it("lets providers behind a hanging handler run", async () => {
+ const after = vi.fn();
+ const hooks = createInstrumentationHooks(
+ [
+ { events: { "turn.started": hang }, name: "hangs" },
+ { events: { "turn.started": after }, name: "after" },
+ ],
+ { handlerTimeoutMs: 1 },
+ );
+ await contextStorage.run(new ContextContainer(), () => hooks.publish(started));
+ expect(after).toHaveBeenCalledOnce();
+ });
+
+ it("persists abandonment across workers and skips the terminal", async () => {
+ const terminal = vi.fn();
+ const context = new ContextContainer();
+ const starts = createInstrumentationHooks(
+ [{ events: { "turn.started": hang }, name: "sink" }],
+ { handlerTimeoutMs: 1 },
+ );
+ await contextStorage.run(context, () => starts.publish(started));
+
+ const restored = await deserializeContext(await serializeContext(context));
+ const terminals = createInstrumentationHooks(
+ [{ events: { "turn.completed": terminal }, name: "sink" }],
+ { handlerTimeoutMs: 1 },
+ );
+ await contextStorage.run(restored, () => terminals.publish(completed));
+ expect(terminal).not.toHaveBeenCalled();
+ });
+
+ it("abandons a timed-out input request across workers", async () => {
+ const key = inputIdempotencyKey(scope.sessionId, scope.turnId, "request-1");
+ const context = new ContextContainer();
+ const starts = createInstrumentationHooks(
+ [{ events: { "input.requested": hang }, name: "sink" }],
+ { handlerTimeoutMs: 1 },
+ );
+ await contextStorage.run(context, () =>
+ starts.publish({
+ action: { callId: "call-1", name: "weather" },
+ idempotencyKey: key,
+ kind: "tool-approval",
+ request: { prompt: "Approve weather?" },
+ requestId: "request-1",
+ scope,
+ type: "input.requested",
+ }),
+ );
+
+ const terminal = vi.fn();
+ const restored = await deserializeContext(await serializeContext(context));
+ const terminals = createInstrumentationHooks([
+ { events: { "input.resolved": terminal }, name: "sink" },
+ ]);
+ await contextStorage.run(restored, () =>
+ terminals.publish({
+ idempotencyKey: key,
+ kind: "tool-approval",
+ outcome: "approved",
+ requestId: "request-1",
+ scope,
+ type: "input.resolved",
+ }),
+ );
+ expect(terminal).not.toHaveBeenCalled();
+ });
+
+ it("ignores a timed-out handler's late state write", async () => {
+ let resume!: () => void;
+ const continued = new Promise<void>((resolve) => {
+ resume = resolve;
+ });
+ const hooks = createInstrumentationHooks(
+ [
+ {
+ events: {
+ "turn.started": async (_event, ctx) => {
+ await continued;
+ ctx.state.set("late");
+ },
+ },
+ name: "slow",
+ },
+ ],
+ { handlerTimeoutMs: 1 },
+ );
+ await contextStorage.run(new ContextContainer(), async () => {
+ await hooks.publish(started);
+ await hooks.publish(completed);
+ resume();
+ await continued;
+ await Promise.resolve();
+ expect(instrumentationStateSlot("slow", key).get()).toBeUndefined();
+ });
+ });
+
+ it("does not abandon an operation for a point-event timeout", async () => {
+ const terminal = vi.fn();
+ const hooks = createInstrumentationHooks(
+ [
+ {
+ events: {
+ "step.attempt.completed": terminal,
+ "step.attempt.metadata": hang,
+ },
+ name: "slow-metadata",
+ },
+ ],
+ { handlerTimeoutMs: 1 },
+ );
+ await contextStorage.run(new ContextContainer(), async () => {
+ await hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ providerMetadata: {},
+ scope,
+ type: "step.attempt.metadata",
+ });
+ await hooks.publish({
+ idempotencyKey: attemptIdempotencyKey(scope),
+ scope,
+ type: "step.attempt.completed",
+ });
+ });
+ expect(terminal).toHaveBeenCalledOnce();
+ expect(logWarn).toHaveBeenCalledWith(
+ "instrumentation provider timed out",
+ expect.objectContaining({ boundary: "step.attempt.metadata", provider: "slow-metadata" }),
+ );
+ });
+});
+
+describe("provider dispatch groups", () => {
+ const started = {
+ idempotencyKey: turnIdempotencyKey("session-1", "turn-1"),
+ rootSessionId: "session-1",
+ sequence: 0,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "turn.started" as const,
+ };
+ const completed = {
+ idempotencyKey: started.idempotencyKey,
+ sessionId: "session-1",
+ turnId: "turn-1",
+ type: "turn.completed" as const,
+ };
+
+ it("starts every parallel handler before either is released", async () => {
+ const first = deferred();
+ const second = deferred();
+ const order: string[] = [];
+ const hooks = createInstrumentationHooks({
+ parallel: [
+ {
+ events: {
+ "turn.started": async () => {
+ order.push("first:start");
+ await first.promise;
+ order.push("first:end");
+ },
+ },
+ name: "first",
+ },
+ {
+ events: {
+ "turn.started": async () => {
+ order.push("second:start");
+ await second.promise;
+ order.push("second:end");
+ },
+ },
+ name: "second",
+ },
+ ],
+ });
+
+ await contextStorage.run(new ContextContainer(), async () => {
+ const publication = hooks.publish(started);
+ expect(order).toEqual(["first:start", "second:start"]);
+ first.resolve();
+ second.resolve();
+ await publication;
+ });
+ });
+
+ it("publishes one immutable snapshot to every dispatch phase", async () => {
+ const observed: unknown[] = [];
+ const mutableScope = { ...scope };
+ const event: InstrumentationModelCallStartedEvent = {
+ idempotencyKey: modelCallIdempotencyKey(mutableScope, 0),
+ input: { messages: [{ content: "private", role: "user" }] },
+ model: { modelId: "model", provider: "test" },
+ scope: mutableScope,
+ type: "model.call.started" as const,
+ };
+ const mutator = (snapshot: InstrumentationModelCallStartedEvent): void => {
+ expect(Object.isFrozen(snapshot)).toBe(true);
+ expect(Object.isFrozen(snapshot.scope)).toBe(true);
+ expect(Object.isFrozen(snapshot.input)).toBe(true);
+ expect(Reflect.set(snapshot, "idempotencyKey", "corrupted")).toBe(false);
+ expect(Reflect.set(snapshot.scope, "turnId", "corrupted")).toBe(false);
+ };
+ const observer = (snapshot: InstrumentationModelCallStartedEvent): void => {
+ observed.push(snapshot.idempotencyKey, snapshot.scope.turnId, snapshot.type);
+ };
+ const hooks = createInstrumentationHooks({
+ parallel: [
+ { events: { "model.call.started": mutator }, name: "mutator" },
+ { events: { "model.call.started": observer }, name: "observer" },
+ ],
+ serialAfter: [{ events: { "model.call.started": observer }, name: "cleanup" }],
+ serialBefore: [{ events: { "model.call.started": observer }, name: "framework" }],
+ });
+
+ await contextStorage.run(new ContextContainer(), () => hooks.publish(event));
+
+ expect(observed).toEqual([
+ event.idempotencyKey,
+ scope.turnId,
+ event.type,
+ event.idempotencyKey,
+ scope.turnId,
+ event.type,
+ event.idempotencyKey,
+ scope.turnId,
+ event.type,
+ ]);
+ expect(Object.isFrozen(event)).toBe(false);
+ expect(mutableScope.turnId).toBe(scope.turnId);
+ });
+
+ it("reuses the frozen scope snapshot across publications", async () => {
+ const observedScopes: InstrumentationAttemptScope[] = [];
+ const hooks = createInstrumentationHooks([
+ {
+ events: {
+ "model.call.completed": (event) => void observedScopes.push(event.scope),
+ "model.call.started": (event) => void observedScopes.push(event.scope),
+ },
+ name: "observer",
+ },
+ ]);
+ const sharedScope = { ...scope };
+
+ await contextStorage.run(new ContextContainer(), async () => {
+ await hooks.publish({
+ idempotencyKey: modelCallIdempotencyKey(sharedScope, 0),
+ input: { messages: [] },
+ model: { modelId: "model", provider: "test" },
+ scope: sharedScope,
+ type: "model.call.started",
+ });
+ await hooks.publish({
+ content: [],
+ finishReason: "stop",
+ idempotencyKey: modelCallIdempotencyKey(sharedScope, 0),
+ scope: sharedScope,
+ type: "model.call.completed",
+ usage: {},
+ });
+ });
+
+ expect(observedScopes).toHaveLength(2);
+ expect(observedScopes[0]).toBe(observedScopes[1]);
+ expect(observedScopes[0]).not.toBe(sharedScope);
+ expect(Object.isFrozen(observedScopes[0])).toBe(true);
+ });
+
+ it("uses one timeout window for concurrent handlers", async () => {
+ vi.useFakeTimers();
+ const fast = vi.fn();
+ const hooks = createInstrumentationHooks(
+ {
+ parallel: [
+ { events: { "turn.started": () => new Promise<void>(() => {}) }, name: "hangs-one" },
+ { events: { "turn.started": fast }, name: "fast" },
+ { events: { "turn.started": () => new Promise<void>(() => {}) }, name: "hangs-two" },
+ ],
+ },
+ { handlerTimeoutMs: 10 },
+ );
+
+ try {
+ await contextStorage.run(new ContextContainer(), async () => {
+ const publication = hooks.publish(started);
+ let settled = false;
+ void publication.then(() => {
+ settled = true;
+ });
+ expect(fast).toHaveBeenCalledOnce();
+ await vi.advanceTimersByTimeAsync(9);
+ expect(settled).toBe(false);
+ await vi.advanceTimersByTimeAsync(1);
+ await publication;
+ expect(settled).toBe(true);
+ });
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("continues parallel delivery when another handler throws", async () => {
+ const received = vi.fn();
+ const hooks = createInstrumentationHooks({
+ parallel: [
+ {
+ events: {
+ "turn.started": () => {
+ throw new Error("expected provider failure");
+ },
+ },
+ name: "throws",
+ },
+ { events: { "turn.started": received }, name: "receives" },
+ ],
+ });
+
+ await contextStorage.run(new ContextContainer(), () => hooks.publish(started));
+ expect(received).toHaveBeenCalledOnce();
+ });
+
+ it("runs serial phases around the parallel publication", async () => {
+ vi.useFakeTimers();
+ const before = deferred();
+ const order: string[] = [];
+ const hooks = createInstrumentationHooks(
+ {
+ parallel: [
+ {
+ events: {
+ "turn.started": () => {
+ order.push("parallel");
+ return new Promise<void>(() => {});
+ },
+ },
+ name: "parallel",
+ },
+ ],
+ serialAfter: [
+ { events: { "turn.started": () => void order.push("after") }, name: "after" },
+ ],
+ serialBefore: [
+ {
+ events: {
+ "turn.started": async () => {
+ order.push("before:start");
+ await before.promise;
+ order.push("before:end");
+ },
+ },
+ name: "before",
+ },
+ ],
+ },
+ { handlerTimeoutMs: 10 },
+ );
+
+ try {
+ await contextStorage.run(new ContextContainer(), async () => {
+ const publication = hooks.publish(started);
+ expect(order).toEqual(["before:start"]);
+ before.resolve();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(order).toEqual(["before:start", "before:end", "parallel"]);
+ await vi.advanceTimersByTimeAsync(10);
+ await publication;
+ });
+ expect(order).toEqual(["before:start", "before:end", "parallel", "after"]);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("runs serial cleanup before rethrowing an unexpected parallel rejection", async () => {
+ const failure = new Error("framework state failed");
+ const cleanup = vi.fn();
+ const broken = Object.defineProperty({ name: "broken" }, "events", {
+ get: () => {
+ throw failure;
+ },
+ }) as InstrumentationProviderDefinition;
+ const hooks = createInstrumentationHooks({
+ parallel: [broken],
+ serialAfter: [{ events: { "turn.started": cleanup }, name: "cleanup" }],
+ });
+
+ await contextStorage.run(new ContextContainer(), async () => {
+ await expect(hooks.publish(started)).rejects.toBe(failure);
+ });
+ expect(cleanup).toHaveBeenCalledOnce();
+ });
+
+ it("keeps durable state isolated between parallel providers", async () => {
+ const observed = new Map<string, unknown>();
+ const hooks = createInstrumentationHooks({
+ parallel: [
+ {
+ events: {
+ "turn.completed": (_event, ctx) => {
+ observed.set("first", ctx.state.get());
+ },
+ "turn.started": (_event, ctx) => ctx.state.set("one"),
+ },
+ name: "first",
+ },
+ {
+ events: {
+ "turn.completed": (_event, ctx) => {
+ observed.set("second", ctx.state.get());
+ },
+ "turn.started": (_event, ctx) => ctx.state.set("two"),
+ },
+ name: "second",
+ },
+ ],
+ });
+
+ await contextStorage.run(new ContextContainer(), async () => {
+ await hooks.publish(started);
+ await hooks.publish(completed);
+ });
+ expect(observed).toEqual(
+ new Map([
+ ["first", "one"],
+ ["second", "two"],
+ ]),
+ );
+ });
+
+ it("suppresses only a timed-out provider terminal and revokes its late write", async () => {
+ vi.useFakeTimers();
+ const continuation = deferred();
+ const unaffectedTerminal = vi.fn();
+ const timedOutTerminal = vi.fn();
+ const hooks = createInstrumentationHooks(
+ {
+ parallel: [
+ {
+ events: {
+ "turn.completed": timedOutTerminal,
+ "turn.started": async (_event, ctx) => {
+ await continuation.promise;
+ ctx.state.set("late");
+ },
+ },
+ name: "times-out",
+ },
+ { events: { "turn.completed": unaffectedTerminal }, name: "unaffected" },
+ ],
+ },
+ { handlerTimeoutMs: 10 },
+ );
+
+ try {
+ await contextStorage.run(new ContextContainer(), async () => {
+ const publication = hooks.publish(started);
+ await vi.advanceTimersByTimeAsync(10);
+ await publication;
+ await hooks.publish(completed);
+ continuation.resolve();
+ await continuation.promise;
+ await Promise.resolve();
+ expect(instrumentationStateSlot("times-out", started.idempotencyKey).get()).toBeUndefined();
+ });
+ } finally {
+ vi.useRealTimers();
+ }
+ expect(timedOutTerminal).not.toHaveBeenCalled();
+ expect(unaffectedTerminal).toHaveBeenCalledOnce();
+ });
+
+ it("keeps array input sequential", async () => {
+ const first = deferred();
+ const order: string[] = [];
+ const hooks = createInstrumentationHooks([
+ {
+ events: {
+ "turn.started": async () => {
+ order.push("first:start");
+ await first.promise;
+ order.push("first:end");
+ },
+ },
+ name: "first",
+ },
+ { events: { "turn.started": () => void order.push("second") }, name: "second" },
+ ]);
+
+ await contextStorage.run(new ContextContainer(), async () => {
+ const publication = hooks.publish(started);
+ expect(order).toEqual(["first:start"]);
+ first.resolve();
+ await publication;
+ });
+ expect(order).toEqual(["first:start", "first:end", "second"]);
+ });
+});
diff --git a/packages/eve/src/harness/instrumentation-lifecycle.ts b/packages/eve/src/harness/instrumentation-lifecycle.ts
index 5f78e03e2..bb7df010a 100644
--- a/packages/eve/src/harness/instrumentation-lifecycle.ts
+++ b/packages/eve/src/harness/instrumentation-lifecycle.ts
@@ -1,4 +1,5 @@
-import { createLogger, formatError } from "#internal/logging.js";
+import { createInstrumentationDispatcher } from "#harness/instrumentation-dispatch.js";
+import type { InstrumentationStateSlot } from "#harness/instrumentation-state.js";
/**
* Stable eve identity for one actual model attempt.
@@ -345,16 +346,23 @@ export type InstrumentationToolCallTerminalEvent =
| InstrumentationToolCallCompletedEvent
| InstrumentationToolCallFailedEvent;
+export interface InstrumentationHandlerContext {
+ readonly state: InstrumentationStateSlot;
+}
+
/**
* The AI SDK can omit a model terminal when an incomplete stream closes. A
* provider that correlates starts with terminals must scope that state to the
* attempt and release anything still open when the step attempt terminates.
*/
-export type InstrumentationEventHandler<TEvent> = (event: TEvent) => void | PromiseLike<void>;
+export type InstrumentationEventHandler<TEvent> = (
+ event: TEvent,
+ ctx: InstrumentationHandlerContext,
+) => void | PromiseLike<void>;
/** Internal provider shape mirrored by the future public hook contract. */
export interface InstrumentationProviderDefinition {
- readonly name?: string;
+ readonly name: string;
readonly events?: {
readonly "step.attempt.started"?: InstrumentationEventHandler<InstrumentationStepAttemptStartedEvent>;
readonly "step.attempt.completed"?: InstrumentationEventHandler<InstrumentationStepAttemptCompletedEvent>;
@@ -381,6 +389,20 @@ export interface InstrumentationProviderDefinition {
readonly shutdown?: () => void | PromiseLike<void>;
}
+type InstrumentationProviderInput = Omit<InstrumentationProviderDefinition, "name"> & {
+ readonly name?: string;
+};
+
+export interface InstrumentationDispatchGroups {
+ readonly serialBefore?: readonly InstrumentationProviderDefinition[];
+ readonly parallel?: readonly InstrumentationProviderDefinition[];
+ readonly serialAfter?: readonly InstrumentationProviderDefinition[];
+}
+
+export type InstrumentationHooksInput =
+ | readonly InstrumentationProviderInput[]
+ | InstrumentationDispatchGroups;
+
/** Events that pair a start with its terminal under one `idempotencyKey`. */
export type InstrumentationCorrelatedEvent =
| InstrumentationInputRequestedEvent
@@ -425,26 +447,14 @@ export interface InstrumentationHooks {
publish(event: InstrumentationEvent): Promise<void>;
}
-const log = createLogger("harness.instrumentation-lifecycle");
+export interface CreateInstrumentationHooksOptions {
+ readonly handlerTimeoutMs?: number;
+}
-/** Creates failure-isolated hooks backed by an ordered provider list. */
+/** Creates failure-isolated hooks backed by normalized dispatch groups. */
export function createInstrumentationHooks(
- providers: readonly InstrumentationProviderDefinition[],
+ input: InstrumentationHooksInput,
+ options: CreateInstrumentationHooksOptions = {},
): InstrumentationHooks {
- const publish = async (event: InstrumentationEvent): Promise<void> => {
- for (const provider of providers) {
- const handler = provider.events?.[event.type];
- if (handler === undefined) continue;
- try {
- await (handler as InstrumentationEventHandler<InstrumentationEvent>)(event);
- } catch (error) {
- log.warn("instrumentation provider failed", {
- boundary: event.type,
- error: formatError(error),
- });
- }
- }
- };
-
- return { publish };
+ return createInstrumentationDispatcher(input, options);
}
diff --git a/packages/eve/src/harness/instrumentation-state.test.ts b/packages/eve/src/harness/instrumentation-state.test.ts
new file mode 100644
index 000000000..5659fbd5b
--- /dev/null
+++ b/packages/eve/src/harness/instrumentation-state.test.ts
@@ -0,0 +1,106 @@
+import { describe, expect, it } from "vitest";
+
+import { ContextContainer, contextStorage } from "#context/container.js";
+import { deserializeContext, serializeContext } from "#context/serialize.js";
+import {
+ abandonInstrumentationState,
+ instrumentationStateSlot,
+ isInstrumentationStateAbandoned,
+ preserveSerializedInstrumentationState,
+ releaseAllInstrumentationAttemptState,
+ releaseAllInstrumentationState,
+} from "#harness/instrumentation-state.js";
+
+describe("instrumentation state", () => {
+ it("survives a serialized step boundary", async () => {
+ const context = new ContextContainer();
+ contextStorage.run(context, () => {
+ instrumentationStateSlot("sink", "model:1", "attempt-1").set({ rowId: "row-1" });
+ });
+ const restored = await deserializeContext(await serializeContext(context));
+ contextStorage.run(restored, () => {
+ expect(instrumentationStateSlot("sink", "model:1").get()).toEqual({ rowId: "row-1" });
+ });
+ });
+
+ it("isolates providers and operations", () => {
+ contextStorage.run(new ContextContainer(), () => {
+ instrumentationStateSlot("a", "turn:1").set("a-1");
+ instrumentationStateSlot("b", "turn:1").set("b-1");
+ instrumentationStateSlot("a", "turn:2").set("a-2");
+ expect(instrumentationStateSlot("a", "turn:1").get()).toBe("a-1");
+ expect(instrumentationStateSlot("b", "turn:1").get()).toBe("b-1");
+ expect(instrumentationStateSlot("a", "turn:2").get()).toBe("a-2");
+ });
+ });
+
+ it("rejects values that cannot survive serialization", () => {
+ contextStorage.run(new ContextContainer(), () => {
+ expect(() => instrumentationStateSlot("sink", "turn:1").set(new Date() as never)).toThrow(
+ TypeError,
+ );
+ });
+ });
+
+ it("revokes late reads and writes", () => {
+ contextStorage.run(new ContextContainer(), () => {
+ const lease = instrumentationStateSlot("sink", "turn:1");
+ lease.set("before");
+ lease.revoke();
+ lease.set("after");
+ expect(lease.get()).toBeUndefined();
+ expect(instrumentationStateSlot("sink", "turn:1").get()).toBe("before");
+ });
+ });
+
+ it("does not expose mutable durable state through a retained read", () => {
+ contextStorage.run(new ContextContainer(), () => {
+ const original = { nested: { count: 1 } };
+ const lease = instrumentationStateSlot("sink", "turn:1");
+ lease.set(original);
+ original.nested.count = 2;
+
+ const retained = lease.get() as { nested: { count: number } };
+ lease.revoke();
+
+ expect(Object.isFrozen(retained)).toBe(true);
+ expect(Object.isFrozen(retained.nested)).toBe(true);
+ expect(Reflect.set(retained.nested, "count", 3)).toBe(false);
+ expect(instrumentationStateSlot("sink", "turn:1").get()).toEqual({ nested: { count: 1 } });
+ });
+ });
+
+ it("releases exact and attempt-owned state", () => {
+ contextStorage.run(new ContextContainer(), () => {
+ instrumentationStateSlot("sink", "model:1", "attempt-1").set("one");
+ instrumentationStateSlot("sink", "model:2", "attempt-2").set("two");
+ releaseAllInstrumentationState("model:2");
+ releaseAllInstrumentationAttemptState("attempt-1");
+ expect(instrumentationStateSlot("sink", "model:1").get()).toBeUndefined();
+ expect(instrumentationStateSlot("sink", "model:2").get()).toBeUndefined();
+ });
+ });
+
+ it("preserves state from a discarded step", () => {
+ expect(
+ preserveSerializedInstrumentationState(
+ { authored: "original" },
+ { "eve.harness.instrumentationState": { state: true } },
+ ),
+ ).toEqual({
+ authored: "original",
+ "eve.harness.instrumentationState": { state: true },
+ });
+ });
+
+ it("persists abandonment across serialization", async () => {
+ const context = new ContextContainer();
+ contextStorage.run(context, () => {
+ abandonInstrumentationState("sink", "model:1", "attempt-1");
+ });
+ const restored = await deserializeContext(await serializeContext(context));
+ contextStorage.run(restored, () => {
+ expect(isInstrumentationStateAbandoned("sink", "model:1")).toBe(true);
+ });
+ });
+});
diff --git a/packages/eve/src/harness/instrumentation-state.ts b/packages/eve/src/harness/instrumentation-state.ts
new file mode 100644
index 000000000..18ee462f3
--- /dev/null
+++ b/packages/eve/src/harness/instrumentation-state.ts
@@ -0,0 +1,176 @@
+import { contextStorage, loadContext } from "#context/container.js";
+import { ContextKey } from "#context/key.js";
+import { type JsonValue, parseJsonValue } from "#shared/json.js";
+
+interface InstrumentationStateRecord {
+ abandoned?: true;
+ attemptId?: string;
+ value?: JsonValue;
+}
+
+type InstrumentationStateMap = Readonly<Record<string, InstrumentationStateRecord>>;
+
+const InstrumentationStateKey = new ContextKey<InstrumentationStateMap>(
+ "eve.harness.instrumentationState",
+ {
+ codec: {
+ deserialize: deserializeState,
+ serialize: (state) => state,
+ },
+ },
+);
+
+/** Keeps provider state from an interrupted step's discarded context changes. */
+export function preserveSerializedInstrumentationState(
+ original: Record<string, unknown>,
+ interrupted: Record<string, unknown>,
+): Record<string, unknown> {
+ const state = interrupted[InstrumentationStateKey.name];
+ return state === undefined ? original : { ...original, [InstrumentationStateKey.name]: state };
+}
+
+export interface InstrumentationStateSlot {
+ get(): JsonValue | undefined;
+ set(value: JsonValue | undefined): void;
+}
+
+export interface InstrumentationStateLease extends InstrumentationStateSlot {
+ revoke(): void;
+}
+
+/** One provider's durable state for one operation. */
+export function instrumentationStateSlot(
+ provider: string,
+ idempotencyKey: string,
+ attemptId?: string,
+): InstrumentationStateLease {
+ const key = stateKey(provider, idempotencyKey);
+ let active = true;
+ return {
+ get: () => {
+ if (!active) return undefined;
+ const value = contextStorage.getStore()?.get(InstrumentationStateKey)?.[key]?.value;
+ return value === undefined ? undefined : cloneAndFreezeJson(value);
+ },
+ revoke: () => {
+ active = false;
+ },
+ set: (value) => {
+ if (!active || contextStorage.getStore() === undefined) return;
+ if (value === undefined) {
+ writeState((state) => writeSlot(state, key, undefined));
+ return;
+ }
+ const record: InstrumentationStateRecord = {
+ value: cloneAndFreezeJson(parseJsonValue(value)),
+ };
+ const current = contextStorage.getStore()?.get(InstrumentationStateKey)?.[key];
+ if (current?.abandoned === true) record.abandoned = true;
+ if (attemptId !== undefined) record.attemptId = attemptId;
+ writeState((state) => writeSlot(state, key, record));
+ },
+ };
+}
+
+export function abandonInstrumentationState(
+ provider: string,
+ idempotencyKey: string,
+ attemptId?: string,
+): void {
+ if (contextStorage.getStore() === undefined) return;
+ const key = stateKey(provider, idempotencyKey);
+ writeState((state) => {
+ const current = state[key];
+ const record: InstrumentationStateRecord = { abandoned: true };
+ const owner = attemptId ?? current?.attemptId;
+ if (owner !== undefined) record.attemptId = owner;
+ if (current?.value !== undefined) record.value = current.value;
+ return writeSlot(state, key, record);
+ });
+}
+
+export function isInstrumentationStateAbandoned(provider: string, idempotencyKey: string): boolean {
+ return (
+ contextStorage.getStore()?.get(InstrumentationStateKey)?.[stateKey(provider, idempotencyKey)]
+ ?.abandoned === true
+ );
+}
+
+/** Releases one operation across namespaces, including providers no longer registered. */
+export function releaseAllInstrumentationState(idempotencyKey: string): void {
+ const suffix = `\0${idempotencyKey}`;
+ releaseMatchingInstrumentationState((key) => key.endsWith(suffix));
+}
+
+/** Releases attempt-owned children across namespaces when their terminals are omitted. */
+export function releaseAllInstrumentationAttemptState(attemptId: string): void {
+ releaseMatchingInstrumentationState((_key, record) => record.attemptId === attemptId);
+}
+
+function releaseMatchingInstrumentationState(
+ matches: (key: string, record: InstrumentationStateRecord) => boolean,
+): void {
+ const current = contextStorage.getStore()?.get(InstrumentationStateKey);
+ if (
+ current === undefined ||
+ !Object.entries(current).some(([key, record]) => matches(key, record))
+ ) {
+ return;
+ }
+ writeState((state) => {
+ const next = { ...state };
+ for (const [key, record] of Object.entries(state)) {
+ if (matches(key, record)) delete next[key];
+ }
+ return next;
+ });
+}
+
+function writeState(update: (state: InstrumentationStateMap) => InstrumentationStateMap): void {
+ loadContext().set(InstrumentationStateKey, (state) => update(state ?? {}));
+}
+
+function writeSlot(
+ state: InstrumentationStateMap,
+ key: string,
+ value: InstrumentationStateRecord | undefined,
+): InstrumentationStateMap {
+ const next = { ...state };
+ if (value === undefined) delete next[key];
+ else next[key] = value;
+ return next;
+}
+
+function stateKey(provider: string, idempotencyKey: string): string {
+ return `${provider}\0${idempotencyKey}`;
+}
+
+function deserializeState(data: unknown): InstrumentationStateMap {
+ if (typeof data !== "object" || data === null || Array.isArray(data)) return {};
+ const state: Record<string, InstrumentationStateRecord> = {};
+ for (const [key, value] of Object.entries(data)) {
+ if (typeof value !== "object" || value === null || Array.isArray(value)) continue;
+ const record = value as Record<string, unknown>;
+ const parsed: InstrumentationStateRecord = {};
+ if (record["abandoned"] === true) parsed.abandoned = true;
+ if (typeof record["attemptId"] === "string") parsed.attemptId = record["attemptId"];
+ if (record["value"] !== undefined) {
+ parsed.value = cloneAndFreezeJson(record["value"] as JsonValue);
+ }
+ if (parsed.abandoned === true || parsed.value !== undefined) state[key] = parsed;
+ }
+ return state;
+}
+
+function cloneAndFreezeJson(value: JsonValue): JsonValue {
+ if (Array.isArray(value)) {
+ return Object.freeze(value.map((entry) => cloneAndFreezeJson(entry)));
+ }
+ if (typeof value !== "object" || value === null) return value;
+
+ const copy: Record<string, JsonValue> = {};
+ for (const [key, entry] of Object.entries(value)) {
+ copy[key] = cloneAndFreezeJson(entry);
+ }
+ return Object.freeze(copy);
+}
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index 90f3ebb6e..a780b2cd1 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -10242,6 +10242,7 @@ describe("createToolLoopHarness", () => {
scope: expect.objectContaining({ attemptIndex: 0 }),
type: "step.attempt.completed",
}),
+ expect.anything(),
);
});
diff --git a/packages/eve/src/tracing/agent-otel-provider.test.ts b/packages/eve/src/tracing/agent-otel-provider.test.ts
index 2f617df9d..a5b920303 100644
--- a/packages/eve/src/tracing/agent-otel-provider.test.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.test.ts
@@ -1,12 +1,13 @@
-import { SpanStatusCode } from "@opentelemetry/api";
+import { SpanStatusCode, trace as apiTrace } from "@opentelemetry/api";
import {
BasicTracerProvider,
InMemorySpanExporter,
SimpleSpanProcessor,
type ReadableSpan,
} from "@opentelemetry/sdk-trace-base";
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
+import { context } from "#compiled/@opentelemetry/api/index.js";
import { createAiSdkHookBridge } from "#harness/ai-sdk-hook-bridge.js";
import { createAgentOtelInstrumentation } from "#tracing/agent-otel-provider.js";
import { AgentSpanIdGenerator } from "#tracing/agent-span-id-generator.js";
@@ -263,6 +264,7 @@ function nanos(hrTime: readonly [number, number]): bigint {
describe("createAgentOtelInstrumentation", () => {
it("emits the agent hierarchy in one session trace", async () => {
const runtime = createRuntime();
+ const contextWith = vi.spyOn(context, "with");
await emitAttempt({
hooks: runtime.hooks,
runInContext: runtime.runInContext,
@@ -270,6 +272,8 @@ describe("createAgentOtelInstrumentation", () => {
turnId: "turn-1",
turnSequence: 0,
});
+ const executionParents = contextWith.mock.calls.map(([parent]) => parent);
+ contextWith.mockRestore();
await runtime.provider.forceFlush();
const spans = runtime.exporter.getFinishedSpans();
@@ -292,6 +296,12 @@ describe("createAgentOtelInstrumentation", () => {
expect(step.parentSpanContext?.spanId).toBe(turn.spanContext().spanId);
expect(operation.parentSpanContext?.spanId).toBe(step.spanContext().spanId);
expect(model.parentSpanContext?.spanId).toBe(operation.spanContext().spanId);
+ expect(
+ executionParents.some(
+ (parent) =>
+ apiTrace.getSpan(parent as never)?.spanContext().spanId === model.spanContext().spanId,
+ ),
+ ).toBe(true);
expect(action.parentSpanContext?.spanId).toBe(step.spanContext().spanId);
expect(tool.parentSpanContext?.spanId).toBe(action.spanContext().spanId);
expect(new Set(spans.map((span) => span.spanContext().traceId))).toHaveLength(1);
diff --git a/packages/eve/src/tracing/agent-otel-provider.ts b/packages/eve/src/tracing/agent-otel-provider.ts
index cf711b8b5..245f576a7 100644
--- a/packages/eve/src/tracing/agent-otel-provider.ts
+++ b/packages/eve/src/tracing/agent-otel-provider.ts
@@ -84,6 +84,7 @@ export function createAgentOtelInstrumentation(
InstrumentationAttemptScope,
{ readonly models: Map<string, Context>; readonly tools: Map<string, Context> }
>();
+ const attemptScopes = new Map<string, InstrumentationAttemptScope>();
// A serverless turn runs inside one `turnStep` "use step" invocation. If
// that worker is lost, Workflow retries the whole step from entry rather
// than resuming this callback sequence in a replacement process.
@@ -173,12 +174,15 @@ export function createAgentOtelInstrumentation(
},
step: { context: stepContext, span: stepSpan },
});
+ attemptScopes.set(event.scope.attemptId, event.scope);
};
const onStepTerminal = (event: InstrumentationStepAttemptTerminalEvent): void => {
- executionContexts.delete(event.scope);
- drainOpenSpans(event);
- const attempt = steps.get(event.scope);
+ const scope = attemptScopes.get(event.scope.attemptId) ?? event.scope;
+ executionContexts.delete(scope);
+ drainOpenSpans({ ...event, scope });
+ attemptScopes.delete(event.scope.attemptId);
+ const attempt = steps.get(scope);
if (attempt === undefined) return;
// The span event drops the `attempt` segment: this span *is* one attempt,
// and `agent.step.attempt` on it already says which.
@@ -191,7 +195,7 @@ export function createAgentOtelInstrumentation(
}
attempt.operation.span.end();
attempt.step.span.end();
- steps.delete(event.scope);
+ steps.delete(scope);
};
const onTurnTerminal = async (event: InstrumentationTurnTerminalEvent): Promise<void> => {
@@ -526,7 +530,8 @@ export function createAgentOtelInstrumentation(
},
},
runInContext(operation, execute) {
- const contexts = executionContexts.get(operation.scope);
+ const scope = attemptScopes.get(operation.scope.attemptId) ?? operation.scope;
+ const contexts = executionContexts.get(scope);
const parent =
operation.type === "model.call"
? contexts?.models.get(operation.idempotencyKey)
diff --git a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts
index bd0c07fb8..4d65b92a0 100644
--- a/packages/eve/src/tracing/install-instrumentation-runtime.test.ts
+++ b/packages/eve/src/tracing/install-instrumentation-runtime.test.ts
@@ -45,7 +45,7 @@ describe("installInstrumentationRuntime", () => {
const runtime = installInstrumentationRuntime({
collected: collectOtelPipeline([otelIntegration()]),
frameworkVersion: "test",
- providers: [{ flush: providerFlush, shutdown: providerShutdown }],
+ providers: [{ flush: providerFlush, name: "test", shutdown: providerShutdown }],
serviceName: "weather",
});