The Open Framework for Building Agents
fix(eve): dynamic tools - preserve approval replay (#275)
Andrew Barba30c59653010e
diff --git a/.changeset/guard-dynamic-approval-replay.md b/.changeset/guard-dynamic-approval-replay.mdnew file mode 100644index 000000000..e4152d5d4--- /dev/null+++ b/.changeset/guard-dynamic-approval-replay.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Preserve dynamic tool approval gates when session- and turn-scoped tools are replayed from durable metadata. If a replayed approval callback cannot be recovered, eve now requires approval by default instead of silently running the tool unguarded.diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/dynamic-guarded-echo.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/dynamic-guarded-echo.tsnew file mode 100644index 000000000..0d5262885--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/dynamic-guarded-echo.ts@@ -0,0 +1,27 @@+import { defineDynamic, defineTool } from "eve/tools";+import { z } from "zod";++export const DYNAMIC_GUARDED_ECHO_TOKEN = "dynamic-guarded-echo-ok-L8R6";++export default defineDynamic({+ events: {+ "session.started": async () => {+ return {+ dynamic_guarded_echo: defineTool({+ description:+ "Smoke-test fixture for replayed dynamic tool approval. Echoes the note input. Only call when the user explicitly asks for `dynamic_guarded_echo`.",+ inputSchema: z.object({+ note: z.string().optional(),+ }),+ needsApproval: () => true,+ async execute(input) {+ return {+ echoed: input.note ?? null,+ token: DYNAMIC_GUARDED_ECHO_TOKEN,+ };+ },+ }),+ };+ },+ },+});diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.tsnew file mode 100644index 000000000..a679ae09c--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.ts@@ -0,0 +1,49 @@+import { defineEval } from "eve/evals";+import type { HandleMessageStreamEvent } from "eve/client";++const DYNAMIC_GUARDED_ECHO_TOKEN = "dynamic-guarded-echo-ok-L8R6";+const TOOL_NAME = "dynamic_guarded_echo";++/**+ * HITL flow: a session-scoped dynamic tool's approval gate survives durable+ * replay. If replay drops `needsApproval`, the tool executes immediately and+ * this eval fails before approval.+ */+export default defineEval({+ description: "HITL smoke: replayed dynamic tools preserve needsApproval.",+ async test(t) {+ await t.send(`Call the \`${TOOL_NAME}\` tool with note "before-approval".`);+ const [request] = t.expectInputRequests({+ display: "confirmation",+ toolName: TOOL_NAME,+ });+ if (request === undefined) {+ throw new Error(`Expected ${TOOL_NAME} to park for approval.`);+ }+ if (dynamicGuardedEchoResults(t.events).length > 0) {+ throw new Error(`${TOOL_NAME} executed before approval.`);+ }++ const approved = await t.respondAll("approve");+ approved.expectOk();+ const results = dynamicGuardedEchoResults(t.events);+ if (results.length !== 1 || !results[0]!.includes(DYNAMIC_GUARDED_ECHO_TOKEN)) {+ throw new Error(`Approved ${TOOL_NAME} call did not execute with the expected token.`);+ }++ t.didNotFail();+ t.completed();+ },+});++function dynamicGuardedEchoResults(events: readonly HandleMessageStreamEvent[]): string[] {+ const results: string[] = [];+ for (const event of events) {+ if (event.type !== "action.result") continue;+ if (event.data.status === "rejected") continue;+ const result = event.data.result;+ if (result.kind !== "tool-result" || result.toolName !== TOOL_NAME) continue;+ results.push(JSON.stringify(result.output ?? ""));+ }+ return results;+}diff --git a/packages/eve/src/context/build-dynamic-tools.ts b/packages/eve/src/context/build-dynamic-tools.tsindex 7f090b567..5f1cc4a92 100644--- a/packages/eve/src/context/build-dynamic-tools.ts+++ b/packages/eve/src/context/build-dynamic-tools.ts@@ -10,6 +10,7 @@ import {import type { DurableDynamicToolMetadata } from "#context/keys.js";import { buildCallbackContext } from "#context/build-callback-context.js";import { createLogger } from "#internal/logging.js";+import type { NeedsApprovalContext } from "#public/definitions/tool.js";const log = createLogger("dynamic-tools");@@ -52,6 +53,7 @@ function replayTools(metadata: readonly DurableDynamicToolMetadata[]): HarnessToexecute: (input: unknown) => stepFn(m.closureVars, input, buildCallbackContext()),inputSchema: jsonSchema(m.inputSchema),name: m.name,+ needsApproval: buildReplayedNeedsApproval(m),outputSchema: m.outputSchema === undefined ? undefined : jsonSchema(m.outputSchema),});}@@ -59,6 +61,26 @@ function replayTools(metadata: readonly DurableDynamicToolMetadata[]): HarnessToreturn tools;}+function buildReplayedNeedsApproval(+ metadata: DurableDynamicToolMetadata,+): HarnessToolDefinition["needsApproval"] | undefined {+ if (metadata.needsApprovalStepFnName === undefined) {+ return undefined;+ }++ const approvalStepFn = lookupStepFunction(metadata.needsApprovalStepFnName);+ if (approvalStepFn === null) {+ log.warn(+ `Dynamic tool "${metadata.name}" references approval function "${metadata.needsApprovalStepFnName}" ` ++ "which is not registered — requiring approval by default.",+ );+ return () => true;+ }++ return (approvalCtx: NeedsApprovalContext) =>+ Boolean(approvalStepFn(metadata.closureVars ?? {}, approvalCtx));+}+/*** Builds live dynamic tool definitions. Narrower scopes appear first* so they win on name collision (the tool-loop uses `??=` for dedup).diff --git a/packages/eve/src/context/dynamic-tool-lifecycle.test.ts b/packages/eve/src/context/dynamic-tool-lifecycle.test.tsindex d8910b487..d25dfb23a 100644--- a/packages/eve/src/context/dynamic-tool-lifecycle.test.ts+++ b/packages/eve/src/context/dynamic-tool-lifecycle.test.ts@@ -863,6 +863,8 @@ describe("framework dynamic tools (no bundler transform)", () => {execute: async (): Promise<unknown> => ({ ok: true }),};const resolver = createResolver("connection", ["step.started"], () => ({ risky: entry }));+ testRegistry.delete("eve:framework-dynamic:connection:risky");+ testRegistry.delete("eve:dynamic-tool-approval:connection:risky");await dispatchDynamicToolEvent({ctx,@@ -882,6 +884,47 @@ describe("framework dynamic tools (no bundler transform)", () => {toolName: "risky",}),).toBe(true);+ expect(testRegistry.has("eve:framework-dynamic:connection:risky")).toBe(false);+ expect(testRegistry.has("eve:dynamic-tool-approval:connection:risky")).toBe(false);+ });++ it("replays needsApproval from session-scoped dynamic tools", async () => {+ const ctx = createCtx();+ const approvalFn = vi.fn(() => true);+ const entry: DynamicToolEntry = {+ description: "destructive op",+ inputSchema: { type: "object" },+ needsApproval: approvalFn,+ execute: async (): Promise<unknown> => ({ ok: true }),+ };+ const resolver = createResolver("session_guard", ["session.started"], () => ({+ guarded: entry,+ }));++ await dispatchDynamicToolEvent({+ ctx,+ resolvers: [resolver],+ messages: [],+ event: makeEvent("session.started"),+ });++ ctx.clearVirtualContext();++ const tools = buildDynamicTools(ctx);+ expect(tools).toHaveLength(1);+ expect(tools[0]!.name).toBe("guarded");+ expect(+ tools[0]!.needsApproval!({+ approvedTools: new Set(),+ toolInput: { draftId: "draft_123" },+ toolName: "guarded",+ }),+ ).toBe(true);+ expect(approvalFn).toHaveBeenCalledExactlyOnceWith({+ approvedTools: new Set(),+ toolInput: { draftId: "draft_123" },+ toolName: "guarded",+ });});it("propagates outputSchema from dynamic entries into harness tools and metadata", async () => {diff --git a/packages/eve/src/context/dynamic-tool-lifecycle.ts b/packages/eve/src/context/dynamic-tool-lifecycle.tsindex 4a129849c..134b35e19 100644--- a/packages/eve/src/context/dynamic-tool-lifecycle.ts+++ b/packages/eve/src/context/dynamic-tool-lifecycle.ts@@ -1,6 +1,7 @@import { jsonSchema, zodSchema, type FlexibleSchema, type ModelMessage } from "ai";import type { HarnessToolDefinition } from "#harness/execute-tool.js";+import type { NeedsApprovalContext } from "#public/definitions/tool.js";import type { DynamicToolEntry } from "#shared/dynamic-tool-definition.js";import type { HandleMessageStreamEvent } from "#protocol/message.js";import {@@ -256,6 +257,9 @@ async function resolveToolsFromEvent(dynamicToolOwners.set(name, resolver.slug);liveTools.push(toHarnessToolDefinition(name, entry));+ if (event.type === "step.started") {+ continue;+ }const stepFn ="__executeStepFn" in entry@@ -287,6 +291,17 @@ async function resolveToolsFromEvent(serializedClosureVars = {};}+ let needsApprovalStepFnName: string | undefined;+ if (entry.needsApproval !== undefined) {+ needsApprovalStepFnName = `eve:dynamic-tool-approval:${resolver.slug}:${entryKey}`;+ const originalNeedsApproval = entry.needsApproval.bind(entry);+ registerStepFunction(+ needsApprovalStepFnName,+ (_closureVars: unknown, approvalCtx: unknown) =>+ originalNeedsApproval(approvalCtx as NeedsApprovalContext),+ );+ }+metadata.push({name,description: entry.description,@@ -298,6 +313,7 @@ async function resolveToolsFromEvent(resolverSlug: resolver.slug,entryKey,executeStepFnName,+ needsApprovalStepFnName,closureVars: serializedClosureVars,});}diff --git a/packages/eve/src/context/keys.ts b/packages/eve/src/context/keys.tsindex 788a0bc18..0badc4201 100644--- a/packages/eve/src/context/keys.ts+++ b/packages/eve/src/context/keys.ts@@ -98,6 +98,7 @@ export interface DurableDynamicToolMetadata {readonly resolverSlug: string;readonly entryKey: string;readonly executeStepFnName?: string;+ readonly needsApprovalStepFnName?: string;readonly closureVars?: Record<string, unknown>;}