Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): dynamic tools - preserve approval replay (#275)

Andrew Barba30c59653010e
diff --git a/.changeset/guard-dynamic-approval-replay.md b/.changeset/guard-dynamic-approval-replay.md
new file mode 100644
index 000000000..e4152d5d4
--- /dev/null
+++ b/.changeset/guard-dynamic-approval-replay.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Preserve dynamic tool approval gates when session- and turn-scoped tools are replayed from durable metadata. If a replayed approval callback cannot be recovered, eve now requires approval by default instead of silently running the tool unguarded.
diff --git a/e2e/fixtures/agent-tools-hitl/agent/tools/dynamic-guarded-echo.ts b/e2e/fixtures/agent-tools-hitl/agent/tools/dynamic-guarded-echo.ts
new file mode 100644
index 000000000..0d5262885
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/agent/tools/dynamic-guarded-echo.ts
@@ -0,0 +1,27 @@
+import { defineDynamic, defineTool } from "eve/tools";
+import { z } from "zod";
+
+export const DYNAMIC_GUARDED_ECHO_TOKEN = "dynamic-guarded-echo-ok-L8R6";
+
+export default defineDynamic({
+ events: {
+ "session.started": async () => {
+ return {
+ dynamic_guarded_echo: defineTool({
+ description:
+ "Smoke-test fixture for replayed dynamic tool approval. Echoes the note input. Only call when the user explicitly asks for `dynamic_guarded_echo`.",
+ inputSchema: z.object({
+ note: z.string().optional(),
+ }),
+ needsApproval: () => true,
+ async execute(input) {
+ return {
+ echoed: input.note ?? null,
+ token: DYNAMIC_GUARDED_ECHO_TOKEN,
+ };
+ },
+ }),
+ };
+ },
+ },
+});
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.ts
new file mode 100644
index 000000000..a679ae09c
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/dynamic-guarded-replay.eval.ts
@@ -0,0 +1,49 @@
+import { defineEval } from "eve/evals";
+import type { HandleMessageStreamEvent } from "eve/client";
+
+const DYNAMIC_GUARDED_ECHO_TOKEN = "dynamic-guarded-echo-ok-L8R6";
+const TOOL_NAME = "dynamic_guarded_echo";
+
+/**
+ * HITL flow: a session-scoped dynamic tool's approval gate survives durable
+ * replay. If replay drops `needsApproval`, the tool executes immediately and
+ * this eval fails before approval.
+ */
+export default defineEval({
+ description: "HITL smoke: replayed dynamic tools preserve needsApproval.",
+ async test(t) {
+ await t.send(`Call the \`${TOOL_NAME}\` tool with note "before-approval".`);
+ const [request] = t.expectInputRequests({
+ display: "confirmation",
+ toolName: TOOL_NAME,
+ });
+ if (request === undefined) {
+ throw new Error(`Expected ${TOOL_NAME} to park for approval.`);
+ }
+ if (dynamicGuardedEchoResults(t.events).length > 0) {
+ throw new Error(`${TOOL_NAME} executed before approval.`);
+ }
+
+ const approved = await t.respondAll("approve");
+ approved.expectOk();
+ const results = dynamicGuardedEchoResults(t.events);
+ if (results.length !== 1 || !results[0]!.includes(DYNAMIC_GUARDED_ECHO_TOKEN)) {
+ throw new Error(`Approved ${TOOL_NAME} call did not execute with the expected token.`);
+ }
+
+ t.didNotFail();
+ t.completed();
+ },
+});
+
+function dynamicGuardedEchoResults(events: readonly HandleMessageStreamEvent[]): string[] {
+ const results: string[] = [];
+ for (const event of events) {
+ if (event.type !== "action.result") continue;
+ if (event.data.status === "rejected") continue;
+ const result = event.data.result;
+ if (result.kind !== "tool-result" || result.toolName !== TOOL_NAME) continue;
+ results.push(JSON.stringify(result.output ?? ""));
+ }
+ return results;
+}
diff --git a/packages/eve/src/context/build-dynamic-tools.ts b/packages/eve/src/context/build-dynamic-tools.ts
index 7f090b567..5f1cc4a92 100644
--- a/packages/eve/src/context/build-dynamic-tools.ts
+++ b/packages/eve/src/context/build-dynamic-tools.ts
@@ -10,6 +10,7 @@ import {
import type { DurableDynamicToolMetadata } from "#context/keys.js";
import { buildCallbackContext } from "#context/build-callback-context.js";
import { createLogger } from "#internal/logging.js";
+import type { NeedsApprovalContext } from "#public/definitions/tool.js";
const log = createLogger("dynamic-tools");
@@ -52,6 +53,7 @@ function replayTools(metadata: readonly DurableDynamicToolMetadata[]): HarnessTo
execute: (input: unknown) => stepFn(m.closureVars, input, buildCallbackContext()),
inputSchema: jsonSchema(m.inputSchema),
name: m.name,
+ needsApproval: buildReplayedNeedsApproval(m),
outputSchema: m.outputSchema === undefined ? undefined : jsonSchema(m.outputSchema),
});
}
@@ -59,6 +61,26 @@ function replayTools(metadata: readonly DurableDynamicToolMetadata[]): HarnessTo
return tools;
}
+function buildReplayedNeedsApproval(
+ metadata: DurableDynamicToolMetadata,
+): HarnessToolDefinition["needsApproval"] | undefined {
+ if (metadata.needsApprovalStepFnName === undefined) {
+ return undefined;
+ }
+
+ const approvalStepFn = lookupStepFunction(metadata.needsApprovalStepFnName);
+ if (approvalStepFn === null) {
+ log.warn(
+ `Dynamic tool "${metadata.name}" references approval function "${metadata.needsApprovalStepFnName}" ` +
+ "which is not registered — requiring approval by default.",
+ );
+ return () => true;
+ }
+
+ return (approvalCtx: NeedsApprovalContext) =>
+ Boolean(approvalStepFn(metadata.closureVars ?? {}, approvalCtx));
+}
+
/**
* Builds live dynamic tool definitions. Narrower scopes appear first
* so they win on name collision (the tool-loop uses `??=` for dedup).
diff --git a/packages/eve/src/context/dynamic-tool-lifecycle.test.ts b/packages/eve/src/context/dynamic-tool-lifecycle.test.ts
index d8910b487..d25dfb23a 100644
--- a/packages/eve/src/context/dynamic-tool-lifecycle.test.ts
+++ b/packages/eve/src/context/dynamic-tool-lifecycle.test.ts
@@ -863,6 +863,8 @@ describe("framework dynamic tools (no bundler transform)", () => {
execute: async (): Promise<unknown> => ({ ok: true }),
};
const resolver = createResolver("connection", ["step.started"], () => ({ risky: entry }));
+ testRegistry.delete("eve:framework-dynamic:connection:risky");
+ testRegistry.delete("eve:dynamic-tool-approval:connection:risky");
await dispatchDynamicToolEvent({
ctx,
@@ -882,6 +884,47 @@ describe("framework dynamic tools (no bundler transform)", () => {
toolName: "risky",
}),
).toBe(true);
+ expect(testRegistry.has("eve:framework-dynamic:connection:risky")).toBe(false);
+ expect(testRegistry.has("eve:dynamic-tool-approval:connection:risky")).toBe(false);
+ });
+
+ it("replays needsApproval from session-scoped dynamic tools", async () => {
+ const ctx = createCtx();
+ const approvalFn = vi.fn(() => true);
+ const entry: DynamicToolEntry = {
+ description: "destructive op",
+ inputSchema: { type: "object" },
+ needsApproval: approvalFn,
+ execute: async (): Promise<unknown> => ({ ok: true }),
+ };
+ const resolver = createResolver("session_guard", ["session.started"], () => ({
+ guarded: entry,
+ }));
+
+ await dispatchDynamicToolEvent({
+ ctx,
+ resolvers: [resolver],
+ messages: [],
+ event: makeEvent("session.started"),
+ });
+
+ ctx.clearVirtualContext();
+
+ const tools = buildDynamicTools(ctx);
+ expect(tools).toHaveLength(1);
+ expect(tools[0]!.name).toBe("guarded");
+ expect(
+ tools[0]!.needsApproval!({
+ approvedTools: new Set(),
+ toolInput: { draftId: "draft_123" },
+ toolName: "guarded",
+ }),
+ ).toBe(true);
+ expect(approvalFn).toHaveBeenCalledExactlyOnceWith({
+ approvedTools: new Set(),
+ toolInput: { draftId: "draft_123" },
+ toolName: "guarded",
+ });
});
it("propagates outputSchema from dynamic entries into harness tools and metadata", async () => {
diff --git a/packages/eve/src/context/dynamic-tool-lifecycle.ts b/packages/eve/src/context/dynamic-tool-lifecycle.ts
index 4a129849c..134b35e19 100644
--- a/packages/eve/src/context/dynamic-tool-lifecycle.ts
+++ b/packages/eve/src/context/dynamic-tool-lifecycle.ts
@@ -1,6 +1,7 @@
import { jsonSchema, zodSchema, type FlexibleSchema, type ModelMessage } from "ai";
import type { HarnessToolDefinition } from "#harness/execute-tool.js";
+import type { NeedsApprovalContext } from "#public/definitions/tool.js";
import type { DynamicToolEntry } from "#shared/dynamic-tool-definition.js";
import type { HandleMessageStreamEvent } from "#protocol/message.js";
import {
@@ -256,6 +257,9 @@ async function resolveToolsFromEvent(
dynamicToolOwners.set(name, resolver.slug);
liveTools.push(toHarnessToolDefinition(name, entry));
+ if (event.type === "step.started") {
+ continue;
+ }
const stepFn =
"__executeStepFn" in entry
@@ -287,6 +291,17 @@ async function resolveToolsFromEvent(
serializedClosureVars = {};
}
+ let needsApprovalStepFnName: string | undefined;
+ if (entry.needsApproval !== undefined) {
+ needsApprovalStepFnName = `eve:dynamic-tool-approval:${resolver.slug}:${entryKey}`;
+ const originalNeedsApproval = entry.needsApproval.bind(entry);
+ registerStepFunction(
+ needsApprovalStepFnName,
+ (_closureVars: unknown, approvalCtx: unknown) =>
+ originalNeedsApproval(approvalCtx as NeedsApprovalContext),
+ );
+ }
+
metadata.push({
name,
description: entry.description,
@@ -298,6 +313,7 @@ async function resolveToolsFromEvent(
resolverSlug: resolver.slug,
entryKey,
executeStepFnName,
+ needsApprovalStepFnName,
closureVars: serializedClosureVars,
});
}
diff --git a/packages/eve/src/context/keys.ts b/packages/eve/src/context/keys.ts
index 788a0bc18..0badc4201 100644
--- a/packages/eve/src/context/keys.ts
+++ b/packages/eve/src/context/keys.ts
@@ -98,6 +98,7 @@ export interface DurableDynamicToolMetadata {
readonly resolverSlug: string;
readonly entryKey: string;
readonly executeStepFnName?: string;
+ readonly needsApprovalStepFnName?: string;
readonly closureVars?: Record<string, unknown>;
}