The Open Framework for Building Agents
fix(eve): harness - preserve approval gates on dynamic tools (#25)
Andrew Barbadbac239df1c7
diff --git a/.changeset/fuzzy-tfl-approvals.md b/.changeset/fuzzy-tfl-approvals.mdnew file mode 100644index 000000000..eb29bf6a7--- /dev/null+++ b/.changeset/fuzzy-tfl-approvals.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Fix dynamic connection tools so approval gates from OpenAPI and other connection-backed tools are preserved when the tools are exposed to the model. Calls to connections with `approval: always()` now correctly park for HITL approval before execution.diff --git a/apps/docs/components/geistdocs/footer.tsx b/apps/docs/components/geistdocs/footer.tsxindex 05e49bc8a..4506ada49 100644--- a/apps/docs/components/geistdocs/footer.tsx+++ b/apps/docs/components/geistdocs/footer.tsx@@ -34,9 +34,7 @@ export const Footer = ({</Link></div><div className="flex items-center gap-2">- <p className="text-center sm:text-left text-gray-800 text-sm">- {copyright}- </p>+ <p className="text-center sm:text-left text-gray-800 text-sm">{copyright}</p>{hasMultipleLanguages ? <LanguageSelector /> : null}<Button asChild size="icon-sm" type="button" variant="ghost"><a href="/rss.xml" rel="noopener" target="_blank">diff --git a/docs/getting-started.mdx b/docs/getting-started.mdxindex 7deec7213..cfc6d2b29 100644--- a/docs/getting-started.mdx+++ b/docs/getting-started.mdx@@ -6,7 +6,9 @@ description: "Install Eve, scaffold your first agent, give it a tool, and run itEve is a filesystem-first framework for durable agents. You write capabilities under `agent/`, and Eve runs the model loop, persists every session, and serves the agent over HTTP and platform channels. You'll scaffold an app, add a tool, run it locally, then create, stream, and continue a session over HTTP.<Callout>- Eve is currently in beta and subject to the [Vercel beta terms](https://vercel.com/docs/release-phases/public-beta-agreement); the framework, APIs, documentation, and behavior may change before general availability.+ Eve is currently in beta and subject to the [Vercel beta+ terms](https://vercel.com/docs/release-phases/public-beta-agreement); the framework, APIs,+ documentation, and behavior may change before general availability.</Callout>## Prerequisitesdiff --git a/docs/introduction.mdx b/docs/introduction.mdxindex 93e32b7f2..a2a8dbfed 100644--- a/docs/introduction.mdx+++ b/docs/introduction.mdx@@ -8,7 +8,9 @@ Eve is a framework for building durable agents as ordinary files in a TypeScriptInstead of one large configuration object, each part of your agent gets a clear home. Instructions go in one file, tools in one folder, channels in another. Eve discovers that structure and turns it into an agent that runs locally, serves HTTP, connects to other platforms, and keeps working across many turns.<Callout>- Eve is currently in beta and subject to the [Vercel beta terms](https://vercel.com/docs/release-phases/public-beta-agreement); the framework, APIs, documentation, and behavior may change before general availability.+ Eve is currently in beta and subject to the [Vercel beta+ terms](https://vercel.com/docs/release-phases/public-beta-agreement); the framework, APIs,+ documentation, and behavior may change before general availability.</Callout>## An Eve project at a glancediff --git a/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl-approval.ts b/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl-approval.tsnew file mode 100644index 000000000..48c6bae8a--- /dev/null+++ b/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl-approval.ts@@ -0,0 +1,11 @@+import { defineOpenAPIConnection } from "eve/connections";+import { always } from "eve/tools/approval";++export default defineOpenAPIConnection({+ approval: always(),+ baseUrl: "https://api.tfl.gov.uk",+ spec: "https://api.tfl.gov.uk/swagger/docs/v1",+ description:+ "Approval-gated Transport for London Unified API from its public Swagger 2.0 document.",+ operations: { allow: ["Journey_Meta"] },+});diff --git a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.tsnew file mode 100644index 000000000..646a079fa--- /dev/null+++ b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts@@ -0,0 +1,101 @@+import type { HandleMessageStreamEvent } from "eve/client";+import { defineEval } from "eve/evals";++const SEARCH_TOOL = "connection__search";+const TFL_APPROVAL_JOURNEY_MODES_TOOL = "connection__tfl-approval__Journey_Meta";++export default defineEval({+ description:+ "OpenAPI connection HITL: an approval-gated TfL Swagger operation parks before execution.",++ async test(t) {+ const parked = await t.send(+ [+ "Use the `connection__search` tool with connection `tfl-approval` to find the TfL journey modes operation.",+ "Then call `connection__tfl-approval__Journey_Meta` exactly once with an empty object.",+ "Wait for approval if requested.",+ "After the tool runs, reply with the exact words `bus` and `tube` if both mode names are present in the tool result.",+ ].join("\n"),+ );+ parked.expectOk();++ const [request] = t.expectInputRequests({+ display: "confirmation",+ toolName: TFL_APPROVAL_JOURNEY_MODES_TOOL,+ });+ if (request === undefined) {+ throw new Error("Expected the OpenAPI connection tool to request approval.");+ }++ const optionIds = (request.options ?? []).map((option) => option.id);+ if (!optionIds.includes("approve") || !optionIds.includes("deny")) {+ throw new Error(`Expected approve/deny options, got [${optionIds.join(", ")}].`);+ }+ if (toolResultOutputs(parked.events, TFL_APPROVAL_JOURNEY_MODES_TOOL).length > 0) {+ throw new Error("Approval-gated OpenAPI tool executed before approval.");+ }++ const approved = await t.respondAll("approve");+ approved.expectOk();++ const outputs = toolResultOutputs(t.events, TFL_APPROVAL_JOURNEY_MODES_TOOL);+ if (outputs.length !== 1) {+ throw new Error(+ `Expected "${TFL_APPROVAL_JOURNEY_MODES_TOOL}" to execute exactly once after approval; saw ${outputs.length}.`,+ );+ }+ const [output] = outputs;+ const modes = extractModeNames(output.body);+ if (!modes.has("bus") || !modes.has("tube")) {+ throw new Error(+ `Expected TfL modes to include bus and tube after approval, got ${JSON.stringify([...modes])}`,+ );+ }++ t.didNotFail();+ t.completed();+ t.calledTool(SEARCH_TOOL, { isError: false });+ t.messageIncludes(/\bbus\b/iu);+ t.messageIncludes(/\btube\b/iu);+ },+});++function toolResultOutputs(+ events: readonly HandleMessageStreamEvent[],+ toolName: string,+): Record<string, unknown>[] {+ const outputs: Record<string, unknown>[] = [];+ for (const event of events) {+ if (event.type !== "action.result" || event.data.status === "rejected") {+ continue;+ }+ const result = event.data.result;+ if (result.kind !== "tool-result" || result.toolName !== toolName) {+ continue;+ }+ if (typeof result.output !== "object" || result.output === null) {+ throw new Error(+ `Expected object output from "${toolName}"; got ${JSON.stringify(result.output)}.`,+ );+ }+ outputs.push(result.output as Record<string, unknown>);+ }+ return outputs;+}++function extractModeNames(body: unknown): Set<string> {+ const modes = new Set<string>();+ if (!Array.isArray(body)) {+ return modes;+ }+ for (const item of body) {+ if (typeof item !== "object" || item === null) {+ continue;+ }+ const modeName = (item as { modeName?: unknown }).modeName;+ if (typeof modeName === "string") {+ modes.add(modeName);+ }+ }+ return modes;+}diff --git a/packages/eve/src/harness/code-mode.test.ts b/packages/eve/src/harness/code-mode.test.tsindex 2d8d379eb..8dbeca584 100644--- a/packages/eve/src/harness/code-mode.test.ts+++ b/packages/eve/src/harness/code-mode.test.ts@@ -1,10 +1,11 @@import { jsonSchema, tool } from "ai";import { describe, expect, it } from "vitest";-import { applySandboxToolSet } from "#harness/code-mode.js";+import { applySandboxToolSet, buildSandboxHostTools } from "#harness/code-mode.js";import { CODE_MODE_SURFACE, WORKFLOW_SURFACE } from "#harness/sandbox-surface.js";import { ContextContainer, contextStorage } from "#context/container.js";import { ContextKey } from "#context/key.js";+import { LiveStepToolsKey } from "#context/keys.js";import type { HarnessToolDefinition } from "#harness/execute-tool.js";import { buildToolSet } from "#harness/tools.js";import type { HarnessToolMap } from "#harness/types.js";@@ -278,6 +279,29 @@ describe("applySandboxToolSet", () => {expect(result).toBe(true);});+ it("preserves dynamic tool approval gates in sandbox host tools", async () => {+ const needsApproval = () => true;+ const ctx = new ContextContainer();+ ctx.setVirtualContext(LiveStepToolsKey, [+ {+ description: "Requires approval.",+ execute: async () => "ok",+ inputSchema: jsonSchema({ type: "object" }),+ name: "connection__tfl__getLineStatus",+ needsApproval,+ },+ ]);++ const hostTools = await contextStorage.run(ctx, () =>+ buildSandboxHostTools({ tools: new Map() }),+ );+ const guardedTool = hostTools.connection__tfl__getLineStatus as {+ needsApproval?: (input: unknown) => Promise<boolean> | boolean;+ };++ await expect(guardedTool.needsApproval?.({ line: "victoria" })).resolves.toBe(true);+ });+it("dual-routes runtime action tools to both modelTools and hostTools", async () => {const harnessTools: HarnessToolMap = new Map<string, HarnessToolDefinition>([[diff --git a/packages/eve/src/harness/code-mode.ts b/packages/eve/src/harness/code-mode.tsindex 7ecf165ae..644a309da 100644--- a/packages/eve/src/harness/code-mode.ts+++ b/packages/eve/src/harness/code-mode.ts@@ -15,7 +15,7 @@ import {import { loadCodeModeModule, type CodeModeOptions } from "#shared/code-mode.js";import { ALL_SANDBOX_SURFACES, type SandboxSurface } from "#harness/sandbox-surface.js";import { LOAD_SKILL_TOOL_NAME } from "#runtime/skills/fragment-context.js";-import { buildToolSet } from "#harness/tools.js";+import { buildToolSet, buildToolSetFromDefinitions } from "#harness/tools.js";/*** Framework tools that must never enter a sandbox — they stay directly callable@@ -163,13 +163,13 @@ export async function buildSandboxHostTools(input: {const ctx = contextStorage.getStore();if (ctx !== undefined) {const dynamicTools = buildDynamicTools(ctx);- for (const def of dynamicTools) {- flatTools[def.name] ??= {- description: def.description,- inputSchema: def.inputSchema,- execute: def.execute,- outputSchema: def.outputSchema,- };+ const dynamicToolSet = buildToolSetFromDefinitions({+ approvedTools: input.approvedTools,+ capabilities: input.capabilities,+ tools: dynamicTools,+ });+ for (const [name, toolDefinition] of Object.entries(dynamicToolSet)) {+ flatTools[name] ??= toolDefinition;}}diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex edf208f09..5b4459cb9 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -3,7 +3,7 @@ import { type FilePart, jsonSchema, type LanguageModel, ToolLoopAgent, type Userimport { afterEach, describe, expect, it, vi } from "vitest";import { ContextContainer, contextStorage } from "#context/container.js";-import { SessionDynamicInstructionsKey } from "#context/keys.js";+import { LiveStepToolsKey, SessionDynamicInstructionsKey } from "#context/keys.js";import { ChannelInstrumentationKey, SandboxKey } from "#context/keys.js";import { decodeSandboxRef, isSandboxRefUrl } from "#internal/attachments/sandbox-refs.js";import { mockSandbox } from "#internal/testing/mocks/mock-sandbox.js";@@ -552,6 +552,45 @@ describe("createToolLoopHarness", () => {expect(agentCall!.tools).not.toHaveProperty("code_mode");});+ it("preserves approval gates on step-scoped dynamic tools", async () => {+ setupMockAgent({+ finishReason: "stop",+ response: { messages: [{ content: "Hello!", role: "assistant" }] },+ text: "Hello!",+ toolCalls: [],+ toolResults: [],+ });++ const needsApproval = vi.fn(() => true);+ const ctx = new ContextContainer();+ ctx.setVirtualContext(LiveStepToolsKey, [+ {+ description: "Get TfL line status.",+ execute: vi.fn().mockResolvedValue({ ok: true }),+ inputSchema: jsonSchema({ type: "object" }),+ name: "connection__tfl__getLineStatus",+ needsApproval,+ },+ ]);++ const config = createTestConfig("conversation", undefined, { codeMode: false });+ const runStep = createToolLoopHarness(config);+ await contextStorage.run(ctx, () => runStep(createTestSession(), { message: "Hi" }));++ const agentCall = vi.mocked(ToolLoopAgent).mock.calls[0]?.[0] as+ | { tools: Record<string, { needsApproval?: (toolInput: unknown) => Promise<boolean> }> }+ | undefined;+ expect(agentCall).toBeDefined();+ const dynamicTool = agentCall!.tools.connection__tfl__getLineStatus!;++ await expect(dynamicTool.needsApproval?.({ line: "victoria" })).resolves.toBe(true);+ expect(needsApproval).toHaveBeenCalledExactlyOnceWith({+ approvedTools: new Set(),+ toolInput: { line: "victoria" },+ toolName: "connection__tfl__getLineStatus",+ });+ });+it("preserves a user-authored web_search tool instead of replacing it with the provider tool", async () => {setupMockAgent({finishReason: "stop",diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.tsindex 28393bdf3..1d45ae06a 100644--- a/packages/eve/src/harness/tool-loop.ts+++ b/packages/eve/src/harness/tool-loop.ts@@ -142,7 +142,7 @@ import {isInvalidToolCall,} from "#harness/step-hooks.js";import { pruneToolResults } from "#harness/tool-result-pruning.js";-import { buildToolSetWithProviderTools } from "#harness/tools.js";+import { buildToolSetFromDefinitions, buildToolSetWithProviderTools } from "#harness/tools.js";import {CODE_MODE_TOOL_NAME,loadCodeModeModule,@@ -652,13 +652,14 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {if (ctx !== undefined) {const dynamicTools = buildDynamicTools(ctx);- for (const def of dynamicTools) {- flatTools[def.name] ??= {- description: def.description,- inputSchema: def.inputSchema,- execute: def.execute,- outputSchema: def.outputSchema,- };+ const dynamicToolSet = buildToolSetFromDefinitions({+ approvedTools,+ capabilities: config.capabilities,+ disabledProviderTools: opts.disabledProviderTools,+ tools: dynamicTools,+ });+ for (const [name, toolDefinition] of Object.entries(dynamicToolSet)) {+ flatTools[name] ??= toolDefinition;}}diff --git a/packages/eve/src/harness/tools.ts b/packages/eve/src/harness/tools.tsindex 6ebdb3dce..f7020540c 100644--- a/packages/eve/src/harness/tools.ts+++ b/packages/eve/src/harness/tools.ts@@ -94,6 +94,32 @@ export function buildToolSet(input: {return tools as ToolSet;}+/**+ * Builds a ToolSet from an ordered list of harness definitions.+ *+ * The first definition for a name wins, matching the dynamic-tool scope+ * ordering where step tools override turn/session tools.+ */+export function buildToolSetFromDefinitions(input: {+ readonly approvedTools?: ReadonlySet<string>;+ readonly capabilities?: SessionCapabilities;+ readonly disabledProviderTools?: ReadonlySet<string>;+ readonly tools: readonly HarnessToolDefinition[];+}): ToolSet {+ const tools = new Map<string, HarnessToolDefinition>();+ for (const definition of input.tools) {+ if (!tools.has(definition.name)) {+ tools.set(definition.name, definition);+ }+ }+ return buildToolSet({+ approvedTools: input.approvedTools,+ capabilities: input.capabilities,+ disabledProviderTools: input.disabledProviderTools,+ tools,+ });+}+/*** Wraps a tool's `execute` so a returned {@link AuthorizationSignal} is* stashed out-of-band ({@link stashToolInterrupt}) for the park detector while