Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): harness - preserve approval gates on dynamic tools (#25)

Andrew Barbadbac239df1c7
diff --git a/.changeset/fuzzy-tfl-approvals.md b/.changeset/fuzzy-tfl-approvals.md
new file mode 100644
index 000000000..eb29bf6a7
--- /dev/null
+++ b/.changeset/fuzzy-tfl-approvals.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Fix dynamic connection tools so approval gates from OpenAPI and other connection-backed tools are preserved when the tools are exposed to the model. Calls to connections with `approval: always()` now correctly park for HITL approval before execution.
diff --git a/apps/docs/components/geistdocs/footer.tsx b/apps/docs/components/geistdocs/footer.tsx
index 05e49bc8a..4506ada49 100644
--- a/apps/docs/components/geistdocs/footer.tsx
+++ b/apps/docs/components/geistdocs/footer.tsx
@@ -34,9 +34,7 @@ export const Footer = ({
</Link>
</div>
<div className="flex items-center gap-2">
- <p className="text-center sm:text-left text-gray-800 text-sm">
- {copyright}
- </p>
+ <p className="text-center sm:text-left text-gray-800 text-sm">{copyright}</p>
{hasMultipleLanguages ? <LanguageSelector /> : null}
<Button asChild size="icon-sm" type="button" variant="ghost">
<a href="/rss.xml" rel="noopener" target="_blank">
diff --git a/docs/getting-started.mdx b/docs/getting-started.mdx
index 7deec7213..cfc6d2b29 100644
--- a/docs/getting-started.mdx
+++ b/docs/getting-started.mdx
@@ -6,7 +6,9 @@ description: "Install Eve, scaffold your first agent, give it a tool, and run it
Eve is a filesystem-first framework for durable agents. You write capabilities under `agent/`, and Eve runs the model loop, persists every session, and serves the agent over HTTP and platform channels. You'll scaffold an app, add a tool, run it locally, then create, stream, and continue a session over HTTP.
<Callout>
- Eve is currently in beta and subject to the [Vercel beta terms](https://vercel.com/docs/release-phases/public-beta-agreement); the framework, APIs, documentation, and behavior may change before general availability.
+ Eve is currently in beta and subject to the [Vercel beta
+ terms](https://vercel.com/docs/release-phases/public-beta-agreement); the framework, APIs,
+ documentation, and behavior may change before general availability.
</Callout>
## Prerequisites
diff --git a/docs/introduction.mdx b/docs/introduction.mdx
index 93e32b7f2..a2a8dbfed 100644
--- a/docs/introduction.mdx
+++ b/docs/introduction.mdx
@@ -8,7 +8,9 @@ Eve is a framework for building durable agents as ordinary files in a TypeScript
Instead of one large configuration object, each part of your agent gets a clear home. Instructions go in one file, tools in one folder, channels in another. Eve discovers that structure and turns it into an agent that runs locally, serves HTTP, connects to other platforms, and keeps working across many turns.
<Callout>
- Eve is currently in beta and subject to the [Vercel beta terms](https://vercel.com/docs/release-phases/public-beta-agreement); the framework, APIs, documentation, and behavior may change before general availability.
+ Eve is currently in beta and subject to the [Vercel beta
+ terms](https://vercel.com/docs/release-phases/public-beta-agreement); the framework, APIs,
+ documentation, and behavior may change before general availability.
</Callout>
## An Eve project at a glance
diff --git a/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl-approval.ts b/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl-approval.ts
new file mode 100644
index 000000000..48c6bae8a
--- /dev/null
+++ b/e2e/fixtures/agent-openapi-swagger/agent/connections/tfl-approval.ts
@@ -0,0 +1,11 @@
+import { defineOpenAPIConnection } from "eve/connections";
+import { always } from "eve/tools/approval";
+
+export default defineOpenAPIConnection({
+ approval: always(),
+ baseUrl: "https://api.tfl.gov.uk",
+ spec: "https://api.tfl.gov.uk/swagger/docs/v1",
+ description:
+ "Approval-gated Transport for London Unified API from its public Swagger 2.0 document.",
+ operations: { allow: ["Journey_Meta"] },
+});
diff --git a/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts
new file mode 100644
index 000000000..646a079fa
--- /dev/null
+++ b/e2e/fixtures/agent-openapi-swagger/evals/tfl-swagger-approval.eval.ts
@@ -0,0 +1,101 @@
+import type { HandleMessageStreamEvent } from "eve/client";
+import { defineEval } from "eve/evals";
+
+const SEARCH_TOOL = "connection__search";
+const TFL_APPROVAL_JOURNEY_MODES_TOOL = "connection__tfl-approval__Journey_Meta";
+
+export default defineEval({
+ description:
+ "OpenAPI connection HITL: an approval-gated TfL Swagger operation parks before execution.",
+
+ async test(t) {
+ const parked = await t.send(
+ [
+ "Use the `connection__search` tool with connection `tfl-approval` to find the TfL journey modes operation.",
+ "Then call `connection__tfl-approval__Journey_Meta` exactly once with an empty object.",
+ "Wait for approval if requested.",
+ "After the tool runs, reply with the exact words `bus` and `tube` if both mode names are present in the tool result.",
+ ].join("\n"),
+ );
+ parked.expectOk();
+
+ const [request] = t.expectInputRequests({
+ display: "confirmation",
+ toolName: TFL_APPROVAL_JOURNEY_MODES_TOOL,
+ });
+ if (request === undefined) {
+ throw new Error("Expected the OpenAPI connection tool to request approval.");
+ }
+
+ const optionIds = (request.options ?? []).map((option) => option.id);
+ if (!optionIds.includes("approve") || !optionIds.includes("deny")) {
+ throw new Error(`Expected approve/deny options, got [${optionIds.join(", ")}].`);
+ }
+ if (toolResultOutputs(parked.events, TFL_APPROVAL_JOURNEY_MODES_TOOL).length > 0) {
+ throw new Error("Approval-gated OpenAPI tool executed before approval.");
+ }
+
+ const approved = await t.respondAll("approve");
+ approved.expectOk();
+
+ const outputs = toolResultOutputs(t.events, TFL_APPROVAL_JOURNEY_MODES_TOOL);
+ if (outputs.length !== 1) {
+ throw new Error(
+ `Expected "${TFL_APPROVAL_JOURNEY_MODES_TOOL}" to execute exactly once after approval; saw ${outputs.length}.`,
+ );
+ }
+ const [output] = outputs;
+ const modes = extractModeNames(output.body);
+ if (!modes.has("bus") || !modes.has("tube")) {
+ throw new Error(
+ `Expected TfL modes to include bus and tube after approval, got ${JSON.stringify([...modes])}`,
+ );
+ }
+
+ t.didNotFail();
+ t.completed();
+ t.calledTool(SEARCH_TOOL, { isError: false });
+ t.messageIncludes(/\bbus\b/iu);
+ t.messageIncludes(/\btube\b/iu);
+ },
+});
+
+function toolResultOutputs(
+ events: readonly HandleMessageStreamEvent[],
+ toolName: string,
+): Record<string, unknown>[] {
+ const outputs: Record<string, unknown>[] = [];
+ for (const event of events) {
+ if (event.type !== "action.result" || event.data.status === "rejected") {
+ continue;
+ }
+ const result = event.data.result;
+ if (result.kind !== "tool-result" || result.toolName !== toolName) {
+ continue;
+ }
+ if (typeof result.output !== "object" || result.output === null) {
+ throw new Error(
+ `Expected object output from "${toolName}"; got ${JSON.stringify(result.output)}.`,
+ );
+ }
+ outputs.push(result.output as Record<string, unknown>);
+ }
+ return outputs;
+}
+
+function extractModeNames(body: unknown): Set<string> {
+ const modes = new Set<string>();
+ if (!Array.isArray(body)) {
+ return modes;
+ }
+ for (const item of body) {
+ if (typeof item !== "object" || item === null) {
+ continue;
+ }
+ const modeName = (item as { modeName?: unknown }).modeName;
+ if (typeof modeName === "string") {
+ modes.add(modeName);
+ }
+ }
+ return modes;
+}
diff --git a/packages/eve/src/harness/code-mode.test.ts b/packages/eve/src/harness/code-mode.test.ts
index 2d8d379eb..8dbeca584 100644
--- a/packages/eve/src/harness/code-mode.test.ts
+++ b/packages/eve/src/harness/code-mode.test.ts
@@ -1,10 +1,11 @@
import { jsonSchema, tool } from "ai";
import { describe, expect, it } from "vitest";
-import { applySandboxToolSet } from "#harness/code-mode.js";
+import { applySandboxToolSet, buildSandboxHostTools } from "#harness/code-mode.js";
import { CODE_MODE_SURFACE, WORKFLOW_SURFACE } from "#harness/sandbox-surface.js";
import { ContextContainer, contextStorage } from "#context/container.js";
import { ContextKey } from "#context/key.js";
+import { LiveStepToolsKey } from "#context/keys.js";
import type { HarnessToolDefinition } from "#harness/execute-tool.js";
import { buildToolSet } from "#harness/tools.js";
import type { HarnessToolMap } from "#harness/types.js";
@@ -278,6 +279,29 @@ describe("applySandboxToolSet", () => {
expect(result).toBe(true);
});
+ it("preserves dynamic tool approval gates in sandbox host tools", async () => {
+ const needsApproval = () => true;
+ const ctx = new ContextContainer();
+ ctx.setVirtualContext(LiveStepToolsKey, [
+ {
+ description: "Requires approval.",
+ execute: async () => "ok",
+ inputSchema: jsonSchema({ type: "object" }),
+ name: "connection__tfl__getLineStatus",
+ needsApproval,
+ },
+ ]);
+
+ const hostTools = await contextStorage.run(ctx, () =>
+ buildSandboxHostTools({ tools: new Map() }),
+ );
+ const guardedTool = hostTools.connection__tfl__getLineStatus as {
+ needsApproval?: (input: unknown) => Promise<boolean> | boolean;
+ };
+
+ await expect(guardedTool.needsApproval?.({ line: "victoria" })).resolves.toBe(true);
+ });
+
it("dual-routes runtime action tools to both modelTools and hostTools", async () => {
const harnessTools: HarnessToolMap = new Map<string, HarnessToolDefinition>([
[
diff --git a/packages/eve/src/harness/code-mode.ts b/packages/eve/src/harness/code-mode.ts
index 7ecf165ae..644a309da 100644
--- a/packages/eve/src/harness/code-mode.ts
+++ b/packages/eve/src/harness/code-mode.ts
@@ -15,7 +15,7 @@ import {
import { loadCodeModeModule, type CodeModeOptions } from "#shared/code-mode.js";
import { ALL_SANDBOX_SURFACES, type SandboxSurface } from "#harness/sandbox-surface.js";
import { LOAD_SKILL_TOOL_NAME } from "#runtime/skills/fragment-context.js";
-import { buildToolSet } from "#harness/tools.js";
+import { buildToolSet, buildToolSetFromDefinitions } from "#harness/tools.js";
/**
* Framework tools that must never enter a sandbox — they stay directly callable
@@ -163,13 +163,13 @@ export async function buildSandboxHostTools(input: {
const ctx = contextStorage.getStore();
if (ctx !== undefined) {
const dynamicTools = buildDynamicTools(ctx);
- for (const def of dynamicTools) {
- flatTools[def.name] ??= {
- description: def.description,
- inputSchema: def.inputSchema,
- execute: def.execute,
- outputSchema: def.outputSchema,
- };
+ const dynamicToolSet = buildToolSetFromDefinitions({
+ approvedTools: input.approvedTools,
+ capabilities: input.capabilities,
+ tools: dynamicTools,
+ });
+ for (const [name, toolDefinition] of Object.entries(dynamicToolSet)) {
+ flatTools[name] ??= toolDefinition;
}
}
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index edf208f09..5b4459cb9 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -3,7 +3,7 @@ import { type FilePart, jsonSchema, type LanguageModel, ToolLoopAgent, type User
import { afterEach, describe, expect, it, vi } from "vitest";
import { ContextContainer, contextStorage } from "#context/container.js";
-import { SessionDynamicInstructionsKey } from "#context/keys.js";
+import { LiveStepToolsKey, SessionDynamicInstructionsKey } from "#context/keys.js";
import { ChannelInstrumentationKey, SandboxKey } from "#context/keys.js";
import { decodeSandboxRef, isSandboxRefUrl } from "#internal/attachments/sandbox-refs.js";
import { mockSandbox } from "#internal/testing/mocks/mock-sandbox.js";
@@ -552,6 +552,45 @@ describe("createToolLoopHarness", () => {
expect(agentCall!.tools).not.toHaveProperty("code_mode");
});
+ it("preserves approval gates on step-scoped dynamic tools", async () => {
+ setupMockAgent({
+ finishReason: "stop",
+ response: { messages: [{ content: "Hello!", role: "assistant" }] },
+ text: "Hello!",
+ toolCalls: [],
+ toolResults: [],
+ });
+
+ const needsApproval = vi.fn(() => true);
+ const ctx = new ContextContainer();
+ ctx.setVirtualContext(LiveStepToolsKey, [
+ {
+ description: "Get TfL line status.",
+ execute: vi.fn().mockResolvedValue({ ok: true }),
+ inputSchema: jsonSchema({ type: "object" }),
+ name: "connection__tfl__getLineStatus",
+ needsApproval,
+ },
+ ]);
+
+ const config = createTestConfig("conversation", undefined, { codeMode: false });
+ const runStep = createToolLoopHarness(config);
+ await contextStorage.run(ctx, () => runStep(createTestSession(), { message: "Hi" }));
+
+ const agentCall = vi.mocked(ToolLoopAgent).mock.calls[0]?.[0] as
+ | { tools: Record<string, { needsApproval?: (toolInput: unknown) => Promise<boolean> }> }
+ | undefined;
+ expect(agentCall).toBeDefined();
+ const dynamicTool = agentCall!.tools.connection__tfl__getLineStatus!;
+
+ await expect(dynamicTool.needsApproval?.({ line: "victoria" })).resolves.toBe(true);
+ expect(needsApproval).toHaveBeenCalledExactlyOnceWith({
+ approvedTools: new Set(),
+ toolInput: { line: "victoria" },
+ toolName: "connection__tfl__getLineStatus",
+ });
+ });
+
it("preserves a user-authored web_search tool instead of replacing it with the provider tool", async () => {
setupMockAgent({
finishReason: "stop",
diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.ts
index 28393bdf3..1d45ae06a 100644
--- a/packages/eve/src/harness/tool-loop.ts
+++ b/packages/eve/src/harness/tool-loop.ts
@@ -142,7 +142,7 @@ import {
isInvalidToolCall,
} from "#harness/step-hooks.js";
import { pruneToolResults } from "#harness/tool-result-pruning.js";
-import { buildToolSetWithProviderTools } from "#harness/tools.js";
+import { buildToolSetFromDefinitions, buildToolSetWithProviderTools } from "#harness/tools.js";
import {
CODE_MODE_TOOL_NAME,
loadCodeModeModule,
@@ -652,13 +652,14 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
if (ctx !== undefined) {
const dynamicTools = buildDynamicTools(ctx);
- for (const def of dynamicTools) {
- flatTools[def.name] ??= {
- description: def.description,
- inputSchema: def.inputSchema,
- execute: def.execute,
- outputSchema: def.outputSchema,
- };
+ const dynamicToolSet = buildToolSetFromDefinitions({
+ approvedTools,
+ capabilities: config.capabilities,
+ disabledProviderTools: opts.disabledProviderTools,
+ tools: dynamicTools,
+ });
+ for (const [name, toolDefinition] of Object.entries(dynamicToolSet)) {
+ flatTools[name] ??= toolDefinition;
}
}
diff --git a/packages/eve/src/harness/tools.ts b/packages/eve/src/harness/tools.ts
index 6ebdb3dce..f7020540c 100644
--- a/packages/eve/src/harness/tools.ts
+++ b/packages/eve/src/harness/tools.ts
@@ -94,6 +94,32 @@ export function buildToolSet(input: {
return tools as ToolSet;
}
+/**
+ * Builds a ToolSet from an ordered list of harness definitions.
+ *
+ * The first definition for a name wins, matching the dynamic-tool scope
+ * ordering where step tools override turn/session tools.
+ */
+export function buildToolSetFromDefinitions(input: {
+ readonly approvedTools?: ReadonlySet<string>;
+ readonly capabilities?: SessionCapabilities;
+ readonly disabledProviderTools?: ReadonlySet<string>;
+ readonly tools: readonly HarnessToolDefinition[];
+}): ToolSet {
+ const tools = new Map<string, HarnessToolDefinition>();
+ for (const definition of input.tools) {
+ if (!tools.has(definition.name)) {
+ tools.set(definition.name, definition);
+ }
+ }
+ return buildToolSet({
+ approvedTools: input.approvedTools,
+ capabilities: input.capabilities,
+ disabledProviderTools: input.disabledProviderTools,
+ tools,
+ });
+}
+
/**
* Wraps a tool's `execute` so a returned {@link AuthorizationSignal} is
* stashed out-of-band ({@link stashToolInterrupt}) for the park detector while