The Open Framework for Building Agents
test(eve): cover describe, invokeTool, and mcpChannel tools (#4139)
Ruic4907e25285b
diff --git a/docs/channels/custom.mdx b/docs/channels/custom.mdxindex 194d1f4ba..103d71cd8 100644--- a/docs/channels/custom.mdx+++ b/docs/channels/custom.mdx@@ -70,6 +70,8 @@ Each route receives these operation surfaces:- `describe()` returns the agent's name, description, and the compiled tools a caller can run outside a turn, with their JSON schemas, sorted by name. It carries no paths, config, or other inspection detail, so a channel can publish it to its callers. Tools that need a turn (framework tools, workflow and provider tools, tools without `execute`) and tools added by dynamic resolvers or subagents are left out.- `invokeTool(name, input, { auth })` runs one of those tools as `auth`, outside any conversation: no model, turn, or workflow run. Each call gets its own session. Authored state starts from its initial value and is not kept, and a sandbox the tool opens is deleted when the call ends. It resolves to `completed` with the tool's output, or to `invalid-input`, `denied`, `failed`, `approval-required` when the tool's approval policy asks a person (the tool did not run), or `authorization-required` with the connections the tool needs a sign-in to.+ A direct call doesn't go through the AI SDK, so AI SDK telemetry integrations never see it. Conversation tool calls do: an integration that an `agent/instrumentation/*.ts` module registers with `registerTelemetry()` from `ai` receives `onToolExecutionStart` and `onToolExecutionEnd` for each of them. A direct call is traced by eve itself instead. It gets an `execute_tool` span under the caller's trace and emits `tool.call.started` and `tool.call.completed` to instrumentation. Over MCP, the caller's trace comes from the request's `_meta` trace context (see [Trace context](./mcp#trace-context)).+Event handlers receive `(eventData, channel, ctx)`. `ctx.session.id` identifiesthe exact session, while `channel.continuation` exposes the current address and`alias()` when this channel needs to add another address. `session.failed`diff --git a/e2e/fixtures/agent-mcp/.gitignore b/e2e/fixtures/agent-mcp/.gitignorenew file mode 100644index 000000000..eb095f0f8--- /dev/null+++ b/e2e/fixtures/agent-mcp/.gitignore@@ -0,0 +1,10 @@+node_modules+.env*+.eve+.vercel+.next+.output+.nitro+dist+.DS_Store+*.tsbuildinfodiff --git a/e2e/fixtures/agent-mcp/.vercelignore b/e2e/fixtures/agent-mcp/.vercelignorenew file mode 100644index 000000000..357664320--- /dev/null+++ b/e2e/fixtures/agent-mcp/.vercelignore@@ -0,0 +1,6 @@+node_modules+.eve+.next+.output+.nitro+distdiff --git a/e2e/fixtures/agent-mcp/agent/agent.ts b/e2e/fixtures/agent-mcp/agent/agent.tsnew file mode 100644index 000000000..b8be7496a--- /dev/null+++ b/e2e/fixtures/agent-mcp/agent/agent.ts@@ -0,0 +1,41 @@+import { e2eAgentConfig } from "@eve-e2e/config";+import { defineAgent } from "eve";+import type { MockModelRequest, MockModelResponse, MockModelToolCall } from "eve/evals";++import { LOOPBACK_CONNECTION } from "../fixture";++const PUBLISH_DIRECTIVE = /MCP_PUBLISH "([^"]+)"/u;++/** One scripted call per directive, with a stable id so the next step finds its result. */+function directiveCall(message: string): MockModelToolCall | undefined {+ if (message.includes("MCP_WHOAMI")) {+ return {+ id: "mcp-whoami",+ input: { connection: LOOPBACK_CONNECTION, input: {}, tool: "whoami" },+ name: "connection_execute",+ };+ }+ const notice = PUBLISH_DIRECTIVE.exec(message)?.[1];+ if (notice !== undefined) {+ return {+ id: "mcp-publish",+ input: { connection: LOOPBACK_CONNECTION, input: { notice }, tool: "publish_notice" },+ name: "connection_execute",+ };+ }+ return undefined;+}++/** Scripted mock for the world suites: each prompt makes exactly one call, then reports how it ended. */+function respond(request: MockModelRequest): MockModelResponse | string {+ const message = request.lastUserMessage ?? "";+ const call = directiveCall(message);+ if (call === undefined) return `Mock reply: ${message}`;+ const result = request.toolResults.find((entry) => entry.id === call.id);+ if (result !== undefined) return `${call.id}: ${result.isError ? "failed" : "done"}`;+ return { toolCalls: [call] };+}++export default defineAgent({+ ...e2eAgentConfig({ mock: respond }),+});diff --git a/e2e/fixtures/agent-mcp/agent/channels/mcp.ts b/e2e/fixtures/agent-mcp/agent/channels/mcp.tsnew file mode 100644index 000000000..398ad5379--- /dev/null+++ b/e2e/fixtures/agent-mcp/agent/channels/mcp.ts@@ -0,0 +1,16 @@+import { mcpChannel } from "eve/channels/mcp";++import { SERVICE_ID, SERVICE_TOKEN } from "../../fixture";++export default mcpChannel({+ auth: (request) =>+ request.headers.get("authorization") === `Bearer ${SERVICE_TOKEN}`+ ? {+ attributes: {},+ authenticator: "e2e-fixture",+ principalId: SERVICE_ID,+ principalType: "service",+ }+ : null,+ tools: true,+});diff --git a/e2e/fixtures/agent-mcp/agent/connections/loopback.ts b/e2e/fixtures/agent-mcp/agent/connections/loopback.tsnew file mode 100644index 000000000..4a4861eab--- /dev/null+++ b/e2e/fixtures/agent-mcp/agent/connections/loopback.ts@@ -0,0 +1,15 @@+import { defineDynamic, defineMcpClientConnection } from "eve/connections";++import { SERVICE_TOKEN, previewBypassHeaders, selfMcpUrl } from "../../fixture";++// Resolved per session so the URL is this deployment's, not the build host's.+export default defineDynamic({+ events: {+ "session.started": () =>+ defineMcpClientConnection({+ description: "The Maple Street kennel agent's own tools, reached over its MCP channel.",+ headers: { ...previewBypassHeaders(), authorization: `Bearer ${SERVICE_TOKEN}` },+ url: selfMcpUrl(),+ }),+ },+});diff --git a/e2e/fixtures/agent-mcp/agent/instructions.md b/e2e/fixtures/agent-mcp/agent/instructions.mdnew file mode 100644index 000000000..0741b7124--- /dev/null+++ b/e2e/fixtures/agent-mcp/agent/instructions.md@@ -0,0 +1,3 @@+# Identity++You run the front desk of the Maple Street kennel. Use the `loopback` connection for kennel tools.diff --git a/e2e/fixtures/agent-mcp/agent/tools/publish_notice.ts b/e2e/fixtures/agent-mcp/agent/tools/publish_notice.tsnew file mode 100644index 000000000..328520a82--- /dev/null+++ b/e2e/fixtures/agent-mcp/agent/tools/publish_notice.ts@@ -0,0 +1,12 @@+import { defineTool } from "eve/tools";+import { always } from "eve/tools/approval";+import { z } from "zod";++export default defineTool({+ approval: always(),+ description: "Publish a notice on the kennel's front-desk board. Asks before every post.",+ inputSchema: z.object({ notice: z.string() }),+ async execute({ notice }, ctx) {+ return { by: ctx.session.auth.current?.principalId ?? null, published: notice };+ },+});diff --git a/e2e/fixtures/agent-mcp/agent/tools/whoami.ts b/e2e/fixtures/agent-mcp/agent/tools/whoami.tsnew file mode 100644index 000000000..798093658--- /dev/null+++ b/e2e/fixtures/agent-mcp/agent/tools/whoami.ts@@ -0,0 +1,14 @@+import { defineTool } from "eve/tools";+import { z } from "zod";++export default defineTool({+ description: "Report who this call runs as.",+ inputSchema: z.object({}),+ async execute(_input, ctx) {+ const current = ctx.session.auth.current;+ return {+ principalId: current?.principalId ?? null,+ principalType: current?.principalType ?? null,+ };+ },+});diff --git a/e2e/fixtures/agent-mcp/evals/approval.eval.ts b/e2e/fixtures/agent-mcp/evals/approval.eval.tsnew file mode 100644index 000000000..55cca15cf--- /dev/null+++ b/e2e/fixtures/agent-mcp/evals/approval.eval.ts@@ -0,0 +1,28 @@+import { defineEval } from "eve/evals";++import { requireMockModel } from "./mock-model";++const NOTICE = "The lobby closes early on Sunday.";++const mentions = (text: string) => (value: unknown) => JSON.stringify(value).includes(text);++export default defineEval({+ description:+ "A tool whose approval policy asks a person comes back from the MCP channel as an approval error, and never runs.",++ async test(t) {+ requireMockModel(t);+ const turn = await t.send(+ `Alice drafts the weekend notice for the front-desk board. MCP_PUBLISH "${NOTICE}"`,+ );+ turn.expectOk();++ turn.notEvent("input.requested");+ turn.calledTool("loopback__publish_notice", {+ count: 1,+ output: mentions("needs a person's approval, which this MCP channel cannot ask for"),+ status: "failed",+ });+ turn.calledTool("loopback__publish_notice", { count: 0 });+ },+});diff --git a/e2e/fixtures/agent-mcp/evals/call.eval.ts b/e2e/fixtures/agent-mcp/evals/call.eval.tsnew file mode 100644index 000000000..076e0b801--- /dev/null+++ b/e2e/fixtures/agent-mcp/evals/call.eval.ts@@ -0,0 +1,25 @@+import { isDeepStrictEqual } from "node:util";++import { defineEval } from "eve/evals";++import { SERVICE_ID } from "../fixture";+import { requireMockModel } from "./mock-model";++export default defineEval({+ description:+ "The agent calls one of its own tools through its MCP channel and gets the tool's exact structured output, run as the route-authenticated caller.",++ async test(t) {+ requireMockModel(t);+ const turn = await t.send(+ "Alice starts her shift and checks which account the kennel tools run as. MCP_WHOAMI",+ );+ turn.expectOk();++ turn.calledTool("loopback__whoami", {+ count: 1,+ output: (value) =>+ isDeepStrictEqual(value, { principalId: SERVICE_ID, principalType: "service" }),+ });+ },+});diff --git a/e2e/fixtures/agent-mcp/evals/evals.config.ts b/e2e/fixtures/agent-mcp/evals/evals.config.tsnew file mode 100644index 000000000..7a61a5b55--- /dev/null+++ b/e2e/fixtures/agent-mcp/evals/evals.config.ts@@ -0,0 +1,6 @@+import { e2eJudgeModel } from "@eve-e2e/config";+import { defineEvalConfig } from "eve/evals";++export default defineEvalConfig({+ judge: { model: e2eJudgeModel() },+});diff --git a/e2e/fixtures/agent-mcp/evals/mock-model.ts b/e2e/fixtures/agent-mcp/evals/mock-model.tsnew file mode 100644index 000000000..8692a1de3--- /dev/null+++ b/e2e/fixtures/agent-mcp/evals/mock-model.ts@@ -0,0 +1,6 @@+/** Skips evals whose agent turns depend on the scripted mock model. */+export function requireMockModel(t: { skip(reason: string): never }): void {+ if (process.env.EVE_E2E_MODEL !== "mock") {+ t.skip("Requires the deterministic mock model to make the exact loopback calls.");+ }+}diff --git a/e2e/fixtures/agent-mcp/fixture.ts b/e2e/fixtures/agent-mcp/fixture.tsnew file mode 100644index 000000000..6c53d2f1b--- /dev/null+++ b/e2e/fixtures/agent-mcp/fixture.ts@@ -0,0 +1,31 @@+/**+ * Identities and credentials shared by the agent and its evals. Everything+ * here is fixture-only: the deployment is a protected preview, and these+ * values exist so the fixture can call itself without injected env.+ */++/** The service principal the loopback connection authenticates as. */+export const SERVICE_ID = "maple-router";++/** Bearer token the loopback connection presents to the MCP channel. Fixture-only. */+export const SERVICE_TOKEN = "agent-mcp-fixture-service-token";++/** The connection the agent uses to reach its own MCP channel. */+export const LOOPBACK_CONNECTION = "loopback";++const MCP_PATH = "/eve/v1/mcp";++/** This deployment's MCP channel, the way `kennelUrl()` reaches its own kennel. */+export function selfMcpUrl(): string {+ const deploymentHost = process.env.VERCEL_URL;+ const host = deploymentHost+ ? `https://${deploymentHost}`+ : (process.env.WORKFLOW_LOCAL_BASE_URL ?? "http://127.0.0.1:3000");+ return `${host}${process.env.EVE_PUBLIC_ROUTE_PREFIX ?? ""}${MCP_PATH}`;+}++/** Lets self-calls reach a protected preview deployment of this fixture. */+export function previewBypassHeaders(): Record<string, string> {+ const bypass = process.env.VERCEL_AUTOMATION_BYPASS_SECRET;+ return bypass ? { "x-vercel-protection-bypass": bypass } : {};+}diff --git a/e2e/fixtures/agent-mcp/package.json b/e2e/fixtures/agent-mcp/package.jsonnew file mode 100644index 000000000..6a46ebc43--- /dev/null+++ b/e2e/fixtures/agent-mcp/package.json@@ -0,0 +1,23 @@+{+ "name": "agent-mcp",+ "version": "0.0.0",+ "private": true,+ "type": "module",+ "scripts": {+ "build": "eve build",+ "dev": "eve dev",+ "start": "eve start",+ "typecheck": "eve build --skip-sandbox-prewarm && tsc",+ "test:e2e": "eve eval --strict"+ },+ "dependencies": {+ "@eve-e2e/config": "workspace:*",+ "@workflow/world-postgres": "catalog:",+ "eve": "workspace:*",+ "zod": "catalog:"+ },+ "devDependencies": {+ "@types/node": "catalog:",+ "typescript": "catalog:"+ }+}diff --git a/e2e/fixtures/agent-mcp/tsconfig.json b/e2e/fixtures/agent-mcp/tsconfig.jsonnew file mode 100644index 000000000..85e3eceb8--- /dev/null+++ b/e2e/fixtures/agent-mcp/tsconfig.json@@ -0,0 +1,17 @@+{+ "compilerOptions": {+ "target": "ES2022",+ "module": "esnext",+ "moduleResolution": "bundler",+ "outDir": "dist",+ "rootDir": ".",+ "strict": true,+ "esModuleInterop": true,+ "skipLibCheck": true,+ "forceConsistentCasingInFileNames": true,+ "declaration": true,+ "noEmit": true,+ "types": ["node"]+ },+ "include": ["agent/**/*.ts", "evals/**/*.ts"]+}diff --git a/packages/eve/src/channel/agent-description.test.ts b/packages/eve/src/channel/agent-description.test.tsnew file mode 100644index 000000000..cbac98111--- /dev/null+++ b/packages/eve/src/channel/agent-description.test.ts@@ -0,0 +1,52 @@+import { expect, it } from "vitest";++import { describeCompiledAgent } from "#channel/agent-description.js";++it("describes only the tools invokeTool can run, sorted by name", () => {+ const tool = (name: string, extra: Record<string, unknown> = {}) => ({+ description: `${name} description`,+ hasExecute: true,+ inputSchema: { type: "object" },+ name,+ requiresApproval: false,+ sourceId: `source:${name}`,+ ...extra,+ });+ const manifest = {+ bindings: {+ "source:deploy": { owner: { kind: "application" } },+ "source:lookup": { owner: { kind: "extension" } },+ "source:load_skill": { owner: { feature: "skills", kind: "framework" } },+ "source:plan": { owner: { kind: "application" } },+ "source:remote": { owner: { kind: "application" } },+ },+ config: { description: "Runs the kennel.", name: "kennel" },+ tools: [+ tool("lookup", { outputSchema: { type: "object" } }),+ tool("deploy", { requiresApproval: true }),+ tool("load_skill"),+ tool("plan", { behavior: { handling: { kind: "workflow-tool" } } }),+ tool("remote", { hasExecute: false }),+ ],+ };++ expect(describeCompiledAgent(manifest as never)).toEqual({+ description: "Runs the kennel.",+ name: "kennel",+ tools: [+ {+ approval: true,+ description: "deploy description",+ inputSchema: { type: "object" },+ name: "deploy",+ },+ {+ approval: false,+ description: "lookup description",+ inputSchema: { type: "object" },+ name: "lookup",+ outputSchema: { type: "object" },+ },+ ],+ });+});diff --git a/packages/eve/src/execution/invoke-tool-trace-ownership.integration.test.ts b/packages/eve/src/execution/invoke-tool-trace-ownership.integration.test.tsnew file mode 100644index 000000000..f3e20ba80--- /dev/null+++ b/packages/eve/src/execution/invoke-tool-trace-ownership.integration.test.ts@@ -0,0 +1,178 @@+import { context as apiContext, trace as apiTrace, TraceFlags } from "@opentelemetry/api";+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";+import {+ BasicTracerProvider,+ InMemorySpanExporter,+ SimpleSpanProcessor,+ type ReadableSpan,+} from "@opentelemetry/sdk-trace-base";+import { afterEach, beforeEach, describe, expect, it } from "vitest";++import type { SessionAuthContext } from "#channel/types.js";+import { invokeTool, type InvokeToolRuntime } from "#execution/invoke-tool.js";+import { createToolExecuteWithAuth } from "#execution/tool-auth.js";+import type { HarnessToolDefinition } from "#harness/execute-tool.js";+import { createInstrumentationHooks } from "#instrumentation/lifecycle.js";+import { registerInstrumentationRuntime } from "#instrumentation/runtime.js";+import { createBundledRuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js";+import { defineJsonSchema } from "#tools/schema.js";+import { AgentTraceSpanProcessor } from "#tracing/agent-trace-span-processor.js";++const CLIENT_TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736";+const alice: SessionAuthContext = {+ attributes: {},+ authenticator: "test",+ principalId: "alice",+ principalType: "user",+};++const inputSchema = defineJsonSchema({+ additionalProperties: false,+ properties: { text: { type: "string" } },+ type: "object",+} as const);++let openGate: () => void = () => undefined;+let gate: Promise<void> = Promise.resolve();+const seenActive: boolean[] = [];++function tool(name: string, execute: (input: { text?: string }) => unknown): HarnessToolDefinition {+ return {+ description: name,+ execute: createToolExecuteWithAuth({ execute, scope: name }),+ inputSchema,+ name,+ };+}++const tools = [+ tool("lookup", async () => {+ const traceId = apiTrace.getActiveSpan()?.spanContext().traceId;+ seenActive.push(traceId !== undefined && processor.activeTraceIds().has(traceId));+ await apiTrace.getTracer("test.tool").startActiveSpan("nested lookup", async (span) => {+ span.end();+ });+ return { status: "ok" };+ }),+ tool("gated", async () => {+ await gate;+ return { status: "ok" };+ }),+];++const runtime: InvokeToolRuntime = {+ agentName: "compiled-agent",+ callbackBaseUrl: "https://agent.example",+ compiledArtifactsSource: createBundledRuntimeCompiledArtifactsSource(),+ manifest: {+ bindings: Object.fromEntries(+ tools.map(({ name }) => [`source:${name}`, { owner: { kind: "application" } } as never]),+ ),+ tools: tools.map(({ name }) => ({ hasExecute: true, name, sourceId: `source:${name}` })),+ },+ nodeId: "__root__",+ origin: { adapter: { kind: "mcp" } as never, agentName: "compiled-agent", channelName: "mcp" },+ sandboxRegistry: { sandbox: null } as never,+ tools: new Map(tools.map((definition) => [definition.name, definition])),+};++let exporter: InMemorySpanExporter;+let processor: AgentTraceSpanProcessor;+let provider: BasicTracerProvider;++beforeEach(() => {+ seenActive.length = 0;+ exporter = new InMemorySpanExporter();+ // The production local-trace processor: it claims traces and releases completed ones on flush.+ processor = new AgentTraceSpanProcessor([new SimpleSpanProcessor(exporter)]);+ provider = new BasicTracerProvider({ spanProcessors: [processor] });+ apiContext.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());+ apiTrace.setGlobalTracerProvider(provider);+ registerInstrumentationRuntime({+ forceFlush: async () => {+ await processor.forceFlush();+ processor.releaseCompletedTraces();+ },+ hooks: createInstrumentationHooks([]),+ otelSettings: {+ functionId: undefined,+ recordInputs: false,+ recordOutputs: false,+ traceChannelRequests: false,+ tracePolicy: undefined,+ },+ runInContext: (_operation, execute) => execute(),+ shutdown: async () => undefined,+ });+});++afterEach(async () => {+ await provider.shutdown();+ apiTrace.disable();+ apiContext.disable();+});++function names(): string[] {+ return exporter+ .getFinishedSpans()+ .map((span: ReadableSpan) => span.name)+ .sort();+}++function underClientTrace<T>(run: () => Promise<T>): Promise<T> {+ const remote = apiTrace.wrapSpanContext({+ isRemote: true,+ spanId: "00f067aa0ba902b7",+ traceFlags: TraceFlags.SAMPLED,+ traceId: CLIENT_TRACE_ID,+ });+ return apiContext.with(apiTrace.setSpan(apiContext.active(), remote), run);+}++describe("invokeTool trace ownership", () => {+ it("holds the trace while the call runs and releases it when the call ends", async () => {+ await invokeTool(runtime, "lookup", {}, { auth: alice });+ expect(seenActive).toEqual([true]);+ expect(names()).toEqual(["execute_tool lookup", "nested lookup"]);+ expect(processor.activeTraceIds().size).toBe(0);+ });++ it("keeps a shared caller trace until the last concurrent call ends", async () => {+ gate = new Promise((resolve) => (openGate = resolve));+ const slow = underClientTrace(() => invokeTool(runtime, "gated", {}, { auth: alice }));+ await underClientTrace(() => invokeTool(runtime, "lookup", {}, { auth: alice }));+ expect(processor.activeTraceIds().has(CLIENT_TRACE_ID)).toBe(true);++ openGate();+ await slow;+ expect(processor.activeTraceIds().size).toBe(0);+ expect(names()).toEqual(["execute_tool gated", "execute_tool lookup", "nested lookup"]);+ });++ it("claims a caller trace again for each sequential call", async () => {+ await underClientTrace(() => invokeTool(runtime, "lookup", {}, { auth: alice }));+ await underClientTrace(() => invokeTool(runtime, "lookup", {}, { auth: alice }));+ expect(seenActive).toEqual([true, true]);+ expect(names()).toEqual([+ "execute_tool lookup",+ "execute_tool lookup",+ "nested lookup",+ "nested lookup",+ ]);+ expect(processor.activeTraceIds().size).toBe(0);+ });++ it("leaves a trace a conversation owns for the conversation to release", async () => {+ const turn = apiTrace.getTracer("test.turn").startSpan("agent.turn", {+ attributes: { "agent.run.id": "session-1", "gen_ai.conversation.id": "conversation-1" },+ });+ const traceId = turn.spanContext().traceId;+ await apiContext.with(apiTrace.setSpan(apiContext.active(), turn), () =>+ invokeTool(runtime, "lookup", {}, { auth: alice }),+ );+ expect(processor.activeTraceIds().has(traceId)).toBe(true);+ turn.end();+ processor.releaseCompletedTraces();+ expect(processor.activeTraceIds().size).toBe(0);+ });+});diff --git a/packages/eve/src/execution/invoke-tool.test.ts b/packages/eve/src/execution/invoke-tool.test.tsnew file mode 100644index 000000000..f85bf645b--- /dev/null+++ b/packages/eve/src/execution/invoke-tool.test.ts@@ -0,0 +1,229 @@+import { afterEach, describe, expect, it, vi } from "vitest";++import type { ApprovalPolicy } from "#approval/definition.js";+import type { SessionAuthContext } from "#channel/types.js";+import { shutdownActiveSandboxHandles } from "#execution/sandbox/active-handles.js";+import { invokeTool, type InvokeToolRuntime } from "#execution/invoke-tool.js";+import { createToolExecuteWithAuth } from "#execution/tool-auth.js";+import type { HarnessToolDefinition } from "#harness/execute-tool.js";+import { mockSandbox } from "#internal/testing/mocks/mock-sandbox.js";+import { defineSandbox } from "#public/definitions/sandbox.js";+import { createBundledRuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js";+import type { RuntimeSandboxRegistry } from "#runtime/sandbox/registry.js";+import { defineSandboxProvider } from "#shared/sandbox-provider.js";+import type { ToolContext } from "#tools/definition.js";+import { defineJsonSchema } from "#tools/schema.js";++vi.mock("#runtime/sandbox/prepared-artifacts.js", () => ({+ loadSandboxPreparedArtifact: vi.fn(async () => null),+}));++afterEach(() => shutdownActiveSandboxHandles());++const alice: SessionAuthContext = {+ attributes: {},+ authenticator: "test",+ principalId: "alice",+ principalType: "user",+};++const inputSchema = defineJsonSchema({+ additionalProperties: false,+ properties: { text: { type: "string" } },+ type: "object",+});++function tool(+ name: string,+ execute: (input: any, ctx: ToolContext) => unknown,+ extra: Partial<HarnessToolDefinition> = {},+): HarnessToolDefinition {+ return {+ description: name,+ execute: createToolExecuteWithAuth({ execute, scope: name }),+ inputSchema,+ name,+ ...extra,+ };+}++function sandboxes(options: { readonly startGate?: Promise<void> } = {}) {+ const started: string[] = [];+ const deleted = vi.fn(async () => {});+ const shutdown = vi.fn(async () => {});+ const start = vi.fn(async (context: { readonly session: { readonly id: string } }) => {+ started.push(context.session.id);+ await options.startGate;+ const sandbox = mockSandbox();+ return {+ handle: {+ sandbox: sandbox.session,+ onRuntimeShutdown: shutdown,+ onSessionDelete: deleted,+ onSessionStop: async () => {},+ },+ state: null,+ };+ });+ const environment = defineSandboxProvider({+ name: "memory",+ environment: () => ({+ prepare: async () => null,+ resume: async () => {+ throw new Error("a call's sandbox is never resumed");+ },+ start,+ }),+ }).environment();+ const registry: RuntimeSandboxRegistry = {+ sandbox: {+ definition: {+ environment,+ kind: "independent",+ logicalPath: "sandbox.ts",+ revisionHash: "hash",+ selector: defineSandbox(async () => await environment.open()),+ sourceId: "sandbox",+ sourceKind: "module",+ },+ workspaceResourceRoot: { logicalPath: "", rootEntries: [] },+ },+ };+ return { deleted, registry, shutdown, started };+}++function runtimeWith(+ tools: readonly HarnessToolDefinition[],+ registry = sandboxes().registry,+ owners: Readonly<Record<string, "application" | "framework">> = {},+): InvokeToolRuntime {+ return {+ agentName: "test-agent",+ callbackBaseUrl: "https://agent.example",+ compiledArtifactsSource: createBundledRuntimeCompiledArtifactsSource(),+ manifest: {+ bindings: Object.fromEntries(+ tools.map(({ name }) => [+ `source:${name}`,+ { owner: { kind: owners[name] ?? "application" } } as never,+ ]),+ ),+ tools: tools.map(({ name }) => ({ hasExecute: true, name, sourceId: `source:${name}` })),+ },+ nodeId: "__root__",+ sandboxRegistry: registry,+ tools: new Map(tools.map((definition) => [definition.name, definition])),+ };+}++describe("invokeTool", () => {+ it("evaluates the approval policy and never asks anyone", async () => {+ const policies: Array<[ApprovalPolicy, object, boolean]> = [+ [() => "user-approval", { status: "approval-required" }, false],+ [+ () => ({ reason: "Read-only mode.", type: "denied" }),+ { reason: "Read-only mode.", status: "denied" },+ false,+ ],+ [() => "not-applicable", { output: "ran", status: "completed" }, true],+ ];+ for (const [approval, expected, ran] of policies) {+ const execute = vi.fn(() => "ran");+ const runtime = runtimeWith([tool("deploy", execute, { approval })]);+ expect(await invokeTool(runtime, "deploy", {}, { auth: alice })).toMatchObject(expected);+ expect(execute).toHaveBeenCalledTimes(ran ? 1 : 0);+ }+ });++ it("refuses unknown, framework, and badly typed calls before running anything", async () => {+ const execute = vi.fn();+ const runtime = runtimeWith([tool("note", execute), tool("load_skill", execute)], undefined, {+ load_skill: "framework",+ });+ expect(await invokeTool(runtime, "missing", {}, { auth: alice })).toMatchObject({+ message: 'The agent has no tool named "missing".',+ status: "failed",+ });+ expect(await invokeTool(runtime, "load_skill", {}, { auth: alice })).toMatchObject({+ message: expect.stringContaining("framework tool"),+ status: "failed",+ });+ expect(await invokeTool(runtime, "note", { text: 7 }, { auth: alice })).toMatchObject({+ status: "invalid-input",+ });+ expect(execute).not.toHaveBeenCalled();+ });++ it("checks input as a conversation does: none is {}, and a non-object is refused", async () => {+ const execute = vi.fn((input: unknown) => input);+ const runtime = runtimeWith([tool("note", execute)]);+ for (const input of [[1], "plain", 7, true]) {+ expect(await invokeTool(runtime, "note", input, { auth: alice })).toMatchObject({+ message: expect.stringContaining('Invalid input for tool "note"'),+ status: "invalid-input",+ });+ }+ expect(execute).not.toHaveBeenCalled();+ for (const input of [undefined, null, ""]) {+ expect(await invokeTool(runtime, "note", input, { auth: alice })).toMatchObject({+ output: {},+ status: "completed",+ });+ }+ });++ it("gives each call its own sandbox and deletes it when the call ends", async () => {+ const { deleted, registry, shutdown, started } = sandboxes();+ const write = tool("write", async (input: { text: string }, ctx) => {+ const sandbox = await ctx.getSandbox();+ await sandbox.writeTextFile({ content: input.text, path: "/workspace/note.txt" });+ return ctx.session.auth.current?.principalId;+ });+ const plain = tool("plain", () => "no sandbox");+ const runtime = runtimeWith([write, plain], registry);++ expect(await invokeTool(runtime, "write", { text: "a" }, { auth: alice })).toMatchObject({+ output: "alice",+ status: "completed",+ });+ await invokeTool(runtime, "write", { text: "b" }, { auth: alice });+ await invokeTool(runtime, "plain", {}, { auth: alice });++ expect(started).toHaveLength(2);+ expect(started[0]).not.toBe(started[1]);+ expect(deleted).toHaveBeenCalledTimes(2);+ await shutdownActiveSandboxHandles();+ expect(shutdown).not.toHaveBeenCalled();+ });++ it("deletes a sandbox whose start finishes after the call failed", async () => {+ let finishStart!: () => void;+ const startGate = new Promise<void>((resolve) => {+ finishStart = resolve;+ });+ const { deleted, registry, shutdown, started } = sandboxes({ startGate });+ let gaveUp = false;+ const racing = tool("racing", async (_input: unknown, ctx) => {+ // The call gives up while the provider is still starting, as a lost+ // race against cancellation would.+ void ctx.getSandbox().catch(() => {});+ await vi.waitFor(() => expect(started).toHaveLength(1));+ gaveUp = true;+ throw new Error("gave up");+ });+ const runtime = runtimeWith([racing], registry);++ const result = invokeTool(runtime, "racing", {}, { auth: alice });+ await vi.waitFor(() => expect(gaveUp).toBe(true));+ await new Promise((resolve) => setTimeout(resolve, 0));+ expect(deleted).not.toHaveBeenCalled();+ finishStart();+ // Let a start that outlived the call settle before checking what it left behind.+ await new Promise((resolve) => setTimeout(resolve, 0));++ expect(await result).toMatchObject({ status: "failed" });+ expect(deleted).toHaveBeenCalledTimes(1);+ await shutdownActiveSandboxHandles();+ expect(shutdown).not.toHaveBeenCalled();+ });+});diff --git a/packages/eve/src/public/channels/mcp-tools-tracing.integration.test.ts b/packages/eve/src/public/channels/mcp-tools-tracing.integration.test.tsnew file mode 100644index 000000000..6a902eb97--- /dev/null+++ b/packages/eve/src/public/channels/mcp-tools-tracing.integration.test.ts@@ -0,0 +1,408 @@+import {+ context as apiContext,+ trace as apiTrace,+ SpanStatusCode,+ type Span,+} from "@opentelemetry/api";+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";+import {+ BasicTracerProvider,+ InMemorySpanExporter,+ SimpleSpanProcessor,+ type ReadableSpan,+} from "@opentelemetry/sdk-trace-base";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++import type { AgentToolDescription } from "#channel/agent-description.js";+import type { RouteHandlerArgs } from "#channel/routes.js";+import type { SessionAuthContext } from "#channel/types.js";+import { invokeTool, type InvokeToolRuntime } from "#execution/invoke-tool.js";+import { createToolExecuteWithAuth } from "#execution/tool-auth.js";+import type { HarnessToolDefinition } from "#harness/execute-tool.js";+import { createInstrumentationHooks } from "#instrumentation/lifecycle.js";+import { registerInstrumentationRuntime } from "#instrumentation/runtime.js";+import { createLogger, logError } from "#internal/logging.js";+import { MCP_PROTOCOL_VERSION } from "#internal/mcp/streamable-http-server.js";+import {+ attachAgentInfoRouteResponse,+ attachRouteChannelName,+ attachRouteSessionCreator,+} from "#internal/nitro/routes/channel-route-context.js";+import { mockAgentRouteArgs } from "#internal/testing/mocks/mock-route-args.js";+import { mcpChannel } from "#public/channels/mcp.js";+import { createBundledRuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js";+import type { TraceCapturePolicy } from "#shared/trace-policy.js";+import { defineJsonSchema } from "#tools/schema.js";++// Content a tool sees or produces. It reaches a span only when content capture is on.+const SENTINEL = "sentinel-7f3a-content";+const CLIENT_TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736";+const CLIENT_SPAN_ID = "00f067aa0ba902b7";+const TRACEPARENT = `00-${CLIENT_TRACE_ID}-${CLIENT_SPAN_ID}-01`;+const LEGACY_PROTOCOL_VERSION = "2025-11-25";++const alice: SessionAuthContext = {+ attributes: {},+ authenticator: "test",+ principalId: "alice",+ principalType: "user",+};++const log = createLogger("mcp-tools-tracing-test");+const inputJsonSchema = {+ additionalProperties: false,+ properties: { text: { type: "string" } },+ type: "object",+} as const;+const inputSchema = defineJsonSchema(inputJsonSchema);++function tool(name: string, execute: (input: { text?: string }) => unknown): HarnessToolDefinition {+ return {+ description: name,+ execute: createToolExecuteWithAuth({ execute, scope: name }),+ inputSchema,+ name,+ };+}++const tools = [+ tool(+ "lookup",+ async ({ text }) =>+ // A span the tool opens nests under the tool span.+ await apiTrace.getTracer("test.tool").startActiveSpan("nested lookup", async (span) => {+ span.end();+ return { echoed: text, status: "ok" };+ }),+ ),+ tool("explode", async ({ text }) => {+ // Logged failures record on the active span; their text must not be captured.+ logError(log, "the tool failed", new Error(`logged ${text}`));+ throw new Error(`thrown ${text}`);+ }),+];++const runtime: InvokeToolRuntime = {+ agentName: "compiled-agent",+ callbackBaseUrl: "https://agent.example",+ compiledArtifactsSource: createBundledRuntimeCompiledArtifactsSource(),+ manifest: {+ bindings: Object.fromEntries(+ tools.map(({ name }) => [`source:${name}`, { owner: { kind: "application" } } as never]),+ ),+ tools: tools.map(({ name }) => ({ hasExecute: true, name, sourceId: `source:${name}` })),+ },+ nodeId: "__root__",+ origin: { adapter: { kind: "mcp" } as never, agentName: "compiled-agent", channelName: "mcp" },+ sandboxRegistry: { sandbox: null } as never,+ tools: new Map(tools.map((definition) => [definition.name, definition])),+};++const descriptions: AgentToolDescription[] = tools.map(({ name }) => ({+ approval: false,+ description: name,+ inputSchema: inputJsonSchema,+ name,+}));++let exporter: InMemorySpanExporter;+let provider: BasicTracerProvider;++function useTracing(tracePolicy?: TraceCapturePolicy, capturesContent = false): void {+ registerInstrumentationRuntime({+ forceFlush: async () => undefined,+ hooks: createInstrumentationHooks([]),+ otelSettings: {+ functionId: undefined,+ recordInputs: capturesContent,+ recordOutputs: capturesContent,+ traceChannelRequests: false,+ tracePolicy,+ },+ runInContext: (_operation, execute) => execute(),+ shutdown: async () => undefined,+ });+}++beforeEach(() => {+ // Content is captured for a private audience only in development.+ vi.stubEnv("EVE_DEV", "1");+ exporter = new InMemorySpanExporter();+ provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] });+ apiContext.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());+ apiTrace.setGlobalTracerProvider(provider);+ useTracing();+});++afterEach(async () => {+ vi.unstubAllEnvs();+ await provider.shutdown();+ apiTrace.disable();+ apiContext.disable();+});++type Era = "2025" | "2026";++function callRequest(+ era: Era,+ name: string,+ meta: Readonly<Record<string, unknown>> = { traceparent: TRACEPARENT },+): Request {+ const params =+ era === "2026"+ ? {+ _meta: {+ ...meta,+ "io.modelcontextprotocol/clientCapabilities": {},+ "io.modelcontextprotocol/clientInfo": { name: "test-client", version: "0.0.0" },+ "io.modelcontextprotocol/protocolVersion": MCP_PROTOCOL_VERSION,+ },+ arguments: { text: SENTINEL },+ name,+ }+ : { _meta: meta, arguments: { text: SENTINEL }, name };+ return new Request("https://agent.example/eve/v1/mcp", {+ body: JSON.stringify({ id: 1, jsonrpc: "2.0", method: "tools/call", params }),+ headers: {+ accept: "application/json, text/event-stream",+ "content-type": "application/json",+ host: "agent.example",+ ...(era === "2026"+ ? {+ "mcp-method": "tools/call",+ "mcp-name": name,+ "mcp-protocol-version": MCP_PROTOCOL_VERSION,+ }+ : { "mcp-protocol-version": LEGACY_PROTOCOL_VERSION }),+ },+ method: "POST",+ });+}++async function call(request: Request): Promise<{ status: number; body: any }> {+ const channel = mcpChannel({ agent: false, auth: () => alice, tools: true });+ const post = channel.routes[1]!;+ if (post.transport === "websocket") throw new Error("expected HTTP route");+ const unavailable = () => {+ throw new Error("Route operation is unavailable in this test.");+ };+ const base: RouteHandlerArgs = {+ ...mockAgentRouteArgs(),+ attachSession: unavailable,+ describe: async () => ({ name: "compiled-agent", tools: descriptions }),+ from: unavailable,+ invokeTool: (name, input, options) => invokeTool(runtime, name, input, options),+ params: {},+ requestIp: "127.0.0.1",+ resolveSession: vi.fn(),+ to: unavailable,+ waitUntil: vi.fn(),+ };+ const args = attachRouteChannelName(+ attachAgentInfoRouteResponse(attachRouteSessionCreator(base, vi.fn()), async () =>+ Response.json({ agent: { name: "compiled-agent" } }),+ ),+ "mcp",+ );+ const response = await post.handler(request, args);+ const text = await response.text();+ if (response.status !== 200) throw new Error(`MCP answered ${response.status}: ${text}`);+ const data = response.headers.get("content-type")?.includes("text/event-stream")+ ? text+ .split("\n")+ .find((line) => line.startsWith("data: "))!+ .slice("data: ".length)+ : text;+ return { body: JSON.parse(data), status: response.status };+}++function finished(): ReadableSpan[] {+ return exporter.getFinishedSpans();+}++function toolSpan(name: string): ReadableSpan {+ const span = finished().find((candidate) => candidate.name === `execute_tool ${name}`);+ if (span === undefined) throw new Error(`no execute_tool span for ${name}`);+ return span;+}++function parentSpanId(span: ReadableSpan): string | undefined {+ return span.parentSpanContext?.spanId;+}++/** Everything an exporter would ship for a span, events and status included. */+function exported(span: ReadableSpan): string {+ return JSON.stringify({+ attributes: span.attributes,+ events: span.events,+ links: span.links,+ name: span.name,+ status: span.status,+ });+}++function expectNoContent(): void {+ for (const span of finished()) expect(exported(span)).not.toContain(SENTINEL);+}++describe.each<Era>(["2025", "2026"])("mcpChannel tool tracing over MCP %s", (era) => {+ it("parents a successful call on the client's trace and nests the tool's spans", async () => {+ const { body, status } = await call(callRequest(era, "lookup"));+ expect(status).toBe(200);+ expect(body.result.isError).toBeUndefined();++ const span = toolSpan("lookup");+ expect(span.spanContext().traceId).toBe(CLIENT_TRACE_ID);+ expect(parentSpanId(span)).toBe(CLIENT_SPAN_ID);+ expect(span.status.code).not.toBe(SpanStatusCode.ERROR);+ expect(span.attributes).toMatchObject({+ "eve.channel.name": "mcp",+ "eve.tool.outcome": "completed",+ "gen_ai.agent.name": "compiled-agent",+ "gen_ai.operation.name": "execute_tool",+ "gen_ai.tool.name": "lookup",+ "gen_ai.tool.type": "function",+ });+ expect(span.attributes["gen_ai.tool.call.id"]).toMatch(/^call_/);+ expect(Object.values(span.attributes)).toContainEqual(expect.stringMatching(/^call_session_/));++ const nested = finished().find((candidate) => candidate.name === "nested lookup")!;+ expect(nested.spanContext().traceId).toBe(CLIENT_TRACE_ID);+ expect(parentSpanId(nested)).toBe(span.spanContext().spanId);+ expectNoContent();+ });+});++describe("mcpChannel tool trace context", () => {+ it.each([+ ["malformed", { traceparent: "not-a-traceparent" }],+ ["all-zero", { traceparent: `00-${"0".repeat(32)}-${CLIENT_SPAN_ID}-01` }],+ ["oversized", { traceparent: `${TRACEPARENT}${"0".repeat(4096)}` }],+ ["non-string", { traceparent: 42 }],+ ])("ignores a %s traceparent", async (_label, meta) => {+ await call(callRequest("2026", "lookup", meta));+ const span = toolSpan("lookup");+ expect(span.spanContext().traceId).not.toBe(CLIENT_TRACE_ID);+ expect(parentSpanId(span)).toBeUndefined();+ });++ it("keeps a bounded tracestate and drops an oversized one", async () => {+ await call(callRequest("2026", "lookup", { traceparent: TRACEPARENT, tracestate: "vendor=a" }));+ expect(toolSpan("lookup").parentSpanContext?.traceState?.get("vendor")).toBe("a");++ exporter.reset();+ const oversized = `vendor=${"a".repeat(600)}`;+ await call(callRequest("2026", "lookup", { traceparent: TRACEPARENT, tracestate: oversized }));+ const span = toolSpan("lookup");+ expect(parentSpanId(span)).toBe(CLIENT_SPAN_ID);+ expect(span.parentSpanContext?.traceState).toBeUndefined();+ });++ it("never lets MCP baggage choose the audience or content capture", async () => {+ const seen: string[] = [];+ useTracing((trace) => {+ seen.push(trace.audience);+ return { emit: true, recordInputs: true, recordOutputs: true };+ });+ await call(callRequest("2026", "lookup"));+ await call(+ callRequest("2026", "lookup", {+ baggage: "eve.audience=public,eve.parent_session=attacker,eve.conversation.id=x",+ traceparent: TRACEPARENT,+ }),+ );+ expect(seen).toHaveLength(2);+ expect(seen[1]).toBe(seen[0]);+ for (const span of finished()) {+ expect(JSON.stringify(span.attributes)).not.toContain("attacker");+ }+ // The policy allows content, but the declaration does not capture it.+ expectNoContent();+ });+});++describe("invokeTool trace policy", () => {+ it("emits no spans, nested ones included, when tracePolicy opts out", async () => {+ useTracing(() => false);+ await call(callRequest("2026", "lookup"));+ expect(finished()).toEqual([]);+ });++ it("keeps a failing tool's text off an existing parent span when no span is emitted", async () => {+ useTracing(() => false);+ const parent = apiTrace.getTracer("test.parent").startSpan("platform request");+ await apiContext.with(apiTrace.setSpan(apiContext.active(), parent), () =>+ invokeTool(runtime, "explode", { text: SENTINEL }, { auth: alice }),+ );+ (parent as Span).end();+ expect(finished().map((span) => span.name)).toEqual(["platform request"]);+ expectNoContent();+ });++ it("keeps a throwing tracePolicy's text off an existing parent span", async () => {+ useTracing(() => {+ throw new Error(`policy ${SENTINEL}`);+ });+ const parent = apiTrace.getTracer("test.parent").startSpan("platform request");+ const result = await apiContext.with(apiTrace.setSpan(apiContext.active(), parent), () =>+ invokeTool(runtime, "lookup", { text: "plain" }, { auth: alice }),+ );+ (parent as Span).end();+ expect(result).toMatchObject({ status: "completed" });+ expect(finished().map((span) => span.name)).toEqual(["platform request"]);+ expect(finished()[0]!.status.code).not.toBe(SpanStatusCode.ERROR);+ expectNoContent();+ });++ it("keeps a failing tool's text off an existing parent span with tracing undeclared", async () => {+ registerInstrumentationRuntime({+ forceFlush: async () => undefined,+ hooks: createInstrumentationHooks([]),+ otelSettings: undefined,+ runInContext: (_operation, execute) => execute(),+ shutdown: async () => undefined,+ });+ const parent = apiTrace.getTracer("test.parent").startSpan("platform request");+ const result = await apiContext.with(apiTrace.setSpan(apiContext.active(), parent), () =>+ invokeTool(runtime, "explode", { text: SENTINEL }, { auth: alice }),+ );+ (parent as Span).end();+ expect(result).toMatchObject({ status: "failed" });+ expect(finished().map((span) => span.name)).toEqual(["platform request"]);+ expectNoContent();+ });+});++describe("invokeTool content capture", () => {+ const record = { emit: true, recordInputs: true, recordOutputs: true } as const;++ it("records outputs without inputs when only outputs are allowed", async () => {+ useTracing(() => ({ emit: true, recordInputs: false, recordOutputs: true }), true);+ await invokeTool(runtime, "lookup", { text: SENTINEL }, { auth: alice });+ const span = toolSpan("lookup");+ expect(span.attributes["gen_ai.tool.call.arguments"]).toBeUndefined();+ expect(span.attributes["gen_ai.tool.call.result"]).toBeDefined();+ });++ it("records neither when the OpenTelemetry declaration turns content off", async () => {+ useTracing(() => record, false);+ await invokeTool(runtime, "lookup", { text: SENTINEL }, { auth: alice });+ const span = toolSpan("lookup");+ expect(span.attributes["gen_ai.tool.call.arguments"]).toBeUndefined();+ expect(span.attributes["gen_ai.tool.call.result"]).toBeUndefined();+ expectNoContent();+ });++ it("keeps a recorded failure's error off an existing parent span", async () => {+ useTracing(() => record, true);+ const parent = apiTrace.getTracer("test.parent").startSpan("platform request");+ await apiContext.with(apiTrace.setSpan(apiContext.active(), parent), () =>+ invokeTool(runtime, "explode", { text: SENTINEL }, { auth: alice }),+ );+ (parent as Span).end();++ expect(toolSpan("explode").status.message).toBe(`thrown ${SENTINEL}`);+ const parentSpan = finished().find((candidate) => candidate.name === "platform request")!;+ expect(exported(parentSpan)).not.toContain(SENTINEL);+ });+});diff --git a/packages/eve/src/public/channels/mcp.test.ts b/packages/eve/src/public/channels/mcp.test.tsindex 44375b014..e1a9c39fd 100644--- a/packages/eve/src/public/channels/mcp.test.ts+++ b/packages/eve/src/public/channels/mcp.test.ts@@ -1,5 +1,7 @@import { describe, expect, it, vi } from "vitest";+import type { AgentToolDescription } from "#channel/agent-description.js";+import type { InvokeToolFn, InvokeToolResult } from "#channel/invoke-tool.js";import type { SessionAuthContext } from "#channel/types.js";import type { RouteHandlerArgs } from "#channel/routes.js";import {@@ -486,12 +488,173 @@ describe("mcpChannel", () => {});});-function routeArgs(createSession: () => Promise<never> = vi.fn()): RouteHandlerArgs {+describe("mcpChannel tools", () => {+ const lookup: AgentToolDescription = {+ approval: false,+ description: "Looks up one order.",+ inputSchema: {+ properties: { id: { type: "string" } },+ required: ["id"],+ type: "object",+ },+ name: "lookup",+ outputSchema: { properties: { status: { type: "string" } }, type: "object" },+ };+ const shadow: AgentToolDescription = {+ approval: false,+ description: "An authored tool named like the channel's.",+ inputSchema: { type: "object" },+ name: "agent_start",+ };+ const note: AgentToolDescription = {+ approval: false,+ description: "Writes a note.",+ inputSchema: { type: "object" },+ name: "note",+ };++ function serve(+ options: { readonly agent?: boolean; readonly tools?: boolean },+ invokeTool: InvokeToolFn = vi.fn(),+ tools: readonly AgentToolDescription[] = [shadow, lookup, note],+ ) {+ const channel = mcpChannel({ auth: () => principal, ...options });+ const post = channel.routes[1]!;+ if (post.transport === "websocket") throw new Error("expected HTTP route");+ const args = routeArgs(vi.fn(), {+ describe: async () => ({ name: "compiled-agent", tools }),+ invokeTool,+ });+ return async (method: string, params?: unknown) =>+ (await jsonRpcResponse(+ await post.handler(mcpRequest({ id: 1, jsonrpc: "2.0", method, params }), args),+ )) as { result?: Record<string, any>; error?: { code: number } };+ }++ it("lists the agent's tools after agent_*, and only them with agent: false", async () => {+ const names = async (rpc: ReturnType<typeof serve>) =>+ (await rpc("tools/list")).result!.tools.map((tool: { name: string }) => tool.name);+ const both = serve({ tools: true });+ expect(await names(both)).toEqual([+ "agent_start",+ "agent_get",+ "agent_update",+ "agent_cancel",+ "lookup",+ "note",+ ]);+ const listed = (await both("tools/list")).result!.tools;+ expect(listed[0].description).toContain("Investigates tasks.");+ expect(listed[4]).toMatchObject({+ inputSchema: lookup.inputSchema,+ outputSchema: lookup.outputSchema,+ });++ const toolsOnly = serve({ agent: false, tools: true });+ expect(await names(toolsOnly)).toEqual(["agent_start", "lookup", "note"]);+ const initialize = await toolsOnly("initialize", {+ capabilities: {},+ clientInfo: { name: "test-client", version: "0.0.0" },+ protocolVersion: MCP_LEGACY_PROTOCOL_VERSION,+ });+ expect(initialize.result!.instructions).toBeUndefined();++ expect(await names(serve({}))).toHaveLength(4);+ expect(() => mcpChannel({ agent: false, auth: none() })).toThrow(+ "mcpChannel publishes nothing with agent and tools both false. Enable one.",+ );+ });++ it("returns non-object output as structured content when the tool declares a schema", async () => {+ const count: AgentToolDescription = {+ approval: false,+ description: "Counts orders.",+ inputSchema: { type: "object" },+ name: "count",+ outputSchema: { type: "number" },+ };+ const invokeTool = vi.fn<InvokeToolFn>(async () => ({+ modelOutput: { type: "json", value: 42 },+ output: 42,+ status: "completed",+ }));+ const rpc = serve({ agent: false, tools: true }, invokeTool, [count]);+ const called = (await rpc("tools/call", { arguments: {}, name: "count" })).result!;+ expect(called.isError).toBeUndefined();+ // The SDK wraps a non-object as `{ result }` on 2025-era connections.+ expect(called.structuredContent).toEqual({ result: 42 });+ });++ it("runs each call through invokeTool as the caller and maps its outcome", async () => {+ const completed = (output: unknown, text?: string): InvokeToolResult => ({+ modelOutput:+ text === undefined ? { type: "json", value: output } : { type: "text", value: text },+ output,+ status: "completed",+ });+ const rows: Array<[string, InvokeToolResult, object]> = [+ [+ "lookup",+ completed({ status: "shipped" }),+ {+ content: [{ text: '{"status":"shipped"}', type: "text" }],+ structuredContent: { status: "shipped" },+ },+ ],+ ["note", completed("saved", "Saved."), { content: [{ text: "Saved.", type: "text" }] }],+ ["note", { message: "Bad id.", status: "invalid-input" }, { code: "invalid_input" }],+ ["note", { reason: "Not today.", status: "denied" }, { code: "denied" }],+ ["note", { status: "approval-required" }, { code: "approval_required" }],+ [+ "note",+ { connections: ["linear"], status: "authorization-required" },+ { code: "authorization_required", message: expect.stringContaining("linear") },+ ],+ ["note", { message: "Boom.", status: "failed" }, { code: "internal", message: "Boom." }],+ ];+ for (const [name, result, expected] of rows) {+ const invokeTool = vi.fn<InvokeToolFn>(async () => result);+ const rpc = serve({ tools: true }, invokeTool);+ const args = name === "lookup" ? { id: "7" } : {};+ const called = (await rpc("tools/call", { arguments: args, name })).result!;+ if ("code" in expected) {+ expect(called, result.status).toMatchObject({+ isError: true,+ structuredContent: { error: expected },+ });+ } else {+ expect(called, result.status).toMatchObject(expected);+ }+ expect(invokeTool).toHaveBeenCalledWith(name, args, {+ auth: principal,+ signal: expect.any(AbortSignal),+ });+ }+ });++ it("checks arguments against the tool's JSON schema before invoking it", async () => {+ const invokeTool = vi.fn<InvokeToolFn>();+ const rpc = serve({ tools: true }, invokeTool);+ const called = await rpc("tools/call", { arguments: { id: 7 }, name: "lookup" });+ expect(called.error).toBeUndefined();+ expect(called.result).toMatchObject({+ isError: true,+ structuredContent: { error: { code: "invalid_input", retryable: false } },+ });+ expect(invokeTool).not.toHaveBeenCalled();+ });+});++function routeArgs(+ createSession: () => Promise<never> = vi.fn(),+ overrides: Partial<RouteHandlerArgs> = {},+): RouteHandlerArgs {const unavailable = () => {throw new Error("Route operation is unavailable in this test.");};const args: RouteHandlerArgs = {...mockAgentRouteArgs(),+ ...overrides,attachSession: unavailable,from: unavailable,params: {},diff --git a/packages/eve/src/tracing/invoke-tool-parity.integration.test.ts b/packages/eve/src/tracing/invoke-tool-parity.integration.test.tsnew file mode 100644index 000000000..e8fc47a47--- /dev/null+++ b/packages/eve/src/tracing/invoke-tool-parity.integration.test.ts@@ -0,0 +1,434 @@+import { context as apiContext, propagation, trace as apiTrace } from "@opentelemetry/api";+import {+ InMemorySpanExporter,+ SimpleSpanProcessor,+ type ReadableSpan,+} from "@opentelemetry/sdk-trace-base";+import { generateText, stepCountIs } from "ai";+import { MockLanguageModelV3 } from "ai/test";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++import type { SessionAuthContext } from "#channel/types.js";+import { ContextContainer, contextStorage } from "#context/container.js";+import {+ AuthKey,+ ChannelInstrumentationKey,+ InitiatorAuthKey,+ SessionIdKey,+ SessionKey,+} from "#context/keys.js";+import { invokeTool, type InvokeToolRuntime } from "#execution/invoke-tool.js";+import { createToolExecuteWithAuth } from "#execution/tool-auth.js";+import type { HarnessToolDefinition } from "#harness/execute-tool.js";+import { buildToolSet } from "#harness/tools.js";+import {+ actionIdempotencyKey,+ turnIdempotencyKey,+ type InstrumentationEvent,+ type InstrumentationProviderDefinition,+} from "#instrumentation/lifecycle.js";+import {+ bindInstrumentationRuntime,+ type InstrumentationRuntime,+} from "#instrumentation/runtime.js";+import { createBundledRuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js";+import { ConversationContextKey } from "#shared/conversation-context.js";+import type { TraceCapturePolicy } from "#shared/trace-policy.js";+import { defineJsonSchema } from "#tools/schema.js";+import { installInstrumentationRuntime } from "#tracing/install-instrumentation-runtime.js";+import { collectOtelPipeline, otel, otelIntegration } from "#tracing/otel-declaration.js";++/**+ * One tool, run by a model in a conversation and by `invokeTool`, must export+ * the same `execute_tool` span and publish the same provider events. Only the+ * differences listed here are allowed, each because a direct call has no turn.+ */+const ALLOWED_ATTRIBUTE_DIFFERENCES = new Set([+ // Ids: a model picks a conversation's call id; a direct call mints its own session.+ "agent.run.id",+ "gen_ai.conversation.id",+ "gen_ai.tool.call.id",+ "vercel.session_id",+ // Direct-call markers: the processor's trace ownership and the call's result status.+ "eve.channel.kind",+ "eve.channel.name",+ "eve.tool.invocation",+ "eve.tool.outcome",+ // Both spans carry it; run times differ by construction.+ "gen_ai.execute_tool.duration",+]);++const AGENT = "parity-agent";+const SECRET = "sentinel-parity-content";+const alice: SessionAuthContext = {+ attributes: {},+ authenticator: "test",+ principalId: "alice",+ principalType: "user",+};+/**+ * A clock the duration test drives through a stubbed `performance.now`, so it+ * measures time spent in each step without waiting for real time to pass.+ */+const clock = { now: 0 };+const EXECUTE_MS = 25;++const inputSchema = defineJsonSchema({+ additionalProperties: false,+ properties: { text: { type: "string" } },+ type: "object",+} as const);++function tool(name: string, execute: (input: { text?: string }) => unknown): HarnessToolDefinition {+ return {+ description: name,+ execute: createToolExecuteWithAuth({ execute, scope: name }),+ inputSchema,+ name,+ };+}++const tools = new Map(+ [+ tool("lookup", async ({ text }) => {+ clock.now += EXECUTE_MS;+ return apiTrace.getTracer("test.tool").startActiveSpan("nested lookup", (span) => {+ span.end();+ return { echoed: text };+ });+ }),+ tool("explode", async ({ text }) => {+ throw new Error(`thrown ${text}`);+ }),+ ].map((definition) => [definition.name, definition]),+);++const directRuntime: InvokeToolRuntime = {+ agentName: AGENT,+ callbackBaseUrl: "https://agent.example",+ compiledArtifactsSource: createBundledRuntimeCompiledArtifactsSource(),+ manifest: {+ bindings: Object.fromEntries(+ [...tools.keys()].map((name) => [+ `source:${name}`,+ { owner: { kind: "application" } } as never,+ ]),+ ),+ tools: [...tools.keys()].map((name) => ({+ hasExecute: true,+ name,+ sourceId: `source:${name}`,+ })),+ },+ nodeId: "__root__",+ sandboxRegistry: { sandbox: null } as never,+ tools,+};++let exporter: InMemorySpanExporter;+let events: InstrumentationEvent[];+let runtime: InstrumentationRuntime | undefined;+let conversations = 0;++type Policy = "record" | "inputs only" | "none";+const POLICIES: Readonly<Record<Policy, ReturnType<TraceCapturePolicy>>> = {+ "inputs only": { emit: true, recordInputs: true, recordOutputs: false },+ none: { emit: true, recordInputs: false, recordOutputs: false },+ record: { emit: true, recordInputs: true, recordOutputs: true },+};++/**+ * Installs the runtime a deployment builds from `otel({ tracePolicy })` and a+ * span-processor destination, plus a provider that records tool events under+ * the same policy. Both paths read their content decision from it.+ */+function install(+ policy: Policy,+ options: { readonly declareOtel?: boolean; readonly slowStartMs?: number } = {},+): InstrumentationRuntime {+ const tracePolicy: TraceCapturePolicy = () => POLICIES[policy];+ const recorder: InstrumentationProviderDefinition = {+ events: {+ "tool.call.completed": (event) => void events.push(event),+ "tool.call.failed": (event) => void events.push(event),+ "tool.call.started": async (event) => {+ events.push(event);+ if (options.slowStartMs !== undefined) clock.now += options.slowStartMs;+ },+ },+ name: "recorder",+ tracePolicy,+ };+ delete (globalThis as Record<symbol, unknown>)[Symbol.for("eve.instrumentation-runtime")];+ runtime = installInstrumentationRuntime({+ collected: collectOtelPipeline(+ options.declareOtel === false+ ? []+ : [+ otel({ tracePolicy }),+ otelIntegration({ spanProcessors: [new SimpleSpanProcessor(exporter)] }),+ ],+ ),+ frameworkVersion: "test",+ providers: [recorder],+ serviceName: "parity",+ });+ return runtime;+}++/**+ * Runs `toolName` the way a turn does: the model call gets the attempt's own+ * telemetry, so the AI SDK hook bridge publishes the tool events, inside an+ * action as the harness reports it.+ */+async function runInConversation(toolName: string, modelInput: string): Promise<void> {+ // A fresh session per run: agent trace state is keyed by session and turn.+ const sessionId = `conversation-${++conversations}`;+ const callId = "call-model";+ const trace = {+ agentName: AGENT,+ audience: "private" as const,+ channel: { kind: "http" as const },+ environment: "development" as const,+ principalType: alice.principalType,+ };+ const ctx = new ContextContainer();+ ctx.set(ChannelInstrumentationKey, { kind: "http", metadata: {} });+ ctx.set(ConversationContextKey, trace);+ ctx.set(AuthKey, alice);+ ctx.set(InitiatorAuthKey, alice);+ ctx.set(SessionIdKey, sessionId);+ ctx.setVirtualContext(SessionKey, {+ auth: { current: alice, initiator: alice },+ sessionId,+ turn: { id: "turn_0", sequence: 0 },+ });+ const bound = bindInstrumentationRuntime(runtime!, ctx, {+ agentName: AGENT,+ rootSessionId: sessionId,+ sessionId,+ })!;+ const hooks = runtime!.hooks.forTrace!({ ...trace });+ await contextStorage.run(ctx, async () => {+ await bound.preparePreamble({ sequence: 0, sessionStarted: false, turnId: "turn_0" });+ await bound+ .prepareExecution()+ .runStep(+ { environment: "development", eveVersion: "test", hasInput: true, session: { sessionId } },+ async (step) => {+ const attempt = step.prepareAttempt({ attemptIndex: 0, stepIndex: 0, turnId: "turn_0" });+ const scope = attempt.scope;+ const actionKey = actionIdempotencyKey(sessionId, "turn_0", callId);+ await hooks.publish({+ callId,+ idempotencyKey: actionKey,+ input: { text: SECRET },+ kind: "tool-call",+ name: toolName,+ scope,+ type: "action.started",+ });+ await generateText({+ model: new MockLanguageModelV3({+ doGenerate: async () => ({+ content: [{ input: modelInput, toolCallId: callId, toolName, type: "tool-call" }],+ finishReason: { raw: undefined, unified: "tool-calls" },+ usage: {+ inputTokens: { cacheRead: 0, cacheWrite: 0, noCache: 1, total: 1 },+ outputTokens: { reasoning: 0, text: 1, total: 1 },+ },+ warnings: [],+ }),+ }),+ prompt: "Use the tool.",+ stopWhen: stepCountIs(1),+ telemetry: attempt.telemetry,+ tools: buildToolSet({ tools }),+ });+ await hooks.publish({+ idempotencyKey: actionKey,+ outcome: "completed",+ output: { output: {}, type: "result" },+ scope,+ type: "action.completed",+ });+ await attempt.complete();+ },+ );+ await hooks.publish({+ idempotencyKey: turnIdempotencyKey(sessionId, "turn_0"),+ sessionId,+ turnId: "turn_0",+ type: "turn.completed",+ });+ });+}++interface Captured {+ readonly events: readonly unknown[];+ readonly nestedParent: boolean | undefined;+ readonly raw: ReadableSpan;+ readonly span: ReturnType<typeof comparable>;+}++async function capture(run: () => Promise<unknown>, toolName: string): Promise<Captured> {+ exporter.reset();+ events = [];+ await run();+ await runtime!.forceFlush();+ const spans = exporter.getFinishedSpans();+ const toolSpans = spans.filter((span) => span.name === `execute_tool ${toolName}`);+ expect(toolSpans).toHaveLength(1);+ const toolSpan = toolSpans[0]!;+ const nested = spans.find((span) => span.name === "nested lookup");+ return {+ events: events.map(comparableEvent),+ nestedParent:+ nested === undefined+ ? undefined+ : nested.parentSpanContext?.spanId === toolSpan.spanContext().spanId,+ raw: toolSpan,+ span: comparable(toolSpan),+ };+}++const direct = (toolName: string, input: unknown) => () =>+ invokeTool(directRuntime, toolName, input, { auth: alice });++function comparable(span: ReadableSpan) {+ return {+ attributes: Object.fromEntries(+ Object.entries(span.attributes).filter(([name]) => !ALLOWED_ATTRIBUTE_DIFFERENCES.has(name)),+ ),+ // Stack traces name different frames; the exception's type and message must match.+ exceptions: span.events+ .filter((event) => event.name === "exception")+ .map((event) => ({+ message: event.attributes?.["exception.message"],+ type: event.attributes?.["exception.type"],+ })),+ kind: span.kind,+ name: span.name,+ status: span.status,+ };+}++function comparableEvent(event: InstrumentationEvent) {+ const {+ idempotencyKey: _key,+ scope: _scope,+ ...rest+ } = event as InstrumentationEvent & {+ readonly idempotencyKey: string;+ readonly scope: unknown;+ };+ const shape: Record<string, unknown> = { ...rest };+ // Ids differ by construction, as on the span.+ if ("callId" in shape) shape.callId = "<call>";+ // Run times differ by construction; both sides must report one.+ if (typeof shape.durationMs === "number") shape.durationMs = "<ms>";+ const output = shape.output as { readonly error?: unknown } | undefined;+ if (output?.error instanceof Error) {+ shape.output = { ...output, error: `${output.error.name}: ${output.error.message}` };+ }+ return shape;+}++beforeEach(() => {+ exporter = new InMemorySpanExporter();+ // Content is captured for a private audience only in development, the+ // environment `runInConversation` declares; the direct call reads it from here.+ vi.stubEnv("EVE_DEV", "1");+});++afterEach(async () => {+ vi.restoreAllMocks();+ vi.unstubAllEnvs();+ await runtime?.shutdown();+ runtime = undefined;+ apiTrace.disable();+ apiContext.disable();+ propagation.disable();+});++const ARGUMENTS = JSON.stringify({ text: SECRET });++describe.each<Policy>(["record", "inputs only", "none"])(+ "invokeTool against a conversation's tool call, tracePolicy %s",+ (policy) => {+ it.each(["lookup", "explode"])("exports the same %s span and events", async (toolName) => {+ install(policy);+ const conversation = await capture(() => runInConversation(toolName, ARGUMENTS), toolName);+ const viaInvoke = await capture(direct(toolName, { text: SECRET }), toolName);++ expect(viaInvoke.span).toEqual(conversation.span);+ expect(viaInvoke.events).toEqual(conversation.events);+ expect(viaInvoke.nestedParent).toBe(conversation.nestedParent);+ if (toolName === "lookup") expect(viaInvoke.nestedParent).toBe(true);++ // The policy applies the same way on both paths.+ const text = JSON.stringify(viaInvoke.span);+ expect(text.includes(SECRET)).toBe(policy !== "none");+ expect(viaInvoke.span.attributes["gen_ai.tool.call.result"] !== undefined).toBe(+ policy === "record" && toolName === "lookup",+ );+ });+ },+);++describe("invokeTool's recorded arguments", () => {+ it("records the checked input, as a conversation does for a call with no arguments", async () => {+ install("record");+ const conversation = await capture(() => runInConversation("lookup", ""), "lookup");+ const viaInvoke = await capture(direct("lookup", undefined), "lookup");++ expect(viaInvoke.span.attributes["gen_ai.tool.call.arguments"]).toBe("{}");+ expect(viaInvoke.span).toEqual(conversation.span);+ expect(viaInvoke.events).toEqual(conversation.events);+ });++ it("records a JSON string input as the object execute receives", async () => {+ install("record");+ const viaInvoke = await capture(direct("lookup", ARGUMENTS), "lookup");+ expect(viaInvoke.span.attributes["gen_ai.tool.call.arguments"]).toBe(ARGUMENTS);+ });++ it("records no arguments for rejected input", async () => {+ install("record");+ const viaInvoke = await capture(direct("lookup", { text: 7, secret: SECRET }), "lookup");+ expect(viaInvoke.raw.attributes["eve.tool.outcome"]).toBe("invalid-input");+ expect(viaInvoke.span.attributes["gen_ai.tool.call.arguments"]).toBeUndefined();+ expect(viaInvoke.events).toEqual([]);+ expect(JSON.stringify(viaInvoke.span)).not.toContain(SECRET);+ });+});++describe("invokeTool's execute_tool duration", () => {+ it("counts execute only, not slow tool.call.started handlers", async () => {+ vi.spyOn(performance, "now").mockImplementation(() => clock.now);+ install("record", { slowStartMs: 5_000 });+ const viaInvoke = await capture(direct("lookup", { text: SECRET }), "lookup");++ expect(viaInvoke.raw.attributes["gen_ai.execute_tool.duration"]).toBe(EXECUTE_MS / 1000);+ const completed = events.find((event) => event.type === "tool.call.completed");+ expect(completed).toMatchObject({ durationMs: EXECUTE_MS });+ });+});++describe("invokeTool without declared OpenTelemetry", () => {+ it("still publishes the tool call to providers, and exports no span", async () => {+ install("record", { declareOtel: false });+ events = [];+ await invokeTool(directRuntime, "lookup", { text: SECRET }, { auth: alice });++ expect(events.map(comparableEvent)).toEqual([+ { callId: "<call>", input: { text: SECRET }, toolName: "lookup", type: "tool.call.started" },+ {+ durationMs: "<ms>",+ output: { output: { echoed: SECRET }, type: "result" },+ type: "tool.call.completed",+ },+ ]);+ expect(exporter.getFinishedSpans()).toEqual([]);+ });+});diff --git a/packages/eve/src/tracing/invoke-tool-span.ts b/packages/eve/src/tracing/invoke-tool-span.tsindex 04d19c88f..8d0ed1199 100644--- a/packages/eve/src/tracing/invoke-tool-span.ts+++ b/packages/eve/src/tracing/invoke-tool-span.ts@@ -292,6 +292,7 @@ function toolCallEvents(: { type: "error" as const };await hooks.publish(Object.freeze({+ durationMs: outcome.durationMs,idempotencyKey,output: Object.freeze(output),scope,diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex d11eb254d..76e034b97 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -974,6 +974,28 @@ importers:specifier: 'catalog:'version: 7.0.2+ e2e/fixtures/agent-mcp:+ dependencies:+ '@eve-e2e/config':+ specifier: workspace:*+ version: link:../e2e-config+ '@workflow/world-postgres':+ specifier: 'catalog:'+ version: 5.0.1(@opentelemetry/[email protected])(@types/[email protected])(@upstash/[email protected])([email protected])([email protected])([email protected])+ eve:+ specifier: workspace:*+ version: link:../../../packages/eve+ zod:+ specifier: 'catalog:'+ version: 4.5.4+ devDependencies:+ '@types/node':+ specifier: 'catalog:'+ version: 24.13.3+ typescript:+ specifier: 'catalog:'+ version: 7.0.2+e2e/fixtures/agent-memory:dependencies:'@eve-e2e/config':