Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

Apply tool stubs across turns and local subagents (2/3) (#4432)

Philipp Comanseaadf3c77531
diff --git a/packages/eve/src/channel/types.ts b/packages/eve/src/channel/types.ts
index 7d85f3029..f3088faf1 100644
--- a/packages/eve/src/channel/types.ts
+++ b/packages/eve/src/channel/types.ts
@@ -16,6 +16,7 @@ import type { JsonObject } from "#shared/json.js";
import type { InstrumentationDecision } from "#shared/instrumentation-decision.js";
import type { ForwardedTraceAssertion } from "#shared/forwarded-trace-policy.js";
import type { ConversationContext } from "#shared/conversation-context.js";
+import type { StubScope } from "#tool-stubs/types.js";
export type { ContextAccessor } from "#context/key.js";
export type { ChannelInstrumentationProjection } from "#channel/instrumentation.js";
@@ -430,6 +431,14 @@ export interface SessionCapabilities {
* subagent tool wrapper).
*/
export interface RunInput {
+ /**
+ * @internal Set by the server after it authorizes tool stubs.
+ * Local subagents match rules qualified by their full path, such as researcher/list_tasks.
+ * The root session tracks positions per rule; subagent sessions at the same path
+ * share those positions. Unprefixed rules apply only to root tools.
+ * Remote agents do not receive these stubs.
+ */
+ readonly toolStubs?: StubScope;
/** Server-supplied provenance inherited by locally delegated scheduled work. */
readonly schedule?: import("#context/session-schedule.js").SessionSchedule;
readonly adapter: ChannelAdapter<any>;
diff --git a/packages/eve/src/context/keys.ts b/packages/eve/src/context/keys.ts
index d62dab3ff..13794d040 100644
--- a/packages/eve/src/context/keys.ts
+++ b/packages/eve/src/context/keys.ts
@@ -18,6 +18,7 @@ import type {
SessionTurn,
} from "#channel/types.js";
import { ContextKey } from "#context/key.js";
+import { STUB_CONTEXT_KEY, type StubScope } from "#tool-stubs/types.js";
import {
SESSION_INBOX_CONTEXT_KEY,
type SessionInboxAddress,
@@ -79,6 +80,7 @@ export interface Session {
// ---------------------------------------------------------------------------
export const AuthKey = new ContextKey<SessionAuthContext | null>("eve.auth");
+export const ToolStubsKey = new ContextKey<StubScope>(STUB_CONTEXT_KEY);
export const InitiatorAuthKey = new ContextKey<SessionAuthContext | null>("eve.initiatorAuth");
export const SessionIdKey = new ContextKey<string>("eve.sessionId");
export const ConversationIdKey = new ContextKey<string>("eve.conversationId");
diff --git a/packages/eve/src/context/providers/tool-stubs-key.ts b/packages/eve/src/context/providers/tool-stubs-key.ts
new file mode 100644
index 000000000..280d62348
--- /dev/null
+++ b/packages/eve/src/context/providers/tool-stubs-key.ts
@@ -0,0 +1,10 @@
+import { ContextKey } from "#context/key.js";
+import type { StubCall, StubResult } from "#tool-stubs/types.js";
+
+/** Sends tool calls and failures to the workflow that tracks stub responses. */
+export interface ToolStubPlayback {
+ call(call: StubCall): Promise<StubResult>;
+ fail(callId: string, error: string): Promise<void>;
+}
+
+export const ToolStubPlaybackKey = new ContextKey<ToolStubPlayback>("eve.toolStubPlayback");
diff --git a/packages/eve/src/context/providers/tool-stubs.ts b/packages/eve/src/context/providers/tool-stubs.ts
new file mode 100644
index 000000000..f10e3ade3
--- /dev/null
+++ b/packages/eve/src/context/providers/tool-stubs.ts
@@ -0,0 +1,18 @@
+import type { FrameworkContextProvider } from "#context/provider.js";
+import { ToolStubsKey } from "#context/keys.js";
+import { ToolStubPlaybackKey, type ToolStubPlayback } from "#context/providers/tool-stubs-key.js";
+import { callToolStubStep, reportStubFailureStep } from "#execution/tool-stubs/steps.js";
+
+export const toolStubProvider = {
+ key: ToolStubPlaybackKey,
+ create(ctx) {
+ const scope = ctx.get(ToolStubsKey);
+ if (scope === undefined) return undefined;
+ return {
+ value: {
+ call: (call) => callToolStubStep(scope, call),
+ fail: (callId, error) => reportStubFailureStep(scope, callId, error),
+ },
+ };
+ },
+} satisfies FrameworkContextProvider<ToolStubPlayback>;
diff --git a/packages/eve/src/context/run-step.ts b/packages/eve/src/context/run-step.ts
index f7fc94123..aa93c47d2 100644
--- a/packages/eve/src/context/run-step.ts
+++ b/packages/eve/src/context/run-step.ts
@@ -4,6 +4,7 @@ import type { FrameworkContextProvider } from "#context/provider.js";
import { connectionProvider } from "#context/providers/connection.js";
import { sandboxProvider } from "#context/providers/sandbox.js";
import { sessionProvider } from "#context/providers/session.js";
+import { toolStubProvider } from "#context/providers/tool-stubs.js";
/**
* Framework providers in dependency order.
@@ -13,6 +14,7 @@ import { sessionProvider } from "#context/providers/session.js";
*/
const frameworkProviders: readonly FrameworkContextProvider<any>[] = [
sessionProvider,
+ toolStubProvider,
connectionProvider,
sandboxProvider,
];
diff --git a/packages/eve/src/execution/agent-sessions/context.ts b/packages/eve/src/execution/agent-sessions/context.ts
index 065e9df0c..d97a03b39 100644
--- a/packages/eve/src/execution/agent-sessions/context.ts
+++ b/packages/eve/src/execution/agent-sessions/context.ts
@@ -6,6 +6,7 @@ import type {
SessionTraceRoot,
} from "#channel/types.js";
import type { LocalDevRequestProvenance } from "#context/keys.js";
+import { STUB_CONTEXT_KEY, type StubScope } from "#tool-stubs/types.js";
import { readSerializedSessionSchedule, type SessionSchedule } from "#context/session-schedule.js";
import type { DynamicSubagentSelections } from "#execution/agent-sessions/target.js";
import type { PreparedCoordinationDispatch } from "#execution/coordination-dispatch-shared.js";
@@ -34,6 +35,7 @@ import {
* of it: each message carries the auth of the call that sends it.
*/
export interface AgentSessionContext {
+ readonly toolStubs?: StubScope;
/** The agents the call may open, by name, which `ctx.agents` lists. */
readonly agents: Readonly<Record<string, WorkflowAgentMetadata>>;
readonly bundle: AgentSessionBundle;
@@ -110,6 +112,7 @@ export function captureAgentSessionContext(
const { batch, session } = caller;
return {
agents: caller.workflowAgents,
+ toolStubs: caller.serializedContext[STUB_CONTEXT_KEY] as StubScope | undefined,
bundle: {
nodeId: caller.bundle.nodeId,
source: serializeDurableCompiledArtifactsSource(caller.bundle.compiledArtifactsSource),
diff --git a/packages/eve/src/execution/agent-sessions/steps.ts b/packages/eve/src/execution/agent-sessions/steps.ts
index e6c576be6..86ad3611a 100644
--- a/packages/eve/src/execution/agent-sessions/steps.ts
+++ b/packages/eve/src/execution/agent-sessions/steps.ts
@@ -35,6 +35,7 @@ import {
} from "#execution/agent-sessions/remote.js";
import { buildSubagentRunInput } from "#subagents/tool.js";
import { resolveConversationId } from "#shared/conversation-identity.js";
+import { stubToolPath } from "#tool-stubs/target.js";
const log = createLogger("execution.agent-sessions");
@@ -256,8 +257,18 @@ async function startLocalSession(
dynamicSubagentAgentConfig: target.dynamicSubagentAgentConfig,
nodeId: action.nodeId,
});
+ let childToolStubs = context.toolStubs;
+ if (childToolStubs !== undefined) {
+ childToolStubs = {
+ ...childToolStubs,
+ agentPath: stubToolPath(childToolStubs, action.name),
+ };
+ }
await contextStorage.run(new ContextContainer({ localDevRequest: context.localDevRequest }), () =>
- childRuntime.createSession(runInput),
+ childRuntime.createSession({
+ ...runInput,
+ toolStubs: childToolStubs,
+ }),
);
const owner = await waitForCommandHookOwner(sessionInboxHookToken(childContinuationToken));
return { kind: "local", name: action.name, nodeId: action.nodeId, sessionId: owner.runId };
diff --git a/packages/eve/src/execution/runtime-context.ts b/packages/eve/src/execution/runtime-context.ts
index a3a86d0ba..bdf471f50 100644
--- a/packages/eve/src/execution/runtime-context.ts
+++ b/packages/eve/src/execution/runtime-context.ts
@@ -3,6 +3,7 @@ import { ContextContainer, contextStorage } from "#context/container.js";
import { setChannelContext } from "#execution/channel-context.js";
import {
AuthKey,
+ ToolStubsKey,
CapabilitiesKey,
ChannelInstrumentationKey,
ChannelDeliveryKey,
@@ -72,6 +73,7 @@ export function buildRunContext(input: {
run.schedule?.occurrenceId ?? contextStorage.getStore()?.get(OccurrenceIdKey);
if (occurrenceId !== undefined) ctx.set(OccurrenceIdKey, occurrenceId);
ctx.set(AuthKey, auth);
+ if (run.toolStubs !== undefined) ctx.set(ToolStubsKey, run.toolStubs);
if (run.initiatorAuth !== undefined || run.input.message !== undefined) {
ctx.set(InitiatorAuthKey, run.initiatorAuth ?? auth);
}
diff --git a/packages/eve/src/execution/session-inbox/inbox.ts b/packages/eve/src/execution/session-inbox/inbox.ts
index 78233f5e7..5d9042f80 100644
--- a/packages/eve/src/execution/session-inbox/inbox.ts
+++ b/packages/eve/src/execution/session-inbox/inbox.ts
@@ -13,7 +13,14 @@ export interface AuthorizationCallbackPayload {
readonly payloads: DeliverPayload[];
}
+/** The runtime sends this command; channel request bodies cannot supply it. */
+export interface SessionFailurePayload {
+ readonly kind: "session-failure";
+ readonly error: string;
+}
+
export type SessionInboxPayload =
+ | SessionFailurePayload
| HookPayload
| SessionCommand
| WorkflowToolRunMessage
@@ -66,7 +73,12 @@ export interface SessionInboxHandle extends SessionInbox {
}
export function isInterrupt(value: SessionInboxPayload): boolean {
- return value.kind === "cancel" || value.kind === "reset" || value.kind === "session-timeout";
+ return (
+ value.kind === "cancel" ||
+ value.kind === "reset" ||
+ value.kind === "session-timeout" ||
+ value.kind === "session-failure"
+ );
}
/**
diff --git a/packages/eve/src/execution/session-inbox/resume.ts b/packages/eve/src/execution/session-inbox/resume.ts
index f84fe3bb1..f06778f96 100644
--- a/packages/eve/src/execution/session-inbox/resume.ts
+++ b/packages/eve/src/execution/session-inbox/resume.ts
@@ -1,3 +1,4 @@
+import type { SessionFailurePayload } from "#execution/session-inbox/inbox.js";
import { HookNotFoundError } from "#compiled/@workflow/errors/index.js";
import type {
DeliverHookPayload,
@@ -38,7 +39,7 @@ export interface ResumedSessionInboxHook {
*/
export async function resumeSessionInbox(
address: string | SessionInboxAddress,
- command: DeliverHookPayload | SessionCommand | SessionTimeoutHookPayload,
+ command: DeliverHookPayload | SessionCommand | SessionTimeoutHookPayload | SessionFailurePayload,
): Promise<ResumedSessionInboxHook> {
const token = logicalToken(address);
const deadline = Date.now() + HANDOFF_RETRY_WINDOW_MS;
@@ -49,6 +50,7 @@ export async function resumeSessionInbox(
} catch (error) {
if (!HookNotFoundError.is(error)) throw error;
if (await isHandoffInProgress(token, deadline)) continue;
+ if (command.kind === "session-failure") throw error;
return await resumeLegacyInbox(token, command).catch(rethrowUnsupportedAsNotFound);
}
let identity: Promise<string> | undefined;
diff --git a/packages/eve/src/execution/session/active-turn.ts b/packages/eve/src/execution/session/active-turn.ts
index 81bf79949..129ce8d31 100644
--- a/packages/eve/src/execution/session/active-turn.ts
+++ b/packages/eve/src/execution/session/active-turn.ts
@@ -231,7 +231,7 @@ export class ActiveTurn {
}
private cancelsThisTurn(payload: SessionInboxPayload): boolean {
- if (payload.kind === "reset") return true;
+ if (payload.kind === "reset" || payload.kind === "session-failure") return true;
if (payload.kind !== "cancel") return false;
return payload.turnId === undefined || payload.turnId === this.expectedTurnId;
}
diff --git a/packages/eve/src/execution/session/admission.ts b/packages/eve/src/execution/session/admission.ts
index ae9988019..85e3502df 100644
--- a/packages/eve/src/execution/session/admission.ts
+++ b/packages/eve/src/execution/session/admission.ts
@@ -29,6 +29,7 @@ export async function admitSessionInboxPayload(
readonly queue: SessionInputQueue;
},
): Promise<SessionAdmission> {
+ if (value.kind === "session-failure") throw new Error(value.error);
if (value.kind === "runtime-action-result")
return { kind: "runtime-action-result", payload: value };
if (isWorkflowMessage(value)) return { kind: "workflow", message: value };
diff --git a/packages/eve/src/execution/session/approval-caller.integration.test.ts b/packages/eve/src/execution/session/approval-caller.integration.test.ts
index 1bd406db0..6dbda7d6b 100644
--- a/packages/eve/src/execution/session/approval-caller.integration.test.ts
+++ b/packages/eve/src/execution/session/approval-caller.integration.test.ts
@@ -25,6 +25,7 @@ import {
defineWorkflowTool,
} from "#tools/workflow-definition.js";
import { reportCallerWorkflow } from "#internal/testing/workflow-tool-fixtures.js";
+import { STUB_CONTEXT_KEY, type ToolStub } from "#tool-stubs/types.js";
// Scripted model: calls each tool the first message names, in the order it
// names them, once each, then replies. "in parallel" calls them all in one step.
@@ -129,6 +130,7 @@ async function withChainRun(
options: {
readonly firstApproval?: Approval;
readonly message?: string;
+ readonly stubs?: readonly ToolStub[];
readonly model?: MockLanguageModelV4;
readonly modules?: NonNullable<Parameters<typeof createTestRuntime>[0]>["modules"];
} = {},
@@ -183,6 +185,9 @@ async function withChainRun(
serializedContext: {
...buildSerializedContext({ auth: ALICE, channelKind: "http", continuationToken }),
"eve.capabilities": { requestInput: true },
+ ...(options.stubs === undefined
+ ? {}
+ : { [STUB_CONTEXT_KEY]: { token: `${name}-stubs`, rules: options.stubs } }),
},
},
]);
@@ -240,28 +245,44 @@ function turnBoundaries(stage: Stage) {
}
describe("approval caller", () => {
- it("runs only the call another person approves as that person", async () => {
- const { secondRequesters, seen, stages } = await withChainRun(
- "approval-caller",
- async (run) => {
- await run.approve(BOB);
- await run.approve(ALICE);
- },
- );
+ it.each([false, true])(
+ "runs only the call another person approves as that person (unmatched stub: %s)",
+ async (withStub) => {
+ const { secondRequesters, seen, stages } = await withChainRun(
+ `approval-caller-${withStub}`,
+ async (run) => {
+ await run.approve(BOB);
+ await run.approve(ALICE);
+ },
+ {
+ stubs: withStub
+ ? [
+ {
+ id: "other-deployment",
+ tool: "deploy_change",
+ match: { environment: { const: "other" } },
+ outcome: { response: { ran: "stubbed" } },
+ },
+ ]
+ : undefined,
+ },
+ );
- // Bob's approval resumes Alice's turn; it does not start his own.
- expect(stages.map(turnBoundaries)).toEqual([
- ["turn.started turn_0"],
- [],
- ["turn.completed turn_0"],
- ]);
- expect(seen).toEqual([
- { tool: "deploy_change", caller: "bob" },
- { tool: "read_notes", caller: "alice" },
- { tool: "publish_change", caller: "alice" },
- ]);
- expect(secondRequesters).toEqual(["alice"]);
- }, 60_000);
+ // Bob's approval resumes Alice's turn; it does not start his own.
+ expect(stages.map(turnBoundaries)).toEqual([
+ ["turn.started turn_0"],
+ [],
+ ["turn.completed turn_0"],
+ ]);
+ expect(seen).toEqual([
+ { tool: "deploy_change", caller: "bob" },
+ { tool: "read_notes", caller: "alice" },
+ { tool: "publish_change", caller: "alice" },
+ ]);
+ expect(secondRequesters).toEqual(["alice"]);
+ },
+ 60_000,
+ );
it("lets the requester steer the turn after someone else approves a call", async () => {
const { seen, stages } = await withChainRun("approval-caller-steer", async (run) => {
diff --git a/packages/eve/src/execution/session/checkpoint-migrations.ts b/packages/eve/src/execution/session/checkpoint-migrations.ts
index f1282fefc..01a203ed7 100644
--- a/packages/eve/src/execution/session/checkpoint-migrations.ts
+++ b/packages/eve/src/execution/session/checkpoint-migrations.ts
@@ -91,6 +91,9 @@ const CHECKPOINT_UPGRADES: Readonly<
},
// Lifecycle moved from emission/batch registries to the projection and TurnState.
11: upgradeIdleLifecycle,
+ // Version 13 prevents older deployments from ignoring stubs and running real tools.
+ // Existing checkpoints need no data changes.
+ 12: (checkpoint) => checkpoint,
};
/**
diff --git a/packages/eve/src/execution/session/handoff.ts b/packages/eve/src/execution/session/handoff.ts
index 8ef45efc2..c677ea01e 100644
--- a/packages/eve/src/execution/session/handoff.ts
+++ b/packages/eve/src/execution/session/handoff.ts
@@ -26,7 +26,7 @@ import { isObject } from "#shared/guards.js";
* A newer successor must always accept older checkpoints: every bump adds the
* upgrade from the previous version in `checkpoint-migrations.ts`.
*/
-export const SESSION_CHECKPOINT_VERSION = 12;
+export const SESSION_CHECKPOINT_VERSION = 13;
/** Everything a successor needs to continue an idle session. Hooks are derived from the state. */
export interface SessionCheckpoint {
diff --git a/packages/eve/src/execution/session/program.ts b/packages/eve/src/execution/session/program.ts
index 72444bb05..50bbe1b53 100644
--- a/packages/eve/src/execution/session/program.ts
+++ b/packages/eve/src/execution/session/program.ts
@@ -1,3 +1,5 @@
+import { STUB_CONTEXT_KEY, type StubScope } from "#tool-stubs/types.js";
+import { withStubPlayback } from "#execution/tool-stubs/playback.js";
import type { DeliverHookPayload, SessionCapabilities, TurnCaller } from "#channel/types.js";
import type { AgentWorkflowRetentionDefinition } from "#shared/agent-definition.js";
import {
@@ -96,6 +98,10 @@ export async function runPreparedSession(
boot: SessionBoot,
inbox: SessionInboxHandle,
): Promise<WorkflowEntryResult> {
+ const scope = boot.serializedContext[STUB_CONTEXT_KEY] as StubScope | undefined;
+ if (scope !== undefined && scope.rootSessionId === undefined) {
+ boot.serializedContext[STUB_CONTEXT_KEY] = { ...scope, rootSessionId: boot.sessionId };
+ }
const cursor = new SessionStateCursor({
history: boot.history,
inbox,
@@ -118,22 +124,25 @@ export async function runPreparedSession(
let result: WorkflowEntryResult = { output: "", isError: true };
let loop: SessionLoopOutcome | undefined;
try {
- try {
- loop = await runSessionLoop(boot, { cursor, handoff, inbox, progress });
- } finally {
- await inbox.dispose();
- }
- if (loop.kind === "transferred") {
- if (boot.anchor.kind !== "self") return { output: "" };
- result = await handoff.awaitAnchoredResult();
+ result = await withStubPlayback(scope, boot.sessionId, async () => {
+ try {
+ loop = await runSessionLoop(boot, { cursor, handoff, inbox, progress });
+ } finally {
+ await inbox.dispose();
+ }
+ if (loop.kind === "transferred") {
+ if (boot.anchor.kind !== "self") return { output: "" };
+ result = await handoff.awaitAnchoredResult();
+ return result;
+ }
+ result = await finalizeSession(loop.outcome, {
+ caller: progress.caller,
+ cursor,
+ sessionWritable: boot.sessionWritable,
+ });
+ progress.terminalEmitted = true;
return result;
- }
- result = await finalizeSession(loop.outcome, {
- caller: progress.caller,
- cursor,
- sessionWritable: boot.sessionWritable,
});
- progress.terminalEmitted = true;
return result;
} catch (error) {
if (!progress.terminalEmitted) {
@@ -144,6 +153,8 @@ export async function runPreparedSession(
}
throw createSafeOuterWorkflowError();
} finally {
+ // Also dispose if playback setup fails before the session loop starts. Disposal is idempotent.
+ await inbox.dispose();
await reportResultToAnchor(boot, result, handoff, loop);
}
}
diff --git a/packages/eve/src/execution/tasks/model-step.ts b/packages/eve/src/execution/tasks/model-step.ts
index 8b620ab79..c40bf551e 100644
--- a/packages/eve/src/execution/tasks/model-step.ts
+++ b/packages/eve/src/execution/tasks/model-step.ts
@@ -1,3 +1,4 @@
+import { observeToolOutput } from "#tool-stubs/execute.js";
import type { ModelMessage, SystemModelMessage } from "ai";
import { TASK_CANCEL_TOOL_NAME, TASK_WAIT_TOOL_NAME } from "#protocol/task-tools.js";
@@ -193,20 +194,26 @@ async function toResultBlock(
): Promise<TaskResultBlock> {
const base = { taskId: record.id, tool: record.name };
if (result.status === "failed") return { ...base, body: result.error, status: "failed" };
- return { ...base, body: await projectOutput(result.output, definition), status: "completed" };
+ return {
+ ...base,
+ body: await projectOutput(record.name, result, definition),
+ status: "completed",
+ };
}
async function projectOutput(
- output: unknown,
+ tool: string,
+ result: Extract<TaskResult, { status: "completed" }>,
definition: HarnessToolDefinition | undefined,
): Promise<string> {
- if (definition?.toModelOutput === undefined) return renderModelOutputText(output);
- try {
- return renderModelOutputText(await definition.toModelOutput(output));
- } catch {
+ if (definition?.toModelOutput === undefined) return renderModelOutputText(result.output);
+ return await observeToolOutput(
+ tool,
+ result.calls ?? [],
+ async () => renderModelOutputText(await definition.toModelOutput!(result.output)),
// A projection that throws must not wedge every later model step.
- return renderModelOutputText(output);
- }
+ () => renderModelOutputText(result.output),
+ );
}
/**
diff --git a/packages/eve/src/execution/tasks/table.ts b/packages/eve/src/execution/tasks/table.ts
index e7acdb5ba..0e4b512f5 100644
--- a/packages/eve/src/execution/tasks/table.ts
+++ b/packages/eve/src/execution/tasks/table.ts
@@ -68,7 +68,13 @@ export type TaskOutcome =
* An outcome the model receives, one per reply or run end, however many calls
* it settled. Cancelled work never reports back.
*/
-export type TaskResult = Exclude<TaskOutcome, { readonly status: "cancelled" }>;
+export type TaskResult = Exclude<TaskOutcome, { readonly status: "cancelled" }> & {
+ /**
+ * Keep original call and turn IDs so output-processing failures in a later turn
+ * can be recorded against the stubbed call.
+ */
+ readonly calls?: readonly TaskCall[];
+};
export interface TaskRecord {
readonly id: string;
@@ -306,7 +312,10 @@ export function settleTaskCalls(
const next = updateTask(table, input.taskId, (current) => ({
...current,
calls: current.calls.filter((call) => !settling.has(call.callId)),
- results: outcome.status === "cancelled" ? current.results : [...current.results, outcome],
+ results:
+ outcome.status === "cancelled"
+ ? current.results
+ : [...current.results, { ...outcome, calls: settled }],
}));
return { settled, table: next };
}
@@ -495,6 +504,12 @@ function isTaskRunCommand(value: unknown): value is TaskRunCommand {
function isTaskResult(value: unknown): value is TaskResult {
if (!isObject(value)) return false;
+ // Older saved results do not include call IDs.
+ if (
+ value.calls !== undefined &&
+ (!Array.isArray(value.calls) || !Array.from(value.calls as unknown[]).every(isTaskCall))
+ )
+ return false;
if (value.status === "completed") return "output" in value;
return value.status === "failed" && typeof value.error === "string";
}
diff --git a/packages/eve/src/execution/tool-stubs/connection.integration.test.ts b/packages/eve/src/execution/tool-stubs/connection.integration.test.ts
new file mode 100644
index 000000000..f40160574
--- /dev/null
+++ b/packages/eve/src/execution/tool-stubs/connection.integration.test.ts
@@ -0,0 +1,145 @@
+import { toolStubProvider } from "#context/providers/tool-stubs.js";
+import { describe, expect, it, vi } from "vitest";
+import { ContextContainer, contextStorage } from "#context/container.js";
+import { SessionKey, ToolStubsKey } from "#context/keys.js";
+import { ConnectionRegistryKey } from "#context/providers/connection-key.js";
+import { createTestRuntime } from "#internal/testing/app-harness.js";
+import { buildSerializedContext } from "#internal/testing/entry-test-helpers.js";
+import { waitForHook } from "#internal/testing/workflow-test-helpers.js";
+import { getWorld, start } from "#internal/workflow/runtime.js";
+import { workflowEntry } from "#execution/session/entry.js";
+import { resolveConnectionTools } from "#execution/tools/connection-tools.js";
+import { recordToolStubFailure } from "#tool-stubs/execute.js";
+import { readStubFailure } from "#execution/tool-stubs/steps.js";
+import { STUB_CONTEXT_KEY, STUB_FAILURE_NAMESPACE, type ToolStub } from "#tool-stubs/types.js";
+import type { ConnectionClient } from "#shared/connection-types.js";
+import type { ToolContext } from "#tools/definition.js";
+
+describe("connection operation stubs", () => {
+ it("validates input before replacing a child-scoped connection operation and leaves unmatched calls live", async () => {
+ let liveCalls = 0;
+ const runtime = await createTestRuntime();
+ await runtime.run(async () => {
+ const rules: readonly ToolStub[] = [
+ {
+ id: "open",
+ tool: "researcher/linear__list_issues",
+ match: { status: { const: "open" } },
+ outcome: { response: { issues: ["milk"] } },
+ },
+ ];
+ const token = "connection-stub-playback";
+ const run = await start(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: {},
+ serializedContext: {
+ ...buildSerializedContext({ channelKind: "http" }),
+ [STUB_CONTEXT_KEY]: { token, rules },
+ },
+ },
+ ]);
+ try {
+ await waitForHook({ runId: run.runId }, { token });
+ const client: ConnectionClient = {
+ close: async () => {},
+ connect: async () => {},
+ getToolMetadata: async () => [
+ {
+ name: "list_issues",
+ description: "List issues.",
+ inputSchema: {
+ type: "object",
+ properties: { status: { type: "string" } },
+ required: ["status"],
+ },
+ },
+ ],
+ executeTool: async () => {
+ liveCalls++;
+ return { content: [{ type: "text", text: "live" }] };
+ },
+ };
+ const connection = {
+ sourceId: "test:linear",
+ sourceKind: "module" as const,
+ logicalPath: "connections/linear.ts",
+ connectionName: "linear",
+ description: "Issues",
+ protocol: "mcp" as const,
+ url: "https://linear.example/mcp",
+ };
+ const context = new ContextContainer();
+ context.set(SessionKey, {
+ sessionId: run.runId,
+ auth: { current: null, initiator: null },
+ turn: { id: "turn-0", sequence: 0 },
+ });
+ context.set(ToolStubsKey, {
+ token,
+ rules,
+ rootSessionId: run.runId,
+ agentPath: "researcher",
+ });
+ context.set(ConnectionRegistryKey, {
+ dispose: async () => {},
+ getClient: () => client,
+ getConnectionApproval: () => undefined,
+ getConnectionNames: () => ["linear"],
+ getConnections: () => [connection],
+ });
+ context.setVirtualContext(toolStubProvider.key, toolStubProvider.create(context)!.value);
+ await contextStorage.run(context, async () => {
+ const execute = resolveConnectionTools()!.connection_execute!.execute!;
+ const call = (callId: string, input: unknown) =>
+ execute({ connection: "linear", tool: "list_issues", input }, {
+ callId,
+ } as ToolContext);
+ await expect(call("invalid", { status: 42 })).rejects.toThrow(/Invalid input/);
+ expect(await call("stubbed", { status: "open", limit: 10 })).toEqual({
+ issues: ["milk"],
+ });
+ expect(liveCalls).toBe(0);
+ await call("live", { status: "closed" });
+ expect(liveCalls).toBe(1);
+ // connection_execute converts the operation's output before the model receives it.
+ await recordToolStubFailure("connection_execute", "live");
+ expect(await readStubFailure(run.runId)).toBeUndefined();
+ const world = await getWorld();
+ const append = world.streams.write.bind(world.streams);
+ const failureSuffix = Buffer.from(STUB_FAILURE_NAMESPACE).toString("base64url");
+ const entered = Promise.withResolvers<void>();
+ const release = Promise.withResolvers<void>();
+ const write = vi.spyOn(world.streams, "write").mockImplementation(async (...args) => {
+ if (args[1].endsWith(`_${failureSuffix}`)) {
+ entered.resolve();
+ await release.promise;
+ }
+ return await append(...args);
+ });
+ let acknowledged = false;
+ const recording = recordToolStubFailure("connection_execute", "stubbed").then(() => {
+ acknowledged = true;
+ });
+ try {
+ await entered.promise;
+ expect(acknowledged).toBe(false);
+ expect(await readStubFailure(run.runId)).toBeUndefined();
+ release.resolve();
+ await recording;
+ expect(await readStubFailure(run.runId)).toBe(
+ 'Stubbed tool "connection_execute" failed during output processing.',
+ );
+ } finally {
+ release.resolve();
+ await recording;
+ write.mockRestore();
+ }
+ });
+ } finally {
+ await run.cancel();
+ }
+ });
+ });
+});
diff --git a/packages/eve/src/execution/tool-stubs/execution.integration.test.ts b/packages/eve/src/execution/tool-stubs/execution.integration.test.ts
new file mode 100644
index 000000000..b32da0505
--- /dev/null
+++ b/packages/eve/src/execution/tool-stubs/execution.integration.test.ts
@@ -0,0 +1,516 @@
+import { FatalError } from "#compiled/@workflow/errors/index.js";
+import { STUB_FAILURE_NAMESPACE, stubResponseNamespace } from "#tool-stubs/types.js";
+import { describe, expect, it, vi } from "vitest";
+import { getWorld, start } from "#internal/workflow/runtime.js";
+import { createTestRuntime } from "#internal/testing/app-harness.js";
+import { buildSerializedContext, handoffFollowUp } from "#internal/testing/entry-test-helpers.js";
+import { captureTurnEvents, filterEventsByType } from "#internal/testing/events.js";
+import { workflowEntry } from "#execution/session/entry.js";
+import { dispatchWorkflowSessionCommand } from "#execution/workflow-runtime.js";
+import { readStubFailure } from "#execution/tool-stubs/steps.js";
+import { STUB_CONTEXT_KEY } from "#tool-stubs/types.js";
+import { defineWorkflowTool } from "#tools/workflow-definition.js";
+import { defineTool } from "#tools/definition.js";
+import { createWorkflowToolRuntime } from "#internal/testing/workflow-tool-run-harness.js";
+import {
+ failingDeployWorkflow,
+ failingServeWorkflow,
+} from "#internal/testing/workflow-tool-fixtures.js";
+import { always } from "#tools/approval/policies.js";
+
+describe("tool replacement through the session runtime", () => {
+ it.each(["ordinary", "execute", "task", "serve"] as const)(
+ "records %s output conversion failures even when the agent recovers",
+ async (entryPoint) => {
+ let liveCalls = 0;
+ let projections = 0;
+ const definition = {
+ description: "Deploy a service.",
+ inputSchema: {
+ type: "object" as const,
+ properties: { service: { type: "string" as const } },
+ required: ["service"],
+ },
+ toModelOutput: () => {
+ projections++;
+ throw new Error("Invalid deployment result.");
+ },
+ };
+ const runtime = await createTestRuntime({
+ modules: [
+ {
+ logicalPath: "tools/deploy_service.ts",
+ loadNamespace: async () => ({
+ default:
+ entryPoint === "ordinary"
+ ? defineTool({
+ ...definition,
+ execute: () => {
+ liveCalls++;
+ return "live";
+ },
+ })
+ : entryPoint === "execute"
+ ? defineWorkflowTool({ ...definition, execute: failingDeployWorkflow })
+ : entryPoint === "task"
+ ? defineWorkflowTool({ ...definition, task: failingDeployWorkflow })
+ : defineWorkflowTool({ ...definition, serve: failingServeWorkflow }),
+ }),
+ },
+ ],
+ });
+ await runtime.run(async () => {
+ const run = await start(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: { message: 'Run deploy_service with service "api"' },
+ serializedContext: {
+ ...buildSerializedContext({ channelKind: "http" }),
+ [STUB_CONTEXT_KEY]: {
+ token: "failed-output-playback",
+ rules: [{ id: "deploy", tool: "deploy_service", outcome: { response: "stubbed" } }],
+ },
+ },
+ },
+ ]);
+ const stream = captureTurnEvents(run);
+ try {
+ const events = await stream.nextTurn();
+ expect(projections).toBeGreaterThan(0);
+ expect(liveCalls).toBe(0);
+ if (entryPoint === "task" || entryPoint === "serve") {
+ expect(filterEventsByType(events, "turn.failed")).toEqual([]);
+ expect(
+ filterEventsByType(events, "task.settled").map((event) => event.data),
+ ).toContainEqual(expect.objectContaining({ status: "completed", output: "stubbed" }));
+ }
+ expect(await readStubFailure(run.runId)).toBe(
+ 'Stubbed tool "deploy_service" failed during output processing.',
+ );
+ } finally {
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ },
+ );
+
+ it.each([false, true])(
+ "ends the active session when playback fails (handoff: %s)",
+ async (handoff) => {
+ const runtime = await createTestRuntime({
+ modules: [
+ {
+ logicalPath: "tools/deploy_service.ts",
+ loadNamespace: async () => ({
+ default: defineTool({
+ description: "Deploy a service.",
+ inputSchema: {
+ type: "object",
+ properties: { service: { type: "string" } },
+ required: ["service"],
+ },
+ execute: () => {
+ throw new Error("Live execution must not run.");
+ },
+ }),
+ }),
+ },
+ ],
+ });
+ await runtime.run(async () => {
+ const run = await start(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: { message: 'Run deploy_service with service "api"' },
+ serializedContext: {
+ ...buildSerializedContext({ channelKind: "http" }),
+ [STUB_CONTEXT_KEY]: {
+ token: "failed-playback",
+ rules: [{ id: "deploy", tool: "deploy_service", outcome: { response: "stubbed" } }],
+ },
+ },
+ },
+ ]);
+ const stream = captureTurnEvents(run);
+ const world = await getWorld();
+ const append = world.streams.write.bind(world.streams);
+ let write: ReturnType<typeof vi.spyOn> | undefined;
+ try {
+ await stream.nextTurn();
+ write = vi.spyOn(world.streams, "write").mockImplementation(async (...args) => {
+ const namespace = Buffer.from(args[1].split("_").at(-1)!, "base64url").toString();
+ if (
+ namespace !== STUB_FAILURE_NAMESPACE &&
+ namespace.startsWith(stubResponseNamespace(""))
+ )
+ throw new FatalError("Playback transport failed.");
+ return await append(...args);
+ });
+ await dispatchWorkflowSessionCommand({
+ sessionId: run.runId,
+ command: handoff
+ ? handoffFollowUp(
+ "dpl_successor",
+ 'Run deploy_service with service "api"',
+ "failed-stub-handoff",
+ )
+ : { kind: "send", payload: { message: 'Run deploy_service with service "api"' } },
+ });
+ const events = await stream.nextTurn();
+ expect(filterEventsByType(events, "session.failed")).toHaveLength(1);
+ expect(filterEventsByType(events, "session.completed")).toHaveLength(0);
+ expect(await readStubFailure(run.runId)).toBe("Tool stub playback failed.");
+ expect(
+ await dispatchWorkflowSessionCommand({
+ sessionId: run.runId,
+ command: { kind: "send", payload: { message: "Hello" } },
+ }),
+ ).toMatchObject({ status: "session_not_active" });
+ if (handoff) {
+ await expect(run.returnValue).resolves.toEqual({ output: "" });
+ } else {
+ await expect(run.returnValue).rejects.toThrow("Agent workflow failed.");
+ }
+ } finally {
+ write?.mockRestore();
+ stream.dispose();
+ }
+ });
+ },
+ );
+
+ it("isolates same-named root and child tools while sharing root playback", async () => {
+ const runtime = await createTestRuntime({
+ modules: [
+ {
+ logicalPath: "tools/deploy_service.ts",
+ loadNamespace: async () => ({
+ default: defineTool({
+ description: "Deploy a service.",
+ inputSchema: {
+ type: "object",
+ properties: { service: { type: "string" } },
+ required: ["service"],
+ },
+ execute: () => {
+ throw new Error("Live execution must not run.");
+ },
+ }),
+ }),
+ },
+ ],
+ });
+ await runtime.run(async () => {
+ const run = await start(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: { message: 'Run deploy_service with service "api"' },
+ serializedContext: {
+ ...buildSerializedContext({ channelKind: "http" }),
+ [STUB_CONTEXT_KEY]: {
+ token: "inherited-playback",
+ rules: [
+ {
+ id: "deploy",
+ tool: "deploy_service",
+ outcome: { response: { state: "root" } },
+ },
+ {
+ id: "child-deploy",
+ tool: "agent/deploy_service",
+ outcome: { response: { state: "child" } },
+ },
+ ],
+ },
+ },
+ },
+ ]);
+ const stream = captureTurnEvents(run);
+ try {
+ const rootEvents = await stream.nextTurn();
+ expect(
+ filterEventsByType(rootEvents, "action.result").map((event) => event.data.result),
+ ).toContainEqual(expect.objectContaining({ output: { state: "root" } }));
+ await dispatchWorkflowSessionCommand({
+ sessionId: run.runId,
+ command: {
+ kind: "send",
+ payload: { message: 'Delegate to a subagent: Run deploy_service with service "api"' },
+ },
+ });
+ const events = await stream.nextTurn();
+ const children = filterEventsByType(events, "agent.started");
+ expect(children).toHaveLength(1);
+ const settlements = filterEventsByType(events, "task.settled");
+ expect(
+ settlements.some((event) => JSON.stringify(event.data.output).includes("child")),
+ ).toBe(true);
+ } finally {
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ });
+
+ it("replaces the whole agent tool without starting a remote or local agent", async () => {
+ const runtime = await createTestRuntime();
+ await runtime.run(async () => {
+ const run = await start(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: { message: "Delegate to a subagent: Say hello." },
+ serializedContext: {
+ ...buildSerializedContext({ channelKind: "http" }),
+ [STUB_CONTEXT_KEY]: {
+ token: "whole-agent-playback",
+ rules: [
+ { id: "agent", tool: "agent", outcome: { response: "Hello from the stub." } },
+ ],
+ },
+ },
+ },
+ ]);
+ const stream = captureTurnEvents(run);
+ try {
+ const events = await stream.nextTurn();
+ expect(filterEventsByType(events, "agent.started")).toEqual([]);
+ expect(
+ filterEventsByType(events, "task.settled").map((event) => event.data),
+ ).toContainEqual(
+ expect.objectContaining({ status: "completed", output: "Hello from the stub." }),
+ );
+ } finally {
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ });
+
+ it("keeps approval gates and does not consume a response for a denied call", async () => {
+ const runtime = await createTestRuntime({
+ modules: [
+ {
+ logicalPath: "tools/deploy_service.ts",
+ loadNamespace: async () => ({
+ default: defineTool({
+ description: "Deploy a service.",
+ inputSchema: {
+ type: "object",
+ properties: { service: { type: "string" } },
+ required: ["service"],
+ },
+ approval: always(),
+ execute: () => {
+ throw new Error("Live execution must not run.");
+ },
+ }),
+ }),
+ },
+ ],
+ });
+ await runtime.run(async () => {
+ const run = await start(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: { message: 'Run deploy_service with service "api"' },
+ serializedContext: {
+ ...buildSerializedContext({ channelKind: "http" }),
+ "eve.capabilities": { requestInput: true },
+ [STUB_CONTEXT_KEY]: {
+ token: "approval-playback",
+ rules: [
+ {
+ id: "deploy",
+ tool: "deploy_service",
+ outcomes: [{ response: { first: true } }, { response: { first: false } }],
+ },
+ ],
+ },
+ },
+ },
+ ]);
+ const stream = captureTurnEvents(run);
+ try {
+ const pending = await stream.nextTurn();
+ const request = filterEventsByType(pending, "input.requested")[0]!.data.requests[0]!;
+ expect(filterEventsByType(pending, "action.result")).toEqual([]);
+ await dispatchWorkflowSessionCommand({
+ sessionId: run.runId,
+ command: {
+ kind: "send",
+ payload: { inputResponses: [{ requestId: request.requestId, optionId: "deny" }] },
+ },
+ });
+ await stream.nextTurn();
+ await dispatchWorkflowSessionCommand({
+ sessionId: run.runId,
+ command: { kind: "send", payload: { message: 'Run deploy_service with service "api"' } },
+ });
+ const retry = await stream.nextTurn();
+ const next = filterEventsByType(retry, "input.requested")[0]!.data.requests[0]!;
+ await dispatchWorkflowSessionCommand({
+ sessionId: run.runId,
+ command: {
+ kind: "send",
+ payload: { inputResponses: [{ requestId: next.requestId, optionId: "approve" }] },
+ },
+ });
+ const approved = await stream.nextTurn();
+ expect(
+ filterEventsByType(approved, "action.result").map((event) => event.data.result),
+ ).toContainEqual(expect.objectContaining({ output: { first: true } }));
+ } finally {
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ });
+ it("keeps sequence progress across a deployment handoff and runs unmatched calls normally", async () => {
+ let liveCalls = 0;
+ const runtime = await createTestRuntime({
+ modules: [
+ {
+ logicalPath: "tools/deploy_service.ts",
+ loadNamespace: async () => ({
+ default: defineTool({
+ description: "Deploy a service.",
+ inputSchema: {
+ type: "object",
+ properties: { service: { type: "string" } },
+ required: ["service"],
+ },
+ execute: async () => {
+ liveCalls++;
+ return { state: "live" };
+ },
+ }),
+ }),
+ },
+ ],
+ });
+ await runtime.run(async () => {
+ const run = await start(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: { message: 'Run deploy_service with service "api"' },
+ serializedContext: {
+ ...buildSerializedContext({ channelKind: "http" }),
+ [STUB_CONTEXT_KEY]: {
+ token: "ordinary-tool-playback",
+ rules: [
+ {
+ id: "api",
+ tool: "deploy_service",
+ match: { service: { const: "api" } },
+ outcomes: [
+ { response: { state: "pending" } },
+ { response: { state: "completed" } },
+ ],
+ },
+ ],
+ },
+ },
+ },
+ ]);
+ const stream = captureTurnEvents(run);
+ try {
+ const first = await stream.nextTurn();
+ expect(
+ filterEventsByType(first, "action.result").map((event) => event.data.result),
+ ).toContainEqual(expect.objectContaining({ output: { state: "pending" } }));
+ await dispatchWorkflowSessionCommand({
+ sessionId: run.runId,
+ command: handoffFollowUp(
+ "dpl_successor",
+ 'Run deploy_service with service "api"',
+ "stub-handoff",
+ ),
+ });
+ const second = await stream.nextTurn();
+ expect(
+ filterEventsByType(second, "action.result").map((event) => event.data.result),
+ ).toContainEqual(expect.objectContaining({ output: { state: "completed" } }));
+ await dispatchWorkflowSessionCommand({
+ sessionId: run.runId,
+ command: { kind: "send", payload: { message: 'Run deploy_service with service "web"' } },
+ });
+ const third = await stream.nextTurn();
+ expect(
+ filterEventsByType(third, "action.result").map((event) => event.data.result),
+ ).toContainEqual(expect.objectContaining({ output: { state: "live" } }));
+ expect(liveCalls).toBe(1);
+ expect(await readStubFailure(run.runId)).toBeUndefined();
+ } finally {
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ });
+
+ it("replaces a workflow body while retaining its normal result events", async () => {
+ const runtime = await createWorkflowToolRuntime({
+ agentName: "stubbed-workflow",
+ execute: failingDeployWorkflow,
+ toolName: "deploy_service",
+ });
+ await runtime.run(async () => {
+ const run = await start(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: { message: 'Run deploy_service with service "api"' },
+ serializedContext: {
+ ...buildSerializedContext({ channelKind: "http" }),
+ [STUB_CONTEXT_KEY]: {
+ token: "workflow-tool-playback",
+ rules: [
+ {
+ id: "deploy",
+ tool: "deploy_service",
+ outcome: { response: { state: "stubbed" } },
+ },
+ {
+ id: "child-deploy",
+ tool: "agent/deploy_service",
+ outcome: { response: { state: "child-workflow" } },
+ },
+ ],
+ },
+ },
+ },
+ ]);
+ const stream = captureTurnEvents(run);
+ try {
+ const events = await stream.nextTurn();
+ expect(
+ filterEventsByType(events, "action.result").map((event) => event.data.result),
+ ).toContainEqual(expect.objectContaining({ output: { state: "stubbed" } }));
+ expect(filterEventsByType(events, "turn.failed")).toEqual([]);
+ await dispatchWorkflowSessionCommand({
+ sessionId: run.runId,
+ command: {
+ kind: "send",
+ payload: { message: 'Delegate to a subagent: Run deploy_service with service "api"' },
+ },
+ });
+ const childEvents = await stream.nextTurn();
+ expect(filterEventsByType(childEvents, "agent.started")).toHaveLength(1);
+ expect(
+ filterEventsByType(childEvents, "task.settled").some((event) =>
+ JSON.stringify(event.data.output).includes("child-workflow"),
+ ),
+ ).toBe(true);
+ } finally {
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ });
+});
diff --git a/packages/eve/src/execution/tool-stubs/playback.integration.test.ts b/packages/eve/src/execution/tool-stubs/playback.integration.test.ts
new file mode 100644
index 000000000..55e10932d
--- /dev/null
+++ b/packages/eve/src/execution/tool-stubs/playback.integration.test.ts
@@ -0,0 +1,189 @@
+import { jsonSchema } from "ai";
+import { ContextContainer, contextStorage } from "#context/container.js";
+import { SessionKey, ToolStubsKey } from "#context/keys.js";
+import { toolStubProvider } from "#context/providers/tool-stubs.js";
+import { appendTaskContext } from "#execution/tasks/model-step.js";
+import { toolCallModelOutput } from "#harness/tool-call-io.js";
+import {
+ createTask,
+ readTaskTable,
+ settleTaskCalls,
+ writeTaskTable,
+} from "#execution/tasks/table.js";
+import { createTestSessionState } from "#internal/testing/session-state.js";
+import { describe, expect, it } from "vitest";
+import { start } from "#internal/workflow/runtime.js";
+import { createTestRuntime } from "#internal/testing/app-harness.js";
+import { buildSerializedContext } from "#internal/testing/entry-test-helpers.js";
+import { waitForHook } from "#internal/testing/workflow-test-helpers.js";
+import { workflowEntry } from "#execution/session/entry.js";
+import {
+ callToolStubStep,
+ readStubFailure,
+ readMatchedStubRules,
+} from "#execution/tool-stubs/steps.js";
+import { STUB_CONTEXT_KEY } from "#tool-stubs/types.js";
+
+describe("durable tool stub playback", () => {
+ it("allocates concurrent calls once and reuses an earlier result after later workflow resumes", async () => {
+ const runtime = await createTestRuntime({ agent: { name: "stub-playback" } });
+ await runtime.run(async () => {
+ const rules = [
+ {
+ id: "list",
+ tool: "list_tasks",
+ outcomes: [{ response: ["milk", "dog"] }, { response: ["dog"] }],
+ },
+ ] as const;
+ const run = await start(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: {},
+ serializedContext: {
+ ...buildSerializedContext({ channelKind: "http" }),
+ [STUB_CONTEXT_KEY]: { token: "test-stub-playback", rules },
+ },
+ },
+ ]);
+ await waitForHook({ runId: run.runId }, { token: "test-stub-playback" });
+ const scope = {
+ token: "test-stub-playback",
+ rootSessionId: run.runId,
+ rules,
+ };
+ expect(await readMatchedStubRules(run.runId)).toEqual([]);
+ const call = { tool: "list_tasks", input: {} };
+ const outputs = await Promise.all([
+ callToolStubStep(scope, { ...call, callId: "root:first" }),
+ callToolStubStep(scope, { ...call, callId: "child:second" }),
+ ]);
+ expect(
+ outputs.map((result) => (result.kind === "stub" ? result.position : null)).sort(),
+ ).toEqual([0, 1]);
+ expect(await callToolStubStep(scope, { ...call, callId: "root:first" })).toEqual(outputs[0]);
+ expect(await callToolStubStep(scope, { ...call, callId: "root:third" })).toEqual({
+ kind: "stub",
+ ruleId: "list",
+ position: 1,
+ outcome: { response: ["dog"] },
+ });
+ expect(await readStubFailure(run.runId)).toBeUndefined();
+ expect(await readMatchedStubRules(run.runId)).toEqual(["list"]);
+ });
+ });
+ it.each([
+ { stubbed: true, reportingFails: false, recover: true },
+ { stubbed: false, reportingFails: false, recover: true },
+ { stubbed: true, reportingFails: true, recover: true },
+ { stubbed: true, reportingFails: true, recover: false },
+ ])(
+ "preserves output errors and recovery without passing an unverifiable eval: %j",
+ async ({ stubbed, reportingFails, recover }) => {
+ const runtime = await createTestRuntime();
+ await runtime.run(async () => {
+ const rules = [
+ {
+ id: "task",
+ tool: "lookup",
+ match: { value: { const: "stubbed" } },
+ outcome: { response: "raw" },
+ },
+ ];
+ const token = "delayed-task-projection";
+ const run = await start(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: {},
+ serializedContext: {
+ ...buildSerializedContext({ channelKind: "http" }),
+ [STUB_CONTEXT_KEY]: { token, rules },
+ },
+ },
+ ]);
+ try {
+ await waitForHook({ runId: run.runId }, { token });
+ const scope = { token, rules, rootSessionId: run.runId };
+ await callToolStubStep(scope, {
+ callId: `${run.runId}:turn_0:lookup`,
+ tool: "lookup",
+ input: { value: stubbed ? "stubbed" : "live" },
+ });
+ const task = createTask(readTaskTable(undefined), {
+ callId: "lookup",
+ kind: "tool",
+ name: "lookup",
+ resumable: false,
+ turnId: "turn_0",
+ });
+ const { table } = settleTaskCalls(task.table, {
+ taskId: task.taskId,
+ callIds: ["lookup"],
+ outcome: { status: "completed", output: "raw" },
+ });
+ // Reload the saved task result before processing its output in the next turn.
+ const session = JSON.parse(
+ JSON.stringify(
+ writeTaskTable(
+ createTestSessionState({ sessionId: run.runId }).snapshot.session,
+ table,
+ ),
+ ),
+ );
+ const context = new ContextContainer();
+ context.set(SessionKey, {
+ sessionId: run.runId,
+ turn: { id: "turn_1", sequence: 1 },
+ auth: { current: null, initiator: null },
+ });
+ await contextStorage.run(context, async () => {
+ context.set(
+ ToolStubsKey,
+ reportingFails ? { ...scope, token: "unavailable-playback" } : scope,
+ );
+ context.setVirtualContext(
+ toolStubProvider.key,
+ toolStubProvider.create(context)!.value,
+ );
+ const outputError = new Error("Invalid output.");
+ const definition = {
+ name: "lookup",
+ description: "Lookup",
+ inputSchema: jsonSchema({ type: "object" }),
+ toModelOutput: () => {
+ throw outputError;
+ },
+ };
+ if (!recover) {
+ await expect(toolCallModelOutput(definition, "raw", "lookup")).rejects.toBe(
+ outputError,
+ );
+ return;
+ }
+ const delivered = await appendTaskContext({
+ session,
+ messages: [],
+ projectHistory: (messages) => messages,
+ tools: new Map([["lookup", definition]]),
+ });
+ expect(JSON.stringify(delivered.messages)).toContain("raw");
+ expect(readTaskTable(delivered.session.state).tasks[0]?.results).toEqual([]);
+ });
+ if (reportingFails) {
+ expect(await run.status).toBe("cancelled");
+ expect(await readStubFailure(run.runId)).toBe(
+ "Tool stub session failed before verification.",
+ );
+ } else {
+ expect(await readStubFailure(run.runId)).toBe(
+ stubbed ? 'Stubbed tool "lookup" failed during output processing.' : undefined,
+ );
+ }
+ } finally {
+ if ((await run.status) !== "cancelled") await run.cancel();
+ }
+ });
+ },
+ );
+});
diff --git a/packages/eve/src/execution/tool-stubs/playback.ts b/packages/eve/src/execution/tool-stubs/playback.ts
new file mode 100644
index 000000000..04466e22f
--- /dev/null
+++ b/packages/eve/src/execution/tool-stubs/playback.ts
@@ -0,0 +1,69 @@
+import { createHook } from "#compiled/@workflow/core/index.js";
+import { claimHookOwnership, disposeHook } from "#execution/hook-ownership.js";
+import { StubPlayback } from "#tool-stubs/playback.js";
+import {
+ failStubSessionStep,
+ publishStubFailureStep,
+ publishStubResultStep,
+} from "#execution/tool-stubs/steps.js";
+import type { StubCall, StubScope } from "#tool-stubs/types.js";
+
+export type StubRequest =
+ | { readonly kind: "call"; readonly call: StubCall }
+ | { readonly kind: "failure"; readonly callId: string; readonly error: string };
+
+/** The original workflow serves stub requests even if a newer deployment takes over the session. */
+export async function withStubPlayback<T>(
+ scope: StubScope | undefined,
+ sessionId: string,
+ run: () => Promise<T>,
+): Promise<T> {
+ if (scope === undefined || scope.rootSessionId !== undefined) return await run();
+ let playback: StubPlayback;
+ try {
+ playback = new StubPlayback(scope.rules);
+ } catch (error) {
+ await publishStubFailureStep(
+ error instanceof Error ? error.message : "Could not create tool stub validators.",
+ );
+ throw error;
+ }
+ const hook = createHook<StubRequest>({ token: scope.token });
+ let failed = false;
+ const serve = async (): Promise<never> => {
+ for await (const request of hook) {
+ const matchedBefore = playback.matchedRuleCount;
+ const result =
+ request.kind === "failure"
+ ? playback.fail(request.callId, request.error)
+ : playback.call(request.call);
+ if (result.kind === "error" && !failed) {
+ // Save the failure before replying so eval verification cannot miss it.
+ await publishStubFailureStep(result.error);
+ failed = true;
+ }
+ await publishStubResultStep({
+ callId: request.kind === "call" ? request.call.callId : `${request.callId}:failure`,
+ result,
+ matchedRuleId:
+ result.kind === "stub" && playback.matchedRuleCount > matchedBefore
+ ? result.ruleId
+ : undefined,
+ });
+ }
+ throw new Error("Tool stub playback ended before its session.");
+ };
+ try {
+ await claimHookOwnership(hook);
+ const running = run();
+ const serving = serve().catch(async () => {
+ await publishStubFailureStep("Tool stub playback failed.");
+ // Stop the active turn even if a newer deployment has taken over the session.
+ await failStubSessionStep(sessionId);
+ return await running;
+ });
+ return await Promise.race([running, serving]);
+ } finally {
+ await disposeHook(hook);
+ }
+}
diff --git a/packages/eve/src/execution/tool-stubs/steps.ts b/packages/eve/src/execution/tool-stubs/steps.ts
new file mode 100644
index 000000000..c1a2ae7ea
--- /dev/null
+++ b/packages/eve/src/execution/tool-stubs/steps.ts
@@ -0,0 +1,139 @@
+import { resumeSessionInbox } from "#execution/session-inbox/resume.js";
+import { getWritable } from "#compiled/@workflow/core/index.js";
+import { getRun, resumeHook } from "#internal/workflow/runtime.js";
+import type { StubRequest } from "#execution/tool-stubs/playback.js";
+import {
+ STUB_FAILURE_NAMESPACE,
+ STUB_MATCHES_NAMESPACE,
+ stubResponseNamespace,
+ type StubCall,
+ type StubResult,
+ type StubScope,
+} from "#tool-stubs/types.js";
+
+export async function publishStubResultStep(input: {
+ readonly callId: string;
+ readonly result: StubResult;
+ readonly matchedRuleId?: string;
+}): Promise<void> {
+ "use step";
+ if (input.matchedRuleId !== undefined) {
+ const writer = getWritable<string>({ namespace: STUB_MATCHES_NAMESPACE }).getWriter();
+ try {
+ await writer.write(input.matchedRuleId);
+ } finally {
+ writer.releaseLock();
+ }
+ }
+ const writer = getWritable<StubResult>({
+ namespace: stubResponseNamespace(input.callId),
+ }).getWriter();
+ try {
+ await writer.write(input.result);
+ } finally {
+ writer.releaseLock();
+ }
+}
+
+export async function publishStubFailureStep(error: string): Promise<void> {
+ "use step";
+ const writer = getWritable<string>({ namespace: STUB_FAILURE_NAMESPACE }).getWriter();
+ try {
+ await writer.write(error);
+ } finally {
+ writer.releaseLock();
+ }
+}
+
+/**
+ * Ordinary tools call this inside their existing step.
+ * Workflow tools use this function as a separate step.
+ */
+export async function callToolStubStep(scope: StubScope, call: StubCall): Promise<StubResult> {
+ "use step";
+ return await requestStub(scope, { kind: "call", call });
+}
+
+export async function reportStubFailureStep(
+ scope: StubScope,
+ callId: string,
+ error: string,
+): Promise<void> {
+ "use step";
+ try {
+ await requestStub(scope, { kind: "failure", callId, error });
+ } catch (cause) {
+ // Without an acknowledgement, verification cannot safely declare this eval successful.
+ // Cancel the stub-owning run so the caller can keep its original output error or fallback.
+ if (scope.rootSessionId === undefined) throw cause;
+ await getRun(scope.rootSessionId).cancel({
+ cancelReason: "Could not record a tool stub output-processing failure.",
+ });
+ }
+}
+
+async function requestStub(scope: StubScope, request: StubRequest): Promise<StubResult> {
+ const root = scope.rootSessionId;
+ if (root === undefined)
+ throw new Error("Tool stub session is missing its durable playback owner.");
+ const callId = request.kind === "call" ? request.call.callId : `${request.callId}:failure`;
+ await resumeHook(scope.token, request);
+ const reader = getRun(root)
+ .getReadable<StubResult>({ namespace: stubResponseNamespace(callId) })
+ .getReader();
+ try {
+ const { done, value } = await reader.read();
+ if (done) throw new Error("Tool stub session ended without a response.");
+ return value;
+ } finally {
+ await reader.cancel();
+ reader.releaseLock();
+ }
+}
+
+export async function readStubFailure(sessionId: string): Promise<string | undefined> {
+ const run = getRun(sessionId);
+ const stream = run.getReadable<string>({ namespace: STUB_FAILURE_NAMESPACE });
+ if ((await stream.getTailIndex()) < 0) {
+ await stream.cancel();
+ const status = await run.status;
+ return status === "failed" || status === "cancelled"
+ ? "Tool stub session failed before verification."
+ : undefined;
+ }
+ const reader = stream.getReader();
+ try {
+ return (await reader.read()).value;
+ } finally {
+ await reader.cancel();
+ reader.releaseLock();
+ }
+}
+
+/** Use the session's inbox so failures reach it after a newer deployment takes over. */
+export async function failStubSessionStep(sessionId: string): Promise<void> {
+ "use step";
+ await resumeSessionInbox(
+ { sessionId },
+ { kind: "session-failure", error: "Tool stub playback failed." },
+ );
+}
+
+/** Read completed matches without waiting for the still-running session to finish. */
+export async function readMatchedStubRules(sessionId: string): Promise<readonly string[]> {
+ const stream = getRun(sessionId).getReadable<string>({ namespace: STUB_MATCHES_NAMESPACE });
+ const tail = await stream.getTailIndex();
+ const reader = stream.getReader();
+ const ids = new Set<string>();
+ try {
+ for (let index = 0; index <= tail; index++) {
+ const { done, value } = await reader.read();
+ if (done) throw new Error("Tool stub match records ended unexpectedly.");
+ ids.add(value);
+ }
+ return [...ids];
+ } finally {
+ await reader.cancel();
+ reader.releaseLock();
+ }
+}
diff --git a/packages/eve/src/execution/tools/connection-tools.ts b/packages/eve/src/execution/tools/connection-tools.ts
index 041f72cd5..22e5b1b1d 100644
--- a/packages/eve/src/execution/tools/connection-tools.ts
+++ b/packages/eve/src/execution/tools/connection-tools.ts
@@ -1,3 +1,4 @@
+import { connectionToolStub } from "#tool-stubs/execute.js";
/**
* `connection_search` and `connection_execute`: fixed tools that reach every
* connection tool without adding definitions to the model's tool list, so
@@ -368,6 +369,12 @@ async function executeConnectionTool(
const input = await validToolInput(connection, tool, target.input);
const toolName = qualifiedToolName(target);
+ const stub = await connectionToolStub(toolName, input, ctx.callId);
+ if (stub.kind === "error") throw new Error(stub.error);
+ if (stub.kind === "stub") {
+ reportNestedToolAction(ctx.callId, { input, output: stub.outcome.response, toolName });
+ return stub.outcome.response;
+ }
let raw: unknown;
try {
raw = await client.executeTool(tool.name, input, {
diff --git a/packages/eve/src/execution/tools/workflow/body.ts b/packages/eve/src/execution/tools/workflow/body.ts
index db468036e..35fcdc69f 100644
--- a/packages/eve/src/execution/tools/workflow/body.ts
+++ b/packages/eve/src/execution/tools/workflow/body.ts
@@ -1,4 +1,6 @@
+import { findStubTarget, stubCallId } from "#tool-stubs/target.js";
import { getWorkflowMetadata } from "#compiled/@workflow/core/index.js";
+import { callToolStubStep } from "#execution/tool-stubs/steps.js";
import type { SessionContext } from "#context/session-context.js";
import type { AgentSessionContext } from "#execution/agent-sessions/context.js";
@@ -172,6 +174,16 @@ async function executeCallBody(
from: WorkflowToolRunRef,
): Promise<WorkflowToolRunOutcome> {
try {
+ const target = findStubTarget(input.agentContext.toolStubs, input.toolName);
+ if (target !== undefined) {
+ const result = await callToolStubStep(target.scope, {
+ callId: stubCallId(input.session.id, input.session.turn.id, input.callId),
+ tool: target.tool,
+ input: input.input,
+ });
+ if (result.kind === "error") throw new Error(result.error);
+ if (result.kind === "stub") return { output: result.outcome.response, status: "completed" };
+ }
const entryPoint = resolveWorkflowEntryPoint<WorkflowCallEntryPoint>(input);
const result = entryPoint(input.executeInput ?? input.input, ctx);
let output: JsonValue;
diff --git a/packages/eve/src/execution/tools/workflow/serve.ts b/packages/eve/src/execution/tools/workflow/serve.ts
index 1116a82f9..dc95bcfd5 100644
--- a/packages/eve/src/execution/tools/workflow/serve.ts
+++ b/packages/eve/src/execution/tools/workflow/serve.ts
@@ -1,4 +1,7 @@
+import { findStubTarget, stubCallId } from "#tool-stubs/target.js";
+import type { StubScope } from "#tool-stubs/types.js";
import type { SessionContext } from "#context/session-context.js";
+import { callToolStubStep } from "#execution/tool-stubs/steps.js";
import type { AgentSessionContext } from "#execution/agent-sessions/context.js";
import { createAgentSessions } from "#execution/agent-sessions/session.js";
import { createRunUsageTally, type RunUsageTally } from "#execution/agent-sessions/usage.js";
@@ -375,8 +378,12 @@ async function executeServeBody(
): Promise<WorkflowToolRunOutcome> {
let outcome: WorkflowToolRunOutcome;
try {
- const serve = resolveWorkflowEntryPoint<ServeEntryPoint>(input);
- const output = await serve(() => calls.receive(), ctx);
+ const target = findStubTarget(input.agentContext.toolStubs, input.toolName);
+ const receive = () => calls.receive();
+ const output =
+ target === undefined
+ ? await resolveWorkflowEntryPoint<ServeEntryPoint>(input)(receive, ctx)
+ : await serveStub(target.scope, target.tool, receive, ctx);
outcome = { output, status: "completed" };
} catch (error) {
outcome = toFailedOutcome(error, calls.runSignal);
@@ -429,3 +436,28 @@ function createPendingReceive(): PendingReceive {
});
return { promise, reject, resolve };
}
+
+async function serveStub(
+ scope: StubScope,
+ tool: string,
+ receive: WorkflowServeReceive<JsonValue>,
+ ctx: ServeContext,
+): Promise<JsonValue> {
+ // reply() answers every received call. Reply before receiving the next call
+ // so each call consumes its own stub response.
+ while (true) {
+ const call = await receive();
+ if (call.abortSignal.aborted) continue;
+ const result = await callToolStubStep(scope, {
+ callId: stubCallId(ctx.session.id, ctx.session.turn.id, call.callId),
+ input: call.input,
+ tool,
+ persistent: true,
+ });
+ if (result.kind !== "stub")
+ throw new Error(
+ result.kind === "error" ? result.error : "Persistent stub configuration changed.",
+ );
+ ctx.reply(result.outcome.response);
+ }
+}
diff --git a/packages/eve/src/execution/workflow-runtime.test.ts b/packages/eve/src/execution/workflow-runtime.test.ts
index e696eb0d6..cd4f754f6 100644
--- a/packages/eve/src/execution/workflow-runtime.test.ts
+++ b/packages/eve/src/execution/workflow-runtime.test.ts
@@ -143,7 +143,7 @@ describe("session owner starts", () => {
startSessionOwnerStep({
activationToken: "owner-1:handoff",
anchorRunId: "anchor-1",
- checkpoint: {} as never,
+ checkpoint: { serializedContext: {} } as never,
delivery: { kind: "deliver", payloads: [] },
targetDeploymentId: "latest",
}),
diff --git a/packages/eve/src/harness/coordination.ts b/packages/eve/src/harness/coordination.ts
index f3ca4ea1d..de949477a 100644
--- a/packages/eve/src/harness/coordination.ts
+++ b/packages/eve/src/harness/coordination.ts
@@ -1,3 +1,4 @@
+import { observeToolOutput } from "#tool-stubs/execute.js";
import type { ModelMessage } from "ai";
import type {
@@ -273,11 +274,13 @@ async function projectToolResultOutput(
) {
return toToolResultOutput(result);
}
- return normalizeToolModelOutput({
- output: await definition.toModelOutput(result.output),
- toolCallId: result.callId,
- toolName: result.toolName,
- });
+ return await observeToolOutput(result.toolName, [{ callId: result.callId }], async () =>
+ normalizeToolModelOutput({
+ output: await definition.toModelOutput!(result.output),
+ toolCallId: result.callId,
+ toolName: result.toolName,
+ }),
+ );
}
function toToolResultOutput(result: RuntimeActionResult): ToolResultPart["output"] {
diff --git a/packages/eve/src/harness/tool-call-io.ts b/packages/eve/src/harness/tool-call-io.ts
index 939febc45..8e81f4d57 100644
--- a/packages/eve/src/harness/tool-call-io.ts
+++ b/packages/eve/src/harness/tool-call-io.ts
@@ -1,4 +1,5 @@
import { asSchema } from "ai";
+import { observeToolOutput } from "#tool-stubs/execute.js";
import {
authorizationPendingModelText,
@@ -63,20 +64,26 @@ export async function toolCallModelOutput(
output: unknown,
toolCallId: string | undefined,
): Promise<ToolModelOutputValue> {
- if (isAuthorizationPendingModelOutput(output)) {
- return { type: "text", value: authorizationPendingModelText(output.connections) };
- }
- if (definition.toModelOutput !== undefined) {
- return normalizeToolModelOutput({
- output: await definition.toModelOutput(output),
- toolCallId,
- toolName: definition.name,
- });
- }
- if (typeof output === "string") return { type: "text", value: output };
- return normalizeToolModelOutput({
- output: { type: "json", value: output ?? null },
- toolCallId,
- toolName: definition.name,
- });
+ return await observeToolOutput(
+ definition.name,
+ toolCallId === undefined ? [] : [{ callId: toolCallId }],
+ async () => {
+ if (isAuthorizationPendingModelOutput(output)) {
+ return { type: "text", value: authorizationPendingModelText(output.connections) };
+ }
+ if (definition.toModelOutput !== undefined) {
+ return normalizeToolModelOutput({
+ output: await definition.toModelOutput(output),
+ toolCallId,
+ toolName: definition.name,
+ });
+ }
+ if (typeof output === "string") return { type: "text", value: output };
+ return normalizeToolModelOutput({
+ output: { type: "json", value: output ?? null },
+ toolCallId,
+ toolName: definition.name,
+ });
+ },
+ );
}
diff --git a/packages/eve/src/harness/tools.ts b/packages/eve/src/harness/tools.ts
index 479de028c..2e842fe72 100644
--- a/packages/eve/src/harness/tools.ts
+++ b/packages/eve/src/harness/tools.ts
@@ -16,6 +16,7 @@ import { normalizeToolJsonOutput } from "#harness/tool-model-output.js";
import { toolCallModelOutput } from "#harness/tool-call-io.js";
import type { ToolExecuteOptions } from "#tools/definition.js";
import { isAsyncIterable } from "#shared/async-iterable.js";
+import { executeWithToolStub } from "#tool-stubs/execute.js";
import { iterateAsApprover, runAsApprover } from "#harness/hitl/approved-call-callers.js";
type NativeApprovalStatus = Exclude<ApprovalStatus, boolean>;
@@ -122,7 +123,9 @@ export function wrapToolExecute(
return (input, options) => {
let output: unknown;
try {
- output = runAsApprover(options.toolCallId, () => execute(input, options));
+ output = runAsApprover(options.toolCallId, () =>
+ executeWithToolStub(definition.name, input, options, () => execute(input, options)),
+ );
} catch (error) {
return Promise.reject(error);
}
diff --git a/packages/eve/src/internal/testing/workflow-tool-fixtures.ts b/packages/eve/src/internal/testing/workflow-tool-fixtures.ts
index ec8784f80..442db307b 100644
--- a/packages/eve/src/internal/testing/workflow-tool-fixtures.ts
+++ b/packages/eve/src/internal/testing/workflow-tool-fixtures.ts
@@ -263,3 +263,8 @@ export async function reportCallerWorkflow(_input: unknown, ctx: WorkflowToolCon
"use workflow";
return { caller: ctx.session.auth.current?.principalId ?? null };
}
+
+export async function failingServeWorkflow(): Promise<never> {
+ "use workflow";
+ throw new Error("Live serve workflow must not execute.");
+}
diff --git a/packages/eve/src/tool-stubs/execute.ts b/packages/eve/src/tool-stubs/execute.ts
new file mode 100644
index 000000000..c631c871a
--- /dev/null
+++ b/packages/eve/src/tool-stubs/execute.ts
@@ -0,0 +1,107 @@
+import { findStubTarget, stubCallId } from "#tool-stubs/target.js";
+import { contextStorage } from "#context/container.js";
+import { SessionKey, ToolStubsKey } from "#context/keys.js";
+import { ToolStubPlaybackKey, type ToolStubPlayback } from "#context/providers/tool-stubs-key.js";
+import type { StubResult } from "#tool-stubs/types.js";
+import type { ToolExecuteOptions } from "#tools/definition.js";
+import { isAsyncIterable } from "#shared/async-iterable.js";
+
+/** Tool inputs and approvals have already been checked before a stub can replace execution. */
+export function executeWithToolStub(
+ tool: string,
+ input: unknown,
+ options: ToolExecuteOptions,
+ execute: () => unknown,
+): unknown {
+ const stub = contextStubCall(tool, input, options.toolCallId);
+ if (stub === undefined) return execute();
+ return executeStubbedTool(stub.playback, stub.call, options, execute);
+}
+
+async function* executeStubbedTool(
+ playback: ToolStubPlayback,
+ call: { readonly callId: string; readonly input: unknown; readonly tool: string },
+ options: ToolExecuteOptions,
+ execute: () => unknown,
+): AsyncIterable<unknown> {
+ options.abortSignal?.throwIfAborted();
+ const result = await playback.call(call);
+ if (result.kind === "error") throw new Error(result.error);
+ if (result.kind === "stub") {
+ yield result.outcome.response;
+ return;
+ }
+ const output = await execute();
+ if (isAsyncIterable(output)) yield* output;
+ else yield output;
+}
+
+/** The connection wrapper checks operation inputs and approvals before requesting a stub. */
+export async function connectionToolStub(
+ tool: string,
+ input: unknown,
+ callId: string,
+): Promise<StubResult> {
+ const stub = contextStubCall(tool, input, callId);
+ return stub === undefined ? { kind: "real" } : await stub.playback.call(stub.call);
+}
+
+function contextStubCall(tool: string, input: unknown, callId: string) {
+ const context = contextStorage.getStore();
+ if (context === undefined) return undefined;
+ const target = findStubTarget(context.get(ToolStubsKey), tool);
+ if (target === undefined) return undefined;
+ const session = context.require(SessionKey);
+ return {
+ playback: context.require(ToolStubPlaybackKey),
+ call: {
+ tool: target.tool,
+ input,
+ callId: stubCallId(session.sessionId, session.turn.id, callId),
+ },
+ };
+}
+
+/** Record errors converting stub responses so eval verification can detect them. */
+export async function recordToolStubFailure(
+ tool: string,
+ callId: string | undefined,
+ turnId?: string,
+): Promise<void> {
+ const context = contextStorage.getStore();
+ const scope = context?.get(ToolStubsKey);
+ if (scope === undefined || callId === undefined) return;
+ // Connection output conversion runs on the wrapper, after a qualified operation was stubbed.
+ if (tool !== "connection_execute" && findStubTarget(scope, tool) === undefined) return;
+ const session = context!.require(SessionKey);
+ await context!
+ .require(ToolStubPlaybackKey)
+ .fail(
+ stubCallId(session.sessionId, turnId ?? session.turn.id, callId),
+ `Stubbed tool "${tool}" failed during output processing.`,
+ );
+}
+
+/** Record stub failures even when the caller recovers with fallback output. */
+export async function observeToolOutput<T>(
+ tool: string,
+ calls: readonly { readonly callId: string; readonly turnId?: string }[],
+ project: () => T | Promise<T>,
+ recover?: (error: unknown) => T,
+): Promise<T> {
+ try {
+ return await project();
+ } catch (error) {
+ try {
+ for (const call of calls) await recordToolStubFailure(tool, call.callId, call.turnId);
+ } catch (reportingError) {
+ throw new AggregateError(
+ [error, reportingError],
+ "Tool output processing and failure reporting failed.",
+ { cause: error },
+ );
+ }
+ if (recover !== undefined) return recover(error);
+ throw error;
+ }
+}
diff --git a/packages/eve/src/tool-stubs/playback.ts b/packages/eve/src/tool-stubs/playback.ts
index 377902d4e..8973722cf 100644
--- a/packages/eve/src/tool-stubs/playback.ts
+++ b/packages/eve/src/tool-stubs/playback.ts
@@ -29,6 +29,10 @@ export class StubPlayback {
}));
}
+ get matchedRuleCount(): number {
+ return this.positions.size;
+ }
+
call(call: StubCall): StubResult {
const recorded = this.results.get(call.callId);
if (recorded !== undefined) return recorded;
diff --git a/packages/eve/src/tool-stubs/target.ts b/packages/eve/src/tool-stubs/target.ts
new file mode 100644
index 000000000..fb377e4ac
--- /dev/null
+++ b/packages/eve/src/tool-stubs/target.ts
@@ -0,0 +1,16 @@
+import type { StubScope } from "#tool-stubs/types.js";
+
+/** Qualify names for stub matching without changing the names sent to the model. */
+export function stubToolPath(scope: StubScope | undefined, tool: string): string {
+ return scope?.agentPath === undefined ? tool : `${scope.agentPath}/${tool}`;
+}
+
+export function findStubTarget(scope: StubScope | undefined, tool: string) {
+ if (scope === undefined) return undefined;
+ tool = stubToolPath(scope, tool);
+ return scope.rules.some((rule) => rule.tool === tool) ? { scope, tool } : undefined;
+}
+
+export function stubCallId(sessionId: string, turnId: string, callId: string): string {
+ return `${sessionId}:${turnId}:${callId}`;
+}
diff --git a/packages/eve/src/tool-stubs/types.ts b/packages/eve/src/tool-stubs/types.ts
index 3275fa7f8..ad06b60e6 100644
--- a/packages/eve/src/tool-stubs/types.ts
+++ b/packages/eve/src/tool-stubs/types.ts
@@ -47,5 +47,6 @@ export interface StubScope {
}
export const STUB_CONTEXT_KEY = "eve.toolStubs";
+export const STUB_MATCHES_NAMESPACE = "eve.tool-stubs.matches";
export const STUB_FAILURE_NAMESPACE = "eve.tool-stubs.failure";
export const stubResponseNamespace = (callId: string): string => `eve.tool-stubs.${callId}`;