The Open Framework for Building Agents
Preserve tool results across sequential approval parks (#2656)
vercel-gh-bot-2[bot]4264f0359f96
diff --git a/.changeset/fair-tools-rest.md b/.changeset/fair-tools-rest.mdnew file mode 100644index 000000000..a7cf97df0--- /dev/null+++ b/.changeset/fair-tools-rest.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Preserve completed tool results across sequential approval and authorization pauses so resumed model calls do not repeat successful tool executions.diff --git a/packages/eve/src/harness/authorization.test.ts b/packages/eve/src/harness/authorization.test.tsindex d98bea374..7b75ca5f2 100644--- a/packages/eve/src/harness/authorization.test.ts+++ b/packages/eve/src/harness/authorization.test.ts@@ -9,6 +9,7 @@ import {getPendingAuthorization,getHookUrl,PendingAuthorizationResultKey,+ resolveActiveAuthorizationChallenges,setPendingAuthorization,} from "#harness/authorization.js";import type { ConnectionPrincipal } from "#shared/connection-types.js";@@ -151,6 +152,22 @@ describe("pending authorization attempts", () => {]);});+ it("keeps only the latest same-scope challenge from one batch", () => {+ const userA = { id: "user-a", issuer: "idp", type: "user" } as const;+ const userB = { id: "user-b", issuer: "idp", type: "user" } as const;+ const first = challenge("linear", "linear-a-1", userA);+ const otherPrincipal = challenge("linear", "linear-b", userB);+ const latest = challenge("linear", "linear-a-2", userA);+ const active = resolveActiveAuthorizationChallenges([first, otherPrincipal, latest]);++ expect(active).toEqual([otherPrincipal, latest]);+ expect(+ getPendingAuthorization(+ setPendingAuthorization(undefined, { challenges: [first, otherPrincipal, latest] }),+ )?.challenges,+ ).toEqual([otherPrincipal, latest]);+ });+it("clears by exact attempt identity", () => {const state = setPendingAuthorization(undefined, {challenges: [challenge("linear", "linear-2"), challenge("github", "github-1")],diff --git a/packages/eve/src/harness/authorization.ts b/packages/eve/src/harness/authorization.tsindex a6977675a..5d0ca2b96 100644--- a/packages/eve/src/harness/authorization.ts+++ b/packages/eve/src/harness/authorization.ts@@ -287,19 +287,33 @@ export function setPendingAuthorization(sessionState: Record<string, unknown> | undefined,value: PendingAuthorizationState,): Record<string, unknown> {+ const active = resolveActiveAuthorizationChallenges(value.challenges);const previous = getPendingAuthorization(sessionState)?.challenges ?? [];- const superseded = getSupersededAuthorizationChallenges(sessionState, value.challenges);+ const superseded = getSupersededAuthorizationChallenges(sessionState, active);return {...sessionState,[PENDING_AUTHORIZATION_KEY]: {- challenges: [- ...previous.filter((challenge) => !superseded.includes(challenge)),- ...value.challenges,- ],+ challenges: [...previous.filter((challenge) => !superseded.includes(challenge)), ...active],},};}+/** Keeps the last challenge for each authorization name and principal scope. */+export function resolveActiveAuthorizationChallenges(+ challenges: readonly AuthorizationChallenge[],+): readonly AuthorizationChallenge[] {+ return challenges.filter(+ (candidate, index) =>+ !challenges+ .slice(index + 1)+ .some(+ (replacement) =>+ candidate.name === replacement.name &&+ samePrincipal(candidate.principal, replacement.principal),+ ),+ );+}+/** Existing same-scope attempts replaced by newer attempts for the same principal. */export function getSupersededAuthorizationChallenges(sessionState: Record<string, unknown> | undefined,diff --git a/packages/eve/src/harness/inline-tool-authorization.test.ts b/packages/eve/src/harness/inline-tool-authorization.test.tsnew file mode 100644index 000000000..f3b4271ed--- /dev/null+++ b/packages/eve/src/harness/inline-tool-authorization.test.ts@@ -0,0 +1,116 @@+import type { ModelMessage, ToolSet, TypedToolResult } from "ai";+import { describe, expect, it } from "vitest";++import { requestAuthorization } from "#harness/authorization.js";+import { resolveInlineAuthorizationInterrupt } from "#harness/inline-tool-authorization.js";++function authorizationResult(toolCallId: string, attemptId: string) {+ return {+ input: { action: "authorize" },+ output: requestAuthorization([+ {+ attemptId,+ challenge: { url: `https://idp.example/${attemptId}` },+ hookUrl: `https://app.example/${attemptId}`,+ name: "protected_action",+ principal: { type: "app" },+ },+ ]),+ toolCallId,+ toolName: "protected_action",+ type: "tool-result" as const,+ } satisfies TypedToolResult<ToolSet>;+}++const interruptedCall = {+ input: { action: "authorize" },+ toolCallId: "call-auth",+ toolName: "protected_action",+ type: "tool-call" as const,+};++const siblingCall = {+ input: { action: "complete" },+ toolCallId: "call-sibling",+ toolName: "protected_action",+ type: "tool-call" as const,+};++const siblingResult = {+ input: siblingCall.input,+ output: { type: "text" as const, value: "completed" },+ toolCallId: siblingCall.toolCallId,+ toolName: siblingCall.toolName,+ type: "tool-result" as const,+};++function modelResult(result: ReturnType<typeof authorizationResult>) {+ return {+ input: result.input,+ output: { type: "text" as const, value: "Authorization required." },+ toolCallId: result.toolCallId,+ toolName: result.toolName,+ type: "tool-result" as const,+ };+}++describe("resolveInlineAuthorizationInterrupt", () => {+ it("removes an interrupted call, its result, and its orphaned assistant text", () => {+ const messages: ModelMessage[] = [+ {+ content: [{ text: "I'll authorize this action.", type: "text" }, interruptedCall],+ role: "assistant",+ },+ {+ content: [modelResult(authorizationResult(interruptedCall.toolCallId, "attempt-1"))],+ role: "tool",+ },+ ];++ expect(+ resolveInlineAuthorizationInterrupt({+ messages,+ toolResults: [authorizationResult(interruptedCall.toolCallId, "attempt-1")],+ }),+ ).toMatchObject({+ challenges: [{ attemptId: "attempt-1" }],+ history: [],+ });+ });++ it("preserves protocol-complete sibling calls and results", () => {+ const authorization = authorizationResult(interruptedCall.toolCallId, "attempt-1");+ const messages: ModelMessage[] = [+ {+ content: [interruptedCall, siblingCall],+ role: "assistant",+ },+ {+ content: [modelResult(authorization), siblingResult],+ role: "tool",+ },+ ];++ expect(+ resolveInlineAuthorizationInterrupt({+ messages,+ toolResults: [authorization, siblingResult],+ })?.history,+ ).toEqual([+ { content: [siblingCall], role: "assistant" },+ { content: [siblingResult], role: "tool" },+ ]);+ });++ it("keeps only the latest same-scope challenge", () => {+ const first = authorizationResult("call-first", "attempt-1");+ const latest = authorizationResult("call-latest", "attempt-2");++ expect(+ resolveInlineAuthorizationInterrupt({+ messages: [],+ toolResults: [first, latest],+ })?.challenges,+ ).toMatchObject([{ attemptId: "attempt-2" }]);+ });+});diff --git a/packages/eve/src/harness/inline-tool-authorization.ts b/packages/eve/src/harness/inline-tool-authorization.tsindex f4b9cf177..461b44508 100644--- a/packages/eve/src/harness/inline-tool-authorization.ts+++ b/packages/eve/src/harness/inline-tool-authorization.ts@@ -1,17 +1,89 @@-import type { ToolSet, TypedToolResult } from "ai";+import type { ModelMessage, ToolSet, TypedToolResult } from "ai";+import { contextStorage } from "#context/container.js";-import { isAuthorizationSignal, isPendingAuthorizationToolOutput } from "#harness/authorization.js";+import {+ type AuthorizationSignal,+ isAuthorizationSignal,+ isPendingAuthorizationToolOutput,+ resolveActiveAuthorizationChallenges,+} from "#harness/authorization.js";import { readToolInterrupt } from "#harness/tool-interrupts.js";/** Returns whether an inline tool result represents a pending authorization interrupt. */export function isInlineAuthorizationToolResult(toolResult: TypedToolResult<ToolSet>): boolean {- if (isPendingAuthorizationToolOutput(toolResult.output)) {- return true;+ return (+ isPendingAuthorizationToolOutput(toolResult.output) ||+ readAuthorizationSignal(toolResult) !== undefined+ );+}++/**+ * Resolves authorization interrupts and keeps only protocol-complete sibling+ * calls from the response that produced them.+ */+export function resolveInlineAuthorizationInterrupt(input: {+ readonly messages: readonly ModelMessage[];+ readonly toolResults: readonly TypedToolResult<ToolSet>[] | undefined;+}):+ | {+ readonly challenges: AuthorizationSignal["challenges"];+ readonly history: ModelMessage[];+ }+ | undefined {+ const signals: AuthorizationSignal[] = [];+ const interruptedCallIds = new Set<string>();++ for (const toolResult of input.toolResults ?? []) {+ const signal = readAuthorizationSignal(toolResult);+ if (signal === undefined) continue;+ signals.push(signal);+ interruptedCallIds.add(toolResult.toolCallId);}++ if (signals.length === 0) return undefined;++ return {+ challenges: resolveActiveAuthorizationChallenges(+ signals.flatMap((signal) => signal.challenges),+ ),+ history: projectCompletedSiblingCalls(input.messages, interruptedCallIds),+ };+}++function readAuthorizationSignal(+ toolResult: TypedToolResult<ToolSet>,+): AuthorizationSignal | undefined {const ctx = contextStorage.getStore();- if (ctx === undefined) {- return false;- }- const stashed = readToolInterrupt(ctx, toolResult.toolCallId);- return stashed !== undefined && isAuthorizationSignal(stashed);+ const stashed = ctx === undefined ? undefined : readToolInterrupt(ctx, toolResult.toolCallId);+ if (stashed !== undefined && isAuthorizationSignal(stashed)) return stashed;+ return isAuthorizationSignal(toolResult.output) ? toolResult.output : undefined;+}++function projectCompletedSiblingCalls(+ messages: readonly ModelMessage[],+ interruptedCallIds: ReadonlySet<string>,+): ModelMessage[] {+ const projected = messages.flatMap((message): ModelMessage[] => {+ if (message.role === "assistant" && Array.isArray(message.content)) {+ const interrupted = message.content.some(+ (part) => part.type === "tool-call" && interruptedCallIds.has(part.toolCallId),+ );+ const content = message.content.filter(+ (part) => part.type !== "tool-call" || !interruptedCallIds.has(part.toolCallId),+ );+ const hasSiblingCall = content.some((part) => part.type === "tool-call");+ return content.length === 0 || (interrupted && !hasSiblingCall)+ ? []+ : [{ ...message, content }];+ }+ if (message.role === "tool") {+ const content = message.content.filter(+ (part) => part.type !== "tool-result" || !interruptedCallIds.has(part.toolCallId),+ );+ return content.length === 0 ? [] : [{ ...message, content }];+ }+ return [message];+ });++ return projected;}diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex 59fa8df3b..f4b68c0c6 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -614,7 +614,7 @@ function createPendingBashApprovalSession(): HarnessSession {});}-function createPendingProtectedActionApprovalSession(): HarnessSession {+function createPendingProtectedActionApprovalSessionWithSiblingCall(): HarnessSession {return appendPendingInputBatch({requests: [{@@ -649,6 +649,12 @@ function createPendingProtectedActionApprovalSession(): HarnessSession {toolCallId: "call-1",type: "tool-approval-request",},+ {+ input: { action: "sibling" },+ toolCallId: "call-2",+ toolName: "protected_action",+ type: "tool-call",+ },],role: "assistant",},@@ -1619,7 +1625,7 @@ describe("createToolLoopHarness", () => {setupMockAgent({content: [gateToolCall,- { approvalId: "approval-1", toolCallId: "gate-1", type: "tool-approval-request" },+ { approvalId: "approval-gate", toolCallId: "gate-1", type: "tool-approval-request" },delegateToolCall,],finishReason: "tool-calls",@@ -1628,13 +1634,34 @@ describe("createToolLoopHarness", () => {{content: [gateToolCall,- { approvalId: "approval-1", toolCallId: "gate-1", type: "tool-approval-request" },+ { approvalId: "approval-gate", toolCallId: "gate-1", type: "tool-approval-request" },delegateToolCall,],role: "assistant",},],},+ responseMessages: [+ {+ content: [+ {+ output: { type: "text", value: "/workspace" },+ toolCallId: "call-1",+ toolName: "bash",+ type: "tool-result",+ },+ ],+ role: "tool",+ },+ {+ content: [+ gateToolCall,+ { approvalId: "approval-gate", toolCallId: "gate-1", type: "tool-approval-request" },+ delegateToolCall,+ ],+ role: "assistant",+ },+ ],text: "",toolCalls: [gateToolCall, delegateToolCall],toolResults: [],@@ -1645,7 +1672,9 @@ describe("createToolLoopHarness", () => {createTestConfig("conversation", emit, { tools: createDelegationToolMap() }),);- const parked = await runStep(createTestSession(), { message: "Gate and delegate." });+ const parked = await runStep(createPendingBashApprovalSession(), {+ inputResponses: [{ optionId: "approve", requestId: "approval-1" }],+ });expect(parked.next).toBeNull();expect(getPendingRuntimeActionBatch(parked.session.state)?.actions).toEqual([@@ -1653,6 +1682,15 @@ describe("createToolLoopHarness", () => {]);expect(hasPendingInputBatch(parked.session.state)).toBe(true);expect(events.filter((event) => event.type === "input.requested")).toHaveLength(1);+ expect(+ parked.session.history.flatMap((message) =>+ Array.isArray(message.content)+ ? message.content.filter(+ (part) => part.type === "tool-result" && part.toolCallId === "call-1",+ )+ : [],+ ),+ ).toHaveLength(1);const parkedWithRunningDelegate = {...parked.session,@@ -5949,109 +5987,47 @@ describe("createToolLoopHarness", () => {toolName: "protected_action",type: "tool-result",},- { finishReason: "stop", type: "finish-step" },- ],- response: { messages: [] },- text: "",- toolCalls: [],- toolResults: [],- });-- const { emit, events } = createEventCollector();- const runStep = createToolLoopHarness(- createTestConfig("conversation", emit, {- tools: new Map([- [- "protected_action",- {- description: "Run a protected action",- execute: vi.fn(),- inputSchema: jsonSchema({ type: "object" }),- name: "protected_action",- },- ],- ]),- }),- );- const ctx = new ContextContainer();- stashToolInterrupt(ctx, "call-1", full);-- const result = await contextStorage.run(ctx, () =>- runStep(createPendingProtectedActionApprovalSession(), {- inputResponses: [{ optionId: "approve", requestId: "approval-1" }],- }),- );-- expect(result.next).toBeNull();- expect(result.settledTurn).toBeUndefined();- expect(getPendingAuthorization(result.session.state)).toEqual({- challenges: full.challenges,- });-- const authRequired = events.filter((event) => event.type === "authorization.required");- expect(authRequired).toHaveLength(1);- expect(authRequired[0]?.data).toMatchObject({- authorization: {- url: "https://idp.example/auth",- instructions: "Sign in to continue",- userCode: "GFI-QLM",- },- name: "protected_action",- webhookUrl: "https://app.example/callback",- });-- const actionResults = events.filter((event) => event.type === "action.result");- expect(actionResults).toHaveLength(0);- });-- it("still parks on authorization without emitting action.result when interactive auth fires in the same step", async () => {- const { full, modelFacing } = createAuthSignals();-- setupMockAgent({- finishReason: "tool-calls",- fullStreamParts: [- {- input: { action: "run" },- toolCallId: "call-1",- toolName: "protected_action",- type: "tool-call",- },{- output: modelFacing,- toolCallId: "call-1",+ output: { type: "text", value: "sibling completed" },+ toolCallId: "call-2",toolName: "protected_action",type: "tool-result",},- { finishReason: "tool-calls", type: "finish-step" },+ { finishReason: "stop", type: "finish-step" },],response: {messages: [{content: [{- input: { action: "run" },+ output: modelFacing,toolCallId: "call-1",toolName: "protected_action",- type: "tool-call",+ type: "tool-result",+ },+ {+ output: { type: "text", value: "sibling completed" },+ toolCallId: "call-2",+ toolName: "protected_action",+ type: "tool-result",},],- role: "assistant",+ role: "tool",},],},text: "",- toolCalls: [+ toolCalls: [],+ toolResults: [{- input: { action: "run" },+ output: modelFacing,toolCallId: "call-1",toolName: "protected_action",- type: "tool-call",+ type: "tool-result",},- ],- toolResults: [{- output: modelFacing,- toolCallId: "call-1",+ output: { type: "text", value: "sibling completed" },+ toolCallId: "call-2",toolName: "protected_action",type: "tool-result",},@@ -6078,7 +6054,9 @@ describe("createToolLoopHarness", () => {stashToolInterrupt(ctx, "call-1", full);const result = await contextStorage.run(ctx, () =>- runStep(createTestSession(), { message: "run protected action" }),+ runStep(createPendingProtectedActionApprovalSessionWithSiblingCall(), {+ inputResponses: [{ optionId: "approve", requestId: "approval-1" }],+ }),);expect(result.next).toBeNull();@@ -6089,9 +6067,57 @@ describe("createToolLoopHarness", () => {const authRequired = events.filter((event) => event.type === "authorization.required");expect(authRequired).toHaveLength(1);+ expect(authRequired[0]?.data).toMatchObject({+ authorization: {+ url: "https://idp.example/auth",+ instructions: "Sign in to continue",+ userCode: "GFI-QLM",+ },+ name: "protected_action",+ webhookUrl: "https://app.example/callback",+ });const actionResults = events.filter((event) => event.type === "action.result");- expect(actionResults).toHaveLength(0);+ expect(actionResults).toHaveLength(1);+ expect(actionResults[0]?.data).toMatchObject({+ result: { callId: "call-2", kind: "tool-result", toolName: "protected_action" },+ });+ expect(+ result.session.history.flatMap((message) =>+ message.role === "tool"+ ? message.content.filter(+ (part) => part.type === "tool-result" && part.toolCallId === "call-1",+ )+ : [],+ ),+ ).toHaveLength(0);+ expect(+ result.session.history.flatMap((message) =>+ message.role === "assistant" && Array.isArray(message.content)+ ? message.content.filter(+ (part) => part.type === "tool-call" && part.toolCallId === "call-1",+ )+ : [],+ ),+ ).toHaveLength(0);+ expect(+ result.session.history.flatMap((message) =>+ message.role === "tool"+ ? message.content.filter(+ (part) => part.type === "tool-result" && part.toolCallId === "call-2",+ )+ : [],+ ),+ ).toHaveLength(1);+ expect(+ result.session.history.flatMap((message) =>+ message.role === "assistant" && Array.isArray(message.content)+ ? message.content.filter(+ (part) => part.type === "tool-call" && part.toolCallId === "call-2",+ )+ : [],+ ),+ ).toHaveLength(1);});});diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.tsindex b05b57227..de430a58f 100644--- a/packages/eve/src/harness/tool-loop.ts+++ b/packages/eve/src/harness/tool-loop.ts@@ -202,12 +202,10 @@ import {} from "#harness/messages.js";import { normalizeProviderToolHistory } from "#harness/provider-tool-history.js";import {- type AuthorizationSignal,getSupersededAuthorizationChallenges,- isAuthorizationSignal,setPendingAuthorization,} from "#harness/authorization.js";-import { readToolInterrupt } from "#harness/tool-interrupts.js";+import { resolveInlineAuthorizationInterrupt } from "#harness/inline-tool-authorization.js";import {createAuthorizationCompletedEvent,createAuthorizationRequiredEvent,@@ -2707,8 +2705,17 @@ async function handleStepResult(input: {});const inputRequests: InputRequest[] = [...approvalRequests, ...questionRequests];const pendingApprovals = renderPendingApprovalsSnippet(approvalRequests);+ // Keep outcomes from resumed work ahead of the synthetic pending-approval+ // message; only the unresolved assistant response belongs to the parked batch.+ const pendingResponseStart = responseMessages.findIndex((message) => message.role !== "tool");+ const committedResponseMessages =+ pendingResponseStart === -1+ ? responseMessages+ : responseMessages.slice(0, pendingResponseStart);+ const pendingResponseMessages = responseMessages.slice(committedResponseMessages.length);const parkedInputHistory: ModelMessage[] = [...promptMessages,+ ...committedResponseMessages,...(pendingApprovals === undefined? []: [{ content: pendingApprovals, role: "user" as const }]),@@ -2774,7 +2781,7 @@ async function handleStepResult(input: {stepIndex: emissionState.stepIndex,turnId: emissionState.turnId,},- responseMessages,+ responseMessages: pendingResponseMessages,session: { ...baseSession, history: parkedInputHistory },});@@ -2831,7 +2838,7 @@ async function handleStepResult(input: {);}).map((request) => request.requestId),- responseMessages,+ responseMessages: pendingResponseMessages,session: { ...baseSession, history: parkedInputHistory },});@@ -2859,9 +2866,12 @@ async function handleStepResult(input: {// --- Park on authorization request ------------------------------------------- const authSignal = findAuthorizationSignalFromToolResults(result.toolResults);- if (authSignal) {- const { challenges } = authSignal;+ const authorizationInterrupt = resolveInlineAuthorizationInterrupt({+ messages: [...promptMessages, ...responseMessages],+ toolResults: result.toolResults,+ });+ if (authorizationInterrupt) {+ const { challenges, history: authorizationHistory } = authorizationInterrupt;if (emit) {for (const superseded of getSupersededAuthorizationChallenges(@@ -2910,7 +2920,7 @@ async function handleStepResult(input: {session: setHarnessEmissionState({...baseSession,- history: [...promptMessages],+ history: authorizationHistory,state: setPendingAuthorization(baseSession.state, { challenges }),},emissionState,@@ -3494,25 +3504,3 @@ async function runModelCallWithRetries<T>(}}}--function findAuthorizationSignalFromToolResults(- toolResults: readonly TypedToolResult<ToolSet>[] | undefined,-): AuthorizationSignal | undefined {- const ctx = contextStorage.getStore();- if (ctx !== undefined) {- for (const toolResult of toolResults ?? []) {- const stashed = readToolInterrupt(ctx, toolResult.toolCallId);- if (stashed !== undefined && isAuthorizationSignal(stashed)) {- return stashed;- }- }- }-- for (const toolResult of toolResults ?? []) {- if (isAuthorizationSignal(toolResult.output)) {- return toolResult.output;- }- }-- return undefined;-}