The Open Framework for Building Agents
fix(eve): emit sibling sign-ins beside pending approvals (#4497)
Rui7f87e286a543
diff --git a/.changeset/quiet-sign-in-siblings.md b/.changeset/quiet-sign-in-siblings.mdnew file mode 100644index 000000000..b483b68db--- /dev/null+++ b/.changeset/quiet-sign-in-siblings.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Emit sign-in requests for executing tools even when a sibling call in the same model step needs approval.diff --git a/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.ts b/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.tsindex 6baca07fc..812f9c9f0 100644--- a/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.ts+++ b/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.ts@@ -11,6 +11,7 @@ export const SAY = {secondEcho: "Alice asks for a guarded echo of the second note.",retiring: "Alice asks for the retiring change.",checkAccess: "Alice asks to check her Fixture Auth access.",+ checkAndPublish: "Alice asks to check her access and publish the draft together.",publish: "Alice asks to publish the draft.",hello: "Alice changes her mind and asks for a short hello instead.",bobStatus: "Bob asks for a status update.",@@ -95,6 +96,11 @@ export function respond(request: MockModelRequest): MockModelResponse {),[SAY.publish]: () =>run([call("publish", "publish-draft")], () => `Publish: ${outcome("publish")}.`),+ [SAY.checkAndPublish]: () =>+ run(+ [call("probe", "auth-probe", { marker: "hitl" }), call("publish", "publish-draft")],+ () => `Access check: ${outcome("probe")}. Publish: ${outcome("publish")}.`,+ ),[SAY.hello]: () => ({text: allText.includes(SIGN_IN_DROPPED) ? REPLY.helloAfterAuthorization : REPLY.hello,}),diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md b/e2e/fixtures/agent-tools-hitl/evals/hitl/README.mdindex b88de5ea9..5b4e4578a 100644--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md@@ -23,28 +23,29 @@ and [`agent-workflow-tools/evals/hitl`](../../../agent-workflow-tools/evals/hitl## Coverage-| Rule | Eval |-| ------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |-| Approve runs the call in eve, result before the next step | [approve-runs-the-call](./approvals/approve-runs-the-call.eval.ts) |-| Deny resolves `denied` and records a not-run result | [deny-records-not-run](./approvals/deny-records-not-run.eval.ts) |-| A partial answer keeps the turn held, with no model call | [partial-answer-keeps-turn-held](./approvals/partial-answer-keeps-turn-held.eval.ts) |-| A typed reply naming an option answers | [typed-approve-answers](./approvals/typed-approve-answers.eval.ts) |-| A typed reply answers only what it matches | [typed-reply-answers-only-what-matches](./approvals/typed-reply-answers-only-what-matches.eval.ts) |-| Steering ignores unanswered approvals and keeps given answers | [steer-ignores-unanswered](./approvals/steer-ignores-unanswered.eval.ts) |-| Someone else's message waits for the held turn | [other-person-message-waits](./approvals/other-person-message-waits.eval.ts) |-| An answer to a request that is not open authorizes nothing | [stale-answer-authorizes-nothing](./approvals/stale-answer-authorizes-nothing.eval.ts) |-| A `once()` grant is reused | [once-grant-is-reused](./approvals/once-grant-is-reused.eval.ts) |-| An approved call runs with the asking step's tools | [approved-call-uses-asking-step-tools](./approvals/approved-call-uses-asking-step-tools.eval.ts) |-| A rejected responder leaves the approval open | [rejected-responder-leaves-approval-open](./response-policies/rejected-responder-leaves-approval-open.eval.ts) |-| An allowed responder settles the approval | [allowed-responder-settles](./response-policies/allowed-responder-settles.eval.ts) |-| A responder's authorization, then settlement | [responder-authorization-then-settles](./response-policies/responder-authorization-then-settles.eval.ts) |-| Stop cancels the turn, resolving once | [stop-cancels-once](./budget/stop-cancels-once.eval.ts) |-| No step starts behind the budget question | [no-step-behind-budget](./budget/no-step-behind-budget.eval.ts) |-| A late budget answer is dropped | [stale-budget-answer-dropped](./budget/stale-budget-answer-dropped.eval.ts) |-| An authorization holds the turn; the callback resumes it as the requester | [authorization-holds-then-resumes-as-requester](./authorizations/authorization-holds-then-resumes-as-requester.eval.ts) |-| Steering declines an authorization and tells the model | [steer-declines-authorization](./authorizations/steer-declines-authorization.eval.ts) |-| Cancel declines an authorization | [cancel-declines-authorization](./authorizations/cancel-declines-authorization.eval.ts) |-| An approved call that needs an authorization | [approved-call-needs-authorization](./authorizations/approved-call-needs-authorization.eval.ts) |+| Rule | Eval |+| ---------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |+| Approve runs the call in eve, result before the next step | [approve-runs-the-call](./approvals/approve-runs-the-call.eval.ts) |+| Deny resolves `denied` and records a not-run result | [deny-records-not-run](./approvals/deny-records-not-run.eval.ts) |+| A partial answer keeps the turn held, with no model call | [partial-answer-keeps-turn-held](./approvals/partial-answer-keeps-turn-held.eval.ts) |+| A typed reply naming an option answers | [typed-approve-answers](./approvals/typed-approve-answers.eval.ts) |+| A typed reply answers only what it matches | [typed-reply-answers-only-what-matches](./approvals/typed-reply-answers-only-what-matches.eval.ts) |+| Steering ignores unanswered approvals and keeps given answers | [steer-ignores-unanswered](./approvals/steer-ignores-unanswered.eval.ts) |+| Someone else's message waits for the held turn | [other-person-message-waits](./approvals/other-person-message-waits.eval.ts) |+| An answer to a request that is not open authorizes nothing | [stale-answer-authorizes-nothing](./approvals/stale-answer-authorizes-nothing.eval.ts) |+| A `once()` grant is reused | [once-grant-is-reused](./approvals/once-grant-is-reused.eval.ts) |+| An approved call runs with the asking step's tools | [approved-call-uses-asking-step-tools](./approvals/approved-call-uses-asking-step-tools.eval.ts) |+| A rejected responder leaves the approval open | [rejected-responder-leaves-approval-open](./response-policies/rejected-responder-leaves-approval-open.eval.ts) |+| An allowed responder settles the approval | [allowed-responder-settles](./response-policies/allowed-responder-settles.eval.ts) |+| A responder's authorization, then settlement | [responder-authorization-then-settles](./response-policies/responder-authorization-then-settles.eval.ts) |+| Stop cancels the turn, resolving once | [stop-cancels-once](./budget/stop-cancels-once.eval.ts) |+| No step starts behind the budget question | [no-step-behind-budget](./budget/no-step-behind-budget.eval.ts) |+| A late budget answer is dropped | [stale-budget-answer-dropped](./budget/stale-budget-answer-dropped.eval.ts) |+| An authorization holds the turn; the callback resumes it as the requester | [authorization-holds-then-resumes-as-requester](./authorizations/authorization-holds-then-resumes-as-requester.eval.ts) |+| Steering declines an authorization and tells the model | [steer-declines-authorization](./authorizations/steer-declines-authorization.eval.ts) |+| Cancel declines an authorization | [cancel-declines-authorization](./authorizations/cancel-declines-authorization.eval.ts) |+| An approved call that needs an authorization | [approved-call-needs-authorization](./authorizations/approved-call-needs-authorization.eval.ts) |+| A sibling sign-in is emitted beside an approval; newer attempts replace older ones | [newer-attempt-supersedes](./authorizations/newer-attempt-supersedes.eval.ts) |The rule that the model never runs while a request is open is checked insidethe evals above by `expectNoModelCallWhileOpen` anddiff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/newer-attempt-supersedes.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/newer-attempt-supersedes.eval.tsnew file mode 100644index 000000000..495c21382--- /dev/null+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/newer-attempt-supersedes.eval.ts@@ -0,0 +1,52 @@+import { defineEval } from "eve/evals";++import {+ SAY,+ aliceSession,+ answers,+ approvalFor,+ asAlice,+ completeAuthorization,+ follow,+ authorizationFrom,+} from "../helpers.ts";++/**+ * One step checks Alice's access (authorization) and asks to publish (approval).+ * Both use the same Fixture Auth sign-in. Approving the publish runs it, and+ * its authorization replaces the access check's open attempt, which fails as+ * superseded. Only the newer attempt's callback completes the authorization.+ */+export default defineEval({+ description: "A newer attempt at the same authorization supersedes the open one.",+ tags: ["hitl", "authorization"],+ timeoutMs: 90_000,+ async test(t) {+ const session = await aliceSession(t);+ const asked = await session.send(SAY.checkAndPublish, asAlice);+ const older = authorizationFrom(asked);+ const request = approvalFor(asked, "publish-draft");++ const superseded = await session.respond(answers("approve", request), asAlice);+ superseded.event("authorization.completed", {+ count: 1,+ data: {+ attemptId: older.attemptId,+ outcome: "failed",+ reason: "Superseded by a newer authorization attempt.",+ },+ });+ const newer = authorizationFrom(superseded);+ if (newer.attemptId === older.attemptId)+ throw new Error("Expected a new authorization attempt.");++ const startIndex = session.state.streamIndex;+ await completeAuthorization(newer.url);+ const resumed = (await follow(t, session, startIndex)).expectOk();+ resumed.event("authorization.completed", {+ count: 1,+ data: { attemptId: newer.attemptId, outcome: "authorized" },+ });+ resumed.notEvent("authorization.completed", { data: { attemptId: older.attemptId } });+ },+});diff --git a/packages/eve/src/harness/step/after-model.ts b/packages/eve/src/harness/step/after-model.tsindex 3b7f5aa5f..0cd91470f 100644--- a/packages/eve/src/harness/step/after-model.ts+++ b/packages/eve/src/harness/step/after-model.ts@@ -139,6 +139,11 @@ export async function handleStepResult(step: Step, input: ModelResponse): Promis.filter((toolCall) => !blockedCallIds.has(toolCall.toolCallId)).filter((toolCall) => isWorkflowTool(input.catalog.get(toolCall.toolName)));+ const authorizationInterrupt = resolveInlineAuthorizationInterrupt({+ messages: [...promptMessages, ...responseMessages],+ toolResults: result.toolResults,+ });+// --- Park on approvals or runtime calls ----------------------------------if (workflowToolCalls.length > 0 || approvalRequests.length > 0) {@@ -158,23 +163,33 @@ export async function handleStepResult(step: Step, input: ModelResponse): Promistasks: dispatched.workflowRequests,};if (approvalRequests.length > 0) {- return parkOnApprovals(step, {+ const parkedResult = await parkOnApprovals(step, {...parked,requests: approvalRequests,tools: input.catalog,- waitsOnRuntime: workflowToolCalls.length > 0,+ waitsOnRuntime: workflowToolCalls.length > 0 || authorizationInterrupt !== undefined,});+ // Approval only precedes sign-in for the same call, not an executing sibling.+ if (authorizationInterrupt) {+ return stopForToolSignIn(step, {+ ...authorizationInterrupt,+ history: step.session.history,+ });+ }+ return parkedResult;}await step.apply(suspendStep(step.view(), parked));+ if (authorizationInterrupt) {+ return stopForToolSignIn(step, {+ ...authorizationInterrupt,+ history: step.session.history,+ });+ }return { next: null, session: step.session };}// --- Park on authorization request ------------------------------------------- const authorizationInterrupt = resolveInlineAuthorizationInterrupt({- messages: [...promptMessages, ...responseMessages],- toolResults: result.toolResults,- });if (authorizationInterrupt) return stopForToolSignIn(step, authorizationInterrupt);// --- Continue or terminate ------------------------------------------------diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex 8b4999cc0..651906695 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -6237,6 +6237,86 @@ describe("createToolLoopHarness", () => {return { full, modelFacing: modelFacingAuthorizationOutput(full) };}+ it("emits a sibling sign-in without approval while preserving the parked workflow call", async () => {+ const { full, modelFacing } = createAuthSignals();+ const delegateCall = {+ input: { message: "probe" },+ toolCallId: "delegate-1",+ toolName: "delegate",+ type: "tool-call" as const,+ };+ const signInCall = {+ input: {},+ toolCallId: "sign-in-1",+ toolName: "add",+ type: "tool-call" as const,+ };+ const signInResult = {+ output: modelFacing,+ toolCallId: signInCall.toolCallId,+ toolName: signInCall.toolName,+ type: "tool-result" as const,+ };+ setupMockAgent({+ finishReason: "tool-calls",+ response: {+ messages: [+ { content: [delegateCall, signInCall], role: "assistant" },+ { content: [signInResult], role: "tool" },+ ],+ },+ text: "",+ toolCalls: [delegateCall, signInCall],+ toolResults: [signInResult],+ });+ const { emit, events } = createEventCollector();+ const runStep = createToolLoopHarness(+ createTestConfig(emit, { tools: createDelegationToolMap() }),+ );+ const ctx = new ContextContainer();+ stashToolInterrupt(ctx, signInCall.toolCallId, full);++ const result = await contextStorage.run(ctx, () =>+ runStep(createTestSession(), { message: "Delegate and check access." }),+ );++ expect(result.next).toBeNull();+ expect(result.settledTurn).toBeUndefined();+ expect(getPendingAuthorization(result.session.state)).toEqual({+ challenges: full.challenges,+ });+ expect(events.filter((event) => event.type === "authorization.required")).toEqual([+ expect.objectContaining({+ data: expect.objectContaining({+ attemptId: "attempt-protected-action",+ authorization: full.challenges[0]!.challenge,+ name: "protected_action",+ webhookUrl: "https://app.example/callback",+ }),+ }),+ ]);+ expect(events.filter((event) => event.type === "input.requested")).toHaveLength(0);+ expect(events.filter((event) => event.type === "action.result")).toHaveLength(0);+ expect(runtimeWait(result.session.state)?.tasks).toEqual([+ expect.objectContaining({ callId: delegateCall.toolCallId, kind: "workflow-task" }),+ ]);+ const parked = parkedSteps(result.session);+ expect(parked).toHaveLength(1);+ expect(parked[0]?.requests).toEqual([]);+ expect(parked[0]?.messages).toEqual([{ content: [delegateCall], role: "assistant" }]);+ for (const message of result.session.history) {+ if (!Array.isArray(message.content)) continue;+ expect(+ message.content.filter(+ (part) =>+ (part.type === "tool-call" || part.type === "tool-result") &&+ part.toolCallId === signInCall.toolCallId,+ ),+ ).toHaveLength(0);+ }+ expect(ToolLoopAgent).toHaveBeenCalledTimes(1);+ });+it("emits authorization.required and parks when an approved tool returns an auth signal on the inline resume path", async () => {const { full, modelFacing } = createAuthSignals();