Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): emit sibling sign-ins beside pending approvals (#4497)

Rui7f87e286a543
diff --git a/.changeset/quiet-sign-in-siblings.md b/.changeset/quiet-sign-in-siblings.md
new file mode 100644
index 000000000..b483b68db
--- /dev/null
+++ b/.changeset/quiet-sign-in-siblings.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Emit sign-in requests for executing tools even when a sibling call in the same model step needs approval.
diff --git a/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.ts b/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.ts
index 6baca07fc..812f9c9f0 100644
--- a/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.ts
+++ b/e2e/fixtures/agent-tools-hitl/agent/lib/hitl/respond.ts
@@ -11,6 +11,7 @@ export const SAY = {
secondEcho: "Alice asks for a guarded echo of the second note.",
retiring: "Alice asks for the retiring change.",
checkAccess: "Alice asks to check her Fixture Auth access.",
+ checkAndPublish: "Alice asks to check her access and publish the draft together.",
publish: "Alice asks to publish the draft.",
hello: "Alice changes her mind and asks for a short hello instead.",
bobStatus: "Bob asks for a status update.",
@@ -95,6 +96,11 @@ export function respond(request: MockModelRequest): MockModelResponse {
),
[SAY.publish]: () =>
run([call("publish", "publish-draft")], () => `Publish: ${outcome("publish")}.`),
+ [SAY.checkAndPublish]: () =>
+ run(
+ [call("probe", "auth-probe", { marker: "hitl" }), call("publish", "publish-draft")],
+ () => `Access check: ${outcome("probe")}. Publish: ${outcome("publish")}.`,
+ ),
[SAY.hello]: () => ({
text: allText.includes(SIGN_IN_DROPPED) ? REPLY.helloAfterAuthorization : REPLY.hello,
}),
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md b/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md
index b88de5ea9..5b4e4578a 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/README.md
@@ -23,28 +23,29 @@ and [`agent-workflow-tools/evals/hitl`](../../../agent-workflow-tools/evals/hitl
## Coverage
-| Rule | Eval |
-| ------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
-| Approve runs the call in eve, result before the next step | [approve-runs-the-call](./approvals/approve-runs-the-call.eval.ts) |
-| Deny resolves `denied` and records a not-run result | [deny-records-not-run](./approvals/deny-records-not-run.eval.ts) |
-| A partial answer keeps the turn held, with no model call | [partial-answer-keeps-turn-held](./approvals/partial-answer-keeps-turn-held.eval.ts) |
-| A typed reply naming an option answers | [typed-approve-answers](./approvals/typed-approve-answers.eval.ts) |
-| A typed reply answers only what it matches | [typed-reply-answers-only-what-matches](./approvals/typed-reply-answers-only-what-matches.eval.ts) |
-| Steering ignores unanswered approvals and keeps given answers | [steer-ignores-unanswered](./approvals/steer-ignores-unanswered.eval.ts) |
-| Someone else's message waits for the held turn | [other-person-message-waits](./approvals/other-person-message-waits.eval.ts) |
-| An answer to a request that is not open authorizes nothing | [stale-answer-authorizes-nothing](./approvals/stale-answer-authorizes-nothing.eval.ts) |
-| A `once()` grant is reused | [once-grant-is-reused](./approvals/once-grant-is-reused.eval.ts) |
-| An approved call runs with the asking step's tools | [approved-call-uses-asking-step-tools](./approvals/approved-call-uses-asking-step-tools.eval.ts) |
-| A rejected responder leaves the approval open | [rejected-responder-leaves-approval-open](./response-policies/rejected-responder-leaves-approval-open.eval.ts) |
-| An allowed responder settles the approval | [allowed-responder-settles](./response-policies/allowed-responder-settles.eval.ts) |
-| A responder's authorization, then settlement | [responder-authorization-then-settles](./response-policies/responder-authorization-then-settles.eval.ts) |
-| Stop cancels the turn, resolving once | [stop-cancels-once](./budget/stop-cancels-once.eval.ts) |
-| No step starts behind the budget question | [no-step-behind-budget](./budget/no-step-behind-budget.eval.ts) |
-| A late budget answer is dropped | [stale-budget-answer-dropped](./budget/stale-budget-answer-dropped.eval.ts) |
-| An authorization holds the turn; the callback resumes it as the requester | [authorization-holds-then-resumes-as-requester](./authorizations/authorization-holds-then-resumes-as-requester.eval.ts) |
-| Steering declines an authorization and tells the model | [steer-declines-authorization](./authorizations/steer-declines-authorization.eval.ts) |
-| Cancel declines an authorization | [cancel-declines-authorization](./authorizations/cancel-declines-authorization.eval.ts) |
-| An approved call that needs an authorization | [approved-call-needs-authorization](./authorizations/approved-call-needs-authorization.eval.ts) |
+| Rule | Eval |
+| ---------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
+| Approve runs the call in eve, result before the next step | [approve-runs-the-call](./approvals/approve-runs-the-call.eval.ts) |
+| Deny resolves `denied` and records a not-run result | [deny-records-not-run](./approvals/deny-records-not-run.eval.ts) |
+| A partial answer keeps the turn held, with no model call | [partial-answer-keeps-turn-held](./approvals/partial-answer-keeps-turn-held.eval.ts) |
+| A typed reply naming an option answers | [typed-approve-answers](./approvals/typed-approve-answers.eval.ts) |
+| A typed reply answers only what it matches | [typed-reply-answers-only-what-matches](./approvals/typed-reply-answers-only-what-matches.eval.ts) |
+| Steering ignores unanswered approvals and keeps given answers | [steer-ignores-unanswered](./approvals/steer-ignores-unanswered.eval.ts) |
+| Someone else's message waits for the held turn | [other-person-message-waits](./approvals/other-person-message-waits.eval.ts) |
+| An answer to a request that is not open authorizes nothing | [stale-answer-authorizes-nothing](./approvals/stale-answer-authorizes-nothing.eval.ts) |
+| A `once()` grant is reused | [once-grant-is-reused](./approvals/once-grant-is-reused.eval.ts) |
+| An approved call runs with the asking step's tools | [approved-call-uses-asking-step-tools](./approvals/approved-call-uses-asking-step-tools.eval.ts) |
+| A rejected responder leaves the approval open | [rejected-responder-leaves-approval-open](./response-policies/rejected-responder-leaves-approval-open.eval.ts) |
+| An allowed responder settles the approval | [allowed-responder-settles](./response-policies/allowed-responder-settles.eval.ts) |
+| A responder's authorization, then settlement | [responder-authorization-then-settles](./response-policies/responder-authorization-then-settles.eval.ts) |
+| Stop cancels the turn, resolving once | [stop-cancels-once](./budget/stop-cancels-once.eval.ts) |
+| No step starts behind the budget question | [no-step-behind-budget](./budget/no-step-behind-budget.eval.ts) |
+| A late budget answer is dropped | [stale-budget-answer-dropped](./budget/stale-budget-answer-dropped.eval.ts) |
+| An authorization holds the turn; the callback resumes it as the requester | [authorization-holds-then-resumes-as-requester](./authorizations/authorization-holds-then-resumes-as-requester.eval.ts) |
+| Steering declines an authorization and tells the model | [steer-declines-authorization](./authorizations/steer-declines-authorization.eval.ts) |
+| Cancel declines an authorization | [cancel-declines-authorization](./authorizations/cancel-declines-authorization.eval.ts) |
+| An approved call that needs an authorization | [approved-call-needs-authorization](./authorizations/approved-call-needs-authorization.eval.ts) |
+| A sibling sign-in is emitted beside an approval; newer attempts replace older ones | [newer-attempt-supersedes](./authorizations/newer-attempt-supersedes.eval.ts) |
The rule that the model never runs while a request is open is checked inside
the evals above by `expectNoModelCallWhileOpen` and
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/newer-attempt-supersedes.eval.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/newer-attempt-supersedes.eval.ts
new file mode 100644
index 000000000..495c21382
--- /dev/null
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/authorizations/newer-attempt-supersedes.eval.ts
@@ -0,0 +1,52 @@
+import { defineEval } from "eve/evals";
+
+import {
+ SAY,
+ aliceSession,
+ answers,
+ approvalFor,
+ asAlice,
+ completeAuthorization,
+ follow,
+ authorizationFrom,
+} from "../helpers.ts";
+
+/**
+ * One step checks Alice's access (authorization) and asks to publish (approval).
+ * Both use the same Fixture Auth sign-in. Approving the publish runs it, and
+ * its authorization replaces the access check's open attempt, which fails as
+ * superseded. Only the newer attempt's callback completes the authorization.
+ */
+export default defineEval({
+ description: "A newer attempt at the same authorization supersedes the open one.",
+ tags: ["hitl", "authorization"],
+ timeoutMs: 90_000,
+ async test(t) {
+ const session = await aliceSession(t);
+ const asked = await session.send(SAY.checkAndPublish, asAlice);
+ const older = authorizationFrom(asked);
+ const request = approvalFor(asked, "publish-draft");
+
+ const superseded = await session.respond(answers("approve", request), asAlice);
+ superseded.event("authorization.completed", {
+ count: 1,
+ data: {
+ attemptId: older.attemptId,
+ outcome: "failed",
+ reason: "Superseded by a newer authorization attempt.",
+ },
+ });
+ const newer = authorizationFrom(superseded);
+ if (newer.attemptId === older.attemptId)
+ throw new Error("Expected a new authorization attempt.");
+
+ const startIndex = session.state.streamIndex;
+ await completeAuthorization(newer.url);
+ const resumed = (await follow(t, session, startIndex)).expectOk();
+ resumed.event("authorization.completed", {
+ count: 1,
+ data: { attemptId: newer.attemptId, outcome: "authorized" },
+ });
+ resumed.notEvent("authorization.completed", { data: { attemptId: older.attemptId } });
+ },
+});
diff --git a/packages/eve/src/harness/step/after-model.ts b/packages/eve/src/harness/step/after-model.ts
index 3b7f5aa5f..0cd91470f 100644
--- a/packages/eve/src/harness/step/after-model.ts
+++ b/packages/eve/src/harness/step/after-model.ts
@@ -139,6 +139,11 @@ export async function handleStepResult(step: Step, input: ModelResponse): Promis
.filter((toolCall) => !blockedCallIds.has(toolCall.toolCallId))
.filter((toolCall) => isWorkflowTool(input.catalog.get(toolCall.toolName)));
+ const authorizationInterrupt = resolveInlineAuthorizationInterrupt({
+ messages: [...promptMessages, ...responseMessages],
+ toolResults: result.toolResults,
+ });
+
// --- Park on approvals or runtime calls ----------------------------------
if (workflowToolCalls.length > 0 || approvalRequests.length > 0) {
@@ -158,23 +163,33 @@ export async function handleStepResult(step: Step, input: ModelResponse): Promis
tasks: dispatched.workflowRequests,
};
if (approvalRequests.length > 0) {
- return parkOnApprovals(step, {
+ const parkedResult = await parkOnApprovals(step, {
...parked,
requests: approvalRequests,
tools: input.catalog,
- waitsOnRuntime: workflowToolCalls.length > 0,
+ waitsOnRuntime: workflowToolCalls.length > 0 || authorizationInterrupt !== undefined,
});
+ // Approval only precedes sign-in for the same call, not an executing sibling.
+ if (authorizationInterrupt) {
+ return stopForToolSignIn(step, {
+ ...authorizationInterrupt,
+ history: step.session.history,
+ });
+ }
+ return parkedResult;
}
await step.apply(suspendStep(step.view(), parked));
+ if (authorizationInterrupt) {
+ return stopForToolSignIn(step, {
+ ...authorizationInterrupt,
+ history: step.session.history,
+ });
+ }
return { next: null, session: step.session };
}
// --- Park on authorization request ------------------------------------------
- const authorizationInterrupt = resolveInlineAuthorizationInterrupt({
- messages: [...promptMessages, ...responseMessages],
- toolResults: result.toolResults,
- });
if (authorizationInterrupt) return stopForToolSignIn(step, authorizationInterrupt);
// --- Continue or terminate ------------------------------------------------
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index 8b4999cc0..651906695 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -6237,6 +6237,86 @@ describe("createToolLoopHarness", () => {
return { full, modelFacing: modelFacingAuthorizationOutput(full) };
}
+ it("emits a sibling sign-in without approval while preserving the parked workflow call", async () => {
+ const { full, modelFacing } = createAuthSignals();
+ const delegateCall = {
+ input: { message: "probe" },
+ toolCallId: "delegate-1",
+ toolName: "delegate",
+ type: "tool-call" as const,
+ };
+ const signInCall = {
+ input: {},
+ toolCallId: "sign-in-1",
+ toolName: "add",
+ type: "tool-call" as const,
+ };
+ const signInResult = {
+ output: modelFacing,
+ toolCallId: signInCall.toolCallId,
+ toolName: signInCall.toolName,
+ type: "tool-result" as const,
+ };
+ setupMockAgent({
+ finishReason: "tool-calls",
+ response: {
+ messages: [
+ { content: [delegateCall, signInCall], role: "assistant" },
+ { content: [signInResult], role: "tool" },
+ ],
+ },
+ text: "",
+ toolCalls: [delegateCall, signInCall],
+ toolResults: [signInResult],
+ });
+ const { emit, events } = createEventCollector();
+ const runStep = createToolLoopHarness(
+ createTestConfig(emit, { tools: createDelegationToolMap() }),
+ );
+ const ctx = new ContextContainer();
+ stashToolInterrupt(ctx, signInCall.toolCallId, full);
+
+ const result = await contextStorage.run(ctx, () =>
+ runStep(createTestSession(), { message: "Delegate and check access." }),
+ );
+
+ expect(result.next).toBeNull();
+ expect(result.settledTurn).toBeUndefined();
+ expect(getPendingAuthorization(result.session.state)).toEqual({
+ challenges: full.challenges,
+ });
+ expect(events.filter((event) => event.type === "authorization.required")).toEqual([
+ expect.objectContaining({
+ data: expect.objectContaining({
+ attemptId: "attempt-protected-action",
+ authorization: full.challenges[0]!.challenge,
+ name: "protected_action",
+ webhookUrl: "https://app.example/callback",
+ }),
+ }),
+ ]);
+ expect(events.filter((event) => event.type === "input.requested")).toHaveLength(0);
+ expect(events.filter((event) => event.type === "action.result")).toHaveLength(0);
+ expect(runtimeWait(result.session.state)?.tasks).toEqual([
+ expect.objectContaining({ callId: delegateCall.toolCallId, kind: "workflow-task" }),
+ ]);
+ const parked = parkedSteps(result.session);
+ expect(parked).toHaveLength(1);
+ expect(parked[0]?.requests).toEqual([]);
+ expect(parked[0]?.messages).toEqual([{ content: [delegateCall], role: "assistant" }]);
+ for (const message of result.session.history) {
+ if (!Array.isArray(message.content)) continue;
+ expect(
+ message.content.filter(
+ (part) =>
+ (part.type === "tool-call" || part.type === "tool-result") &&
+ part.toolCallId === signInCall.toolCallId,
+ ),
+ ).toHaveLength(0);
+ }
+ expect(ToolLoopAgent).toHaveBeenCalledTimes(1);
+ });
+
it("emits authorization.required and parks when an approved tool returns an auth signal on the inline resume path", async () => {
const { full, modelFacing } = createAuthSignals();