Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): report a sign-in once completeAuthorization settles (#4762)

roboeve[bot]6210b1d7a829
diff --git a/.changeset/sign-in-completed-after-completion.md b/.changeset/sign-in-completed-after-completion.md
new file mode 100644
index 000000000..5d70804d4
--- /dev/null
+++ b/.changeset/sign-in-completed-after-completion.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+A call a sign-in stopped now waits in its step and runs itself once the sign-in calls back, instead of the model being asked again. `authorization.completed` is reported by that call's outcome, just before its `action.result`: `authorized` when the call returns, `failed` (with the error's `reason`, such as `access_denied`) when it throws. Steering a held turn fails the waiting call and declines its sign-in, and cancel stops it with the turn. A turn needing a sign-in now makes one model call fewer. A session already waiting on a sign-in when this ships keeps reporting `authorized` when its callback arrives.
diff --git a/e2e/fixtures/agent-tools-hitl/evals/hitl/helpers.ts b/e2e/fixtures/agent-tools-hitl/evals/hitl/helpers.ts
index 5d1d42be6..614f86967 100644
--- a/e2e/fixtures/agent-tools-hitl/evals/hitl/helpers.ts
+++ b/e2e/fixtures/agent-tools-hitl/evals/hitl/helpers.ts
@@ -77,7 +77,9 @@ export function expectNoModelCallWhileOpen(on: EveEvalAssertions, requestId: str
}
/**
- * No `step.started` falls between the authorization `attemptId` opening and closing.
+ * The model says nothing between the authorization `attemptId` opening and closing. A step may start
+ * in that window: it runs the call the sign-in was for, and reports the sign-in as it settles, before
+ * the step's model call.
* `before` carries the events streamed before `on` started, such as the ask.
*/
export function expectNoModelCallDuringAuthorization(
@@ -85,23 +87,36 @@ export function expectNoModelCallDuringAuthorization(
attemptId: string,
before: EveEvalTurn["events"] = [],
) {
- on.eventsSatisfy(`no model step starts while authorization ${attemptId} is open`, (observed) => {
- const events = [...before, ...observed];
- const asked = events.findIndex(
- (event) => event.type === "authorization.required" && event.data.attemptId === attemptId,
- );
- if (asked < 0) return false;
- const closed = events.findIndex(
- (event, index) =>
- index > asked &&
- event.type === "authorization.completed" &&
- event.data.attemptId === attemptId,
- );
- const open = events.slice(asked + 1, closed < 0 ? events.length : closed);
- return !open.some((event) => event.type === "step.started");
- });
+ on.eventsSatisfy(
+ `the model says nothing while authorization ${attemptId} is open`,
+ (observed) => {
+ const events = [...before, ...observed];
+ const asked = events.findIndex(
+ (event) => event.type === "authorization.required" && event.data.attemptId === attemptId,
+ );
+ if (asked < 0) return false;
+ const closed = events.findIndex(
+ (event, index) =>
+ index > asked &&
+ event.type === "authorization.completed" &&
+ event.data.attemptId === attemptId,
+ );
+ const open = events.slice(asked + 1, closed < 0 ? events.length : closed);
+ return !open.some((event) => MODEL_OUTPUT_EVENTS.has(event.type));
+ },
+ );
}
+const MODEL_OUTPUT_EVENTS = new Set<string>([
+ "actions.requested",
+ "action.input.appended",
+ "message.appended",
+ "message.completed",
+ "reasoning.appended",
+ "reasoning.completed",
+ "step.completed",
+]);
+
/** The latest authorization `turn` asked for: its attempt and its fixture callback URL. */
export function authorizationFrom(turn: Pick<EveEvalTurn, "events">): {
attemptId: string;
diff --git a/e2e/fixtures/agent-workflow-tools/evals/public-catalog.sign-in-runs-call.eval.ts b/e2e/fixtures/agent-workflow-tools/evals/public-catalog.sign-in-runs-call.eval.ts
index 7d7d0189b..5727bcbea 100644
--- a/e2e/fixtures/agent-workflow-tools/evals/public-catalog.sign-in-runs-call.eval.ts
+++ b/e2e/fixtures/agent-workflow-tools/evals/public-catalog.sign-in-runs-call.eval.ts
@@ -4,18 +4,12 @@ import { fixtureAuthorizationCallback } from "../agent/lib/fake-service.ts";
const PROTECTED_TOOL = "public-catalog__list_orders";
-/** Why this eval is skipped: it fails until a session-path sign-in runs its call itself. */
-const KNOWN_RED =
- "Fails today: after sign-in the session path drops the waiting call and asks the model again. Remove this skip with the change that runs the call itself.";
-
export default defineEval({
description:
"After the user signs in for a protected MCP tool, the call that was waiting runs without asking the model again: no model call between authorization.completed and the call's result, and the call is not requested twice.",
timeoutMs: 90_000,
async test(t) {
- t.skip(KNOWN_RED);
-
// A fresh principal per run: the fixture provider remembers each user's token.
const alice = { "x-eve-forwarded-principal-id": `sign-in-alice-${crypto.randomUUID()}` };
const started = await t.send(
diff --git a/packages/eve/src/execution/catalog/step-catalog.integration.test.ts b/packages/eve/src/execution/catalog/step-catalog.integration.test.ts
index 0b7d9cc1d..f5e4bec89 100644
--- a/packages/eve/src/execution/catalog/step-catalog.integration.test.ts
+++ b/packages/eve/src/execution/catalog/step-catalog.integration.test.ts
@@ -361,10 +361,10 @@ describe("step catalog in the harness (real AI SDK)", () => {
(pending) => pending.attemptId !== challenge!.attemptId,
),
});
- reply(
- calls(call("items-after-sign-in", CALL_TOOL_NAME, { name: "private__list_items" })),
- text("The private catalog has Alice's lamp."),
- );
+ // The callback arrives in a later step, whose virtual context starts empty, and the call the
+ // sign-in stopped runs by itself: the model is only asked to read its result.
+ ctx.clearVirtualContext();
+ reply(text("The private catalog has Alice's lamp."));
await drive();
expect(privateCatalog.calls).toEqual([{ input: {}, tool: "list_items" }]);
@@ -616,8 +616,7 @@ describe("step catalog in the harness (real AI SDK)", () => {
"deploy_service",
"research",
"billing_specialist",
- // The call that parked for sign-in, then the model's call after it.
- "private__list_items",
+ // The call that parked for sign-in runs once, without the model asking again.
"private__list_items",
"products__get_status",
SEARCH_TOOL_NAME,
diff --git a/packages/eve/src/execution/session/entry-authorization.integration.test.ts b/packages/eve/src/execution/session/entry-authorization.integration.test.ts
index a687bccf0..e788bd2a1 100644
--- a/packages/eve/src/execution/session/entry-authorization.integration.test.ts
+++ b/packages/eve/src/execution/session/entry-authorization.integration.test.ts
@@ -261,7 +261,7 @@ describe("workflowEntry integration", () => {
});
});
- it.fails("reports a sign-in failed when completeAuthorization rejects its callback", async () => {
+ it("reports a sign-in failed when completeAuthorization rejects its callback", async () => {
const { completeCalls, runtime } = await createWeatherAuthRuntime("workflow-entry-auth-denied");
const continuationToken = "http:workflow-entry-auth-denied";
@@ -305,7 +305,7 @@ describe("workflowEntry integration", () => {
});
});
- it.fails("runs the call a sign-in was for without asking the model again", async () => {
+ it("runs the call a sign-in was for without asking the model again", async () => {
const { executions, runtime } = await createWeatherAuthRuntime("workflow-entry-auth-no-reask");
const continuationToken = "http:workflow-entry-auth-no-reask";
@@ -358,6 +358,44 @@ describe("workflowEntry integration", () => {
});
});
+ it("reports a sign-in once when two calls in one step wait on it, and runs both", async () => {
+ const { executions, runtime } = await createWeatherAuthRuntime("workflow-entry-auth-shared");
+ const continuationToken = "http:workflow-entry-auth-shared";
+
+ await runtime.run(async () => {
+ const run = await startWeatherRun(
+ continuationToken,
+ "Call tools in parallel: get_weather, get_forecast",
+ );
+ const stream = captureEvents(run);
+
+ try {
+ const beforeSignIn = await stream.nextUntil(
+ "initial auth-required event",
+ (event) => event.type === "authorization.required",
+ );
+ await stream.nextUntil("authorization hold", (event) => event.type === "turn.waiting");
+ await resumeHook(sessionInboxHookToken(sessionCommandHookToken(run.runId)), {
+ kind: "authorization-callback",
+ payloads: [weatherCallback(authorizationAttemptId(beforeSignIn))],
+ });
+ const afterSignIn = await stream.nextUntil(
+ "turn after sign-in",
+ (event) => event.type === "session.waiting",
+ );
+
+ const events = [...beforeSignIn, ...afterSignIn];
+ expect(filterEventsByType(events, "authorization.required")).toHaveLength(1);
+ expect(filterEventsByType(events, "authorization.completed")).toHaveLength(1);
+ expect(executions()).toHaveLength(2);
+ expect(filterEventsByType(events, "step.started")).toHaveLength(2);
+ } finally {
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ });
+
it("queues another person's message behind a held sign-in", async () => {
const { completeCalls, completedPrincipals, runtime } = await createWeatherAuthRuntime(
"workflow-entry-auth-queued",
@@ -436,7 +474,7 @@ describe("workflowEntry integration", () => {
});
it("cancels a held sign-in when the same person steers the turn, ignoring its late callback", async () => {
- const { completeCalls, runtime } = await createWeatherAuthRuntime(
+ const { completeCalls, executions, runtime } = await createWeatherAuthRuntime(
"workflow-entry-auth-steered",
);
const continuationToken = "http:workflow-entry-auth-steered";
@@ -471,6 +509,11 @@ describe("workflowEntry integration", () => {
expect(filterEventsByType(replacementAttempt, "authorization.completed")).toMatchObject([
{ data: { name: "weather", outcome: "declined" } },
]);
+ // The call that waited for the sign-in fails with why; it never ran.
+ expect(filterEventsByType(replacementAttempt, "action.result")).toMatchObject([
+ { data: { result: { isError: true, toolName: "get_weather" } } },
+ ]);
+ expect(executions()).toEqual([]);
await stream.nextUntil("replacement hold", (event) => event.type === "turn.waiting");
const stale = weatherCallback(authorizationAttemptId(firstAttempt), "stale-oauth-code");
@@ -496,6 +539,54 @@ describe("workflowEntry integration", () => {
turnId: "turn_0",
});
expect(completeCalls()).toBe(1);
+ expect(executions()).toEqual([{ city: "Lisbon" }]);
+ } finally {
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ });
+
+ it("answers a message that steers a held sign-in, in the same turn, without asking to sign in again", async () => {
+ const { completeCalls, executions, runtime } = await createWeatherAuthRuntime(
+ "workflow-entry-auth-steer-answers",
+ );
+ const continuationToken = "http:workflow-entry-auth-steer-answers";
+
+ await runtime.run(async () => {
+ const run = await startWeatherRun(continuationToken);
+ const stream = captureEvents(run);
+
+ try {
+ await stream.nextUntil(
+ "first auth-required event",
+ (event) => event.type === "authorization.required",
+ );
+ await stream.nextUntil("first hold", (event) => event.type === "turn.waiting");
+
+ await waitForHook(
+ { runId: run.runId },
+ { token: sessionInboxHookToken(continuationToken) },
+ );
+ await resumeHook(sessionInboxHookToken(continuationToken), {
+ auth: userAuth("user-1"),
+ kind: "send",
+ payload: { message: "Never mind that, just say hello." },
+ });
+ // A sign-in nobody completed does not stand between the user and an answer.
+ const answered = await stream.nextUntil(
+ "answer to the steering message",
+ (event) => event.type === "session.waiting",
+ );
+ expect(filterEventsByType(answered, "authorization.completed")).toMatchObject([
+ { data: { name: "weather", outcome: "declined" } },
+ ]);
+ expect(filterEventsByType(answered, "authorization.required")).toHaveLength(0);
+ expect(filterEventsByType(answered, "turn.started")).toHaveLength(0);
+ expect(filterEventsByType(answered, "message.completed")).toHaveLength(1);
+ expect(filterEventsByType(answered, "turn.completed")).toHaveLength(1);
+ expect(executions()).toEqual([]);
+ expect(completeCalls()).toBe(0);
} finally {
stream.dispose();
await run.cancel();
@@ -504,7 +595,9 @@ describe("workflowEntry integration", () => {
});
it("cancelling a held turn drops its sign-in, so a late callback does nothing", async () => {
- const { completeCalls, runtime } = await createWeatherAuthRuntime("workflow-entry-auth-cancel");
+ const { completeCalls, executions, runtime } = await createWeatherAuthRuntime(
+ "workflow-entry-auth-cancel",
+ );
const continuationToken = "http:workflow-entry-auth-cancel";
await runtime.run(async () => {
@@ -547,6 +640,8 @@ describe("workflowEntry integration", () => {
);
expect(filterEventsByType(followup, "authorization.completed")).toHaveLength(0);
expect(completeCalls()).toBe(0);
+ // The cancelled call never runs, even after its sign-in's late callback.
+ expect(executions()).toEqual([]);
} finally {
stream.dispose();
await run.cancel();
@@ -605,10 +700,11 @@ async function createWeatherAuthRuntime(
return { token: "weather-token" };
},
};
- const getWeatherTool: ResolvedToolDefinition = {
- description: "Get the current weather for a city.",
+ // Both tools sign in to the same connection, so one callback serves a step that calls both.
+ const weatherTool = (name: string): ResolvedToolDefinition => ({
+ description: `${name} for a city.`,
execute: createToolExecuteWithAuth({
- scope: "get_weather",
+ scope: name,
async execute(rawInput, rawCtx) {
const ctx = rawCtx as ToolContext;
const token = await ctx.getToken(weatherAuth, {
@@ -638,26 +734,29 @@ async function createWeatherAuthRuntime(
required: ["city"],
type: "object",
}),
- logicalPath: "tools/get_weather.ts",
- name: "get_weather",
+ logicalPath: `tools/${name}.ts`,
+ name,
owner: { kind: "application" },
- sourceId: "tools/get_weather.ts",
+ sourceId: `tools/${name}.ts`,
sourceKind: "module",
- };
+ });
+ const tools = [weatherTool("get_weather"), weatherTool("get_forecast")];
const runtime = await createTestRuntime({
agent: { name: agentName },
modules,
- tools: [getWeatherTool],
+ tools,
});
- const manifestTool = runtime.manifest.tools.find((tool) => tool.name === getWeatherTool.name);
- if (manifestTool === undefined) {
- throw new Error("Expected get_weather to be present in the test manifest.");
+ for (const tool of tools) {
+ const manifestTool = runtime.manifest.tools.find((entry) => entry.name === tool.name);
+ if (manifestTool === undefined) {
+ throw new Error(`Expected ${tool.name} to be present in the test manifest.`);
+ }
+ runtime.moduleMap.nodes[ROOT_COMPILED_AGENT_NODE_ID]!.modules[manifestTool.sourceId] = {
+ default: {
+ execute: tool.execute,
+ },
+ };
}
- runtime.moduleMap.nodes[ROOT_COMPILED_AGENT_NODE_ID]!.modules[manifestTool.sourceId] = {
- default: {
- execute: getWeatherTool.execute,
- },
- };
return {
completeCalls: () => completeCalls,
completedPrincipals: () => completedPrincipals,
@@ -689,12 +788,15 @@ function userAuth(principalId: string) {
};
}
-async function startWeatherRun(continuationToken: string) {
+async function startWeatherRun(
+ continuationToken: string,
+ message = "Use the get_weather tool to check the weather in Lisbon.",
+) {
return await startSessionOwner(workflowEntry, [
{
kind: "initial",
ownerDeploymentId: "dpl_inline",
- input: { message: "Use the get_weather tool to check the weather in Lisbon." },
+ input: { message },
serializedContext: buildSerializedContext({
auth: userAuth("user-1"),
channelKind: "http",
diff --git a/packages/eve/src/harness/hitl/approvals.ts b/packages/eve/src/harness/hitl/approvals.ts
index 6c1529252..ee3562ddb 100644
--- a/packages/eve/src/harness/hitl/approvals.ts
+++ b/packages/eve/src/harness/hitl/approvals.ts
@@ -2,7 +2,8 @@ import type { ModelMessage } from "ai";
import type { getApprovalAuditState } from "#harness/hitl/candidates.js";
import type { AuthorizationChallenge } from "#harness/authorization.js";
-import { supersededChallenges, withSignIns } from "./sign-ins.js";
+import type { StoppedCall } from "#harness/inline-tool-authorization.js";
+import { signInAttemptKey, supersededChallenges, withSignIns } from "./sign-ins.js";
import { authorizationEventFields } from "#harness/authorization-event-fields.js";
import { renderPendingApprovalsSnippet } from "#harness/hitl/approval-prompt.js";
import { isApprovalRequest } from "#harness/input-request-class.js";
@@ -47,10 +48,11 @@ import {
type ResolvedStepInput,
} from "./delivery.js";
import type { StepCoordinates, SuspendedStep, TurnState } from "#harness/session-machine/view.js";
-import { withoutCalls } from "#harness/inline-tool-authorization.js";
import {
+ awaitSignIns,
hold,
ownOpenRequestIds,
+ runnableCalls,
settle,
stepCallIds,
withoutApproved,
@@ -537,18 +539,27 @@ export function stepForRequest(view: SessionView, requestId: string): SuspendedS
}
/**
- * The held turn moves on, steered or cancelled: the sign-ins it waits on end, declined, including
- * a responder's for one of its approvals.
+ * The held turn moves on, steered: the sign-ins it waits on end, declined, including a
+ * responder's for one of its approvals, and the calls that waited for them fail with `reason`.
*/
export function withdrawSignIns(view: SessionView, reason: string): Transition {
- return {
- events: view.signIns.flatMap((challenge) => {
- const attempt = view.projection.authorizations[challenge.attemptId ?? challenge.name];
- return attempt?.status === "required" ? [signInWithdrawn(attempt, reason)] : [];
- }),
- signIns: [],
- turn: view.turn,
- };
+ const waiting = view.turn.suspended.flatMap((step) =>
+ (step.approved ?? []).filter((call) => !runnableCalls(view, step).includes(call)),
+ );
+ const withdrawn = [...view.signIns, ...waiting.flatMap((call) => call.signIns ?? [])];
+ const events = [
+ ...new Map(withdrawn.map((challenge) => [signInAttemptKey(challenge), challenge])),
+ ]
+ .map(([, challenge]) => view.projection.authorizations[challenge.attemptId ?? challenge.name])
+ .flatMap((attempt) =>
+ attempt?.status === "required" ? [signInWithdrawn(attempt, reason)] : [],
+ );
+ const settled = settle(view, {
+ results: waiting.map(({ action }) =>
+ failedCall({ callId: action.callId, failure: reason, toolName: action.toolName }),
+ ),
+ });
+ return { ...settled, events: [...events, ...settled.events], signIns: [] };
}
// ---------------------------------------------------------------------------
@@ -584,17 +595,16 @@ export function requestLimit(
// ---------------------------------------------------------------------------
/**
- * Work needs a sign-in. The calls that need it stop, settling `cancelled` until it completes;
- * an attempt it replaces fails; and each sign-in is asked for and recorded. The turn holds until
- * every sign-in calls back. Input that waited on a responder's sign-in stays queued for it. An
- * approved call that needs the sign-in leaves its step, so the model calls it again once the
- * sign-in completes.
+ * Work needs a sign-in. The calls that need it stop and wait in their steps; an attempt it
+ * replaces fails; and each sign-in is asked for and recorded. The turn holds until every sign-in
+ * calls back, and then the stopped calls run. Input that waited on a responder's sign-in stays
+ * queued for it.
*/
export function requireSignIn(
view: SessionView,
input: {
readonly challenges: readonly AuthorizationChallenge[];
- readonly callIdsByName?: ReadonlyMap<string, readonly string[]>;
+ readonly calls?: readonly StoppedCall[];
readonly queued?: StepInput;
},
): Transition {
@@ -631,15 +641,12 @@ export function requireSignIn(
? input.queued
: coalesceTurnInputs(view.turn.queued, input.queued);
const signIns = withSignIns(view.signIns, input.challenges);
- const stopped = new Set([...(input.callIdsByName?.values() ?? [])].flat());
- const suspended = view.turn.suspended.map((step) =>
- [...stepCallIds(step)].some((callId) => stopped.has(callId))
- ? { ...step, messages: withoutCalls(step.messages, stopped) }
- : step,
- );
- if (suspended.every((step, index) => step === view.turn.suspended[index])) {
- return { events, signIns, turn: { ...view.turn, queued } };
- }
- const settled = settle({ ...view, turn: { ...view.turn, queued, suspended } }, { results: [] });
- return { commit: settled.commit, events, signIns, turn: settled.turn };
+ const { calls } = input;
+ const suspended =
+ calls === undefined
+ ? view.turn.suspended
+ : view.turn.suspended.map((step) =>
+ awaitSignIns(step, { challenges: input.challenges, stopped: calls }),
+ );
+ return { events, signIns, turn: { ...view.turn, queued, suspended } };
}
diff --git a/packages/eve/src/harness/hitl/approved-calls.ts b/packages/eve/src/harness/hitl/approved-calls.ts
index 6c2721bf3..2f46d29ca 100644
--- a/packages/eve/src/harness/hitl/approved-calls.ts
+++ b/packages/eve/src/harness/hitl/approved-calls.ts
@@ -21,11 +21,17 @@ import {
import { throwIfTurnAborted } from "#harness/turn-cancellation.js";
import type { HarnessToolLookup } from "#harness/types.js";
import { SEARCH_TOOL_NAME } from "#protocol/catalog-tools.js";
-import { createActionPartialEvent, createActionResultEvent } from "#protocol/message.js";
+import {
+ createActionPartialEvent,
+ createActionResultEvent,
+ createAuthorizationCompletedEvent,
+} from "#protocol/message.js";
+import { authorizationEventFields } from "#harness/authorization-event-fields.js";
+import { isConnectionAuthorizationFailedError } from "#connections/errors.js";
import { isAsyncIterable } from "#shared/async-iterable.js";
import { toError } from "#shared/errors.js";
import { createLogger, logError } from "#internal/logging.js";
-import type { InputRequest } from "#shared/input.js";
+import type { ApprovedCall } from "#harness/session-machine/state.js";
import { parseJsonObject } from "#shared/json.js";
import { toModelSchema } from "#tools/schema.js";
import type { HandleEventFn } from "#harness/types.js";
@@ -47,7 +53,7 @@ export const APPROVED_CALL_INTERRUPTED_MESSAGE =
* results in the steps that made the calls.
*/
export async function runApprovedCalls(input: {
- readonly requests: readonly InputRequest[];
+ readonly requests: readonly ApprovedCall[];
/** The entries of the step that asked, which the approved calls run. */
readonly tools: HarnessToolLookup;
readonly approvedTools?: ReadonlySet<string>;
@@ -63,7 +69,7 @@ export async function runApprovedCalls(input: {
/** The step attempt's telemetry, which hears each execution as the AI SDK reports it. */
readonly telemetry?: TelemetryOptions;
/** Who approved each request, which the call reads as `ctx.approval`; it still runs as the requester. */
- readonly responderOf: (request: InputRequest) => SessionAuthContext | undefined;
+ readonly responderOf: (request: ApprovedCall) => SessionAuthContext | undefined;
}): Promise<{
readonly settled: readonly ApprovedCallResult[];
readonly toolResults: readonly TypedToolResult<ToolSet>[];
@@ -73,13 +79,35 @@ export async function runApprovedCalls(input: {
stepIndex: input.position.stepIndex,
turnId: input.position.turnId,
};
+ /**
+ * The sign-ins a call waited on end with the call: authorized if it returned, failed with the
+ * reason it didn't. Reported before the call's result.
+ */
+ const reportSignIns = async (call: ApprovedCall, failure?: Error | string): Promise<void> => {
+ const reason = isConnectionAuthorizationFailedError(failure)
+ ? (failure.reason ?? failure.message)
+ : failure === undefined || typeof failure === "string"
+ ? failure
+ : failure.message;
+ for (const challenge of call.signIns ?? []) {
+ await input.publish(
+ createAuthorizationCompletedEvent({
+ ...authorizationEventFields(challenge),
+ ...at,
+ outcome: reason === undefined ? "authorized" : "failed",
+ ...(reason !== undefined && { reason }),
+ }),
+ );
+ }
+ };
/** A call that ends without running: the stream reports it failed, and the model reads why. */
const settleFailure = async (
- callId: string,
- toolName: string,
+ request: ApprovedCall,
failure: Error | string,
): Promise<ApprovedCallResult> => {
+ const { callId, toolName } = request.action;
const failed = failedCall({ callId, failure, toolName });
+ await reportSignIns(request, failure);
await input.publish(createActionResultEvent({ ...at, result: failed.result }));
return { part: failed.part };
};
@@ -101,8 +129,7 @@ export async function runApprovedCalls(input: {
return {
settled: [
await settleFailure(
- callId,
- toolName,
+ request,
unavailableToolMessage(toolName, input.tools.get(SEARCH_TOOL_NAME) !== undefined),
),
],
@@ -116,7 +143,7 @@ export async function runApprovedCalls(input: {
).validate?.(args);
if (validation?.success === false) {
const message = `The approved input is no longer valid for tool "${toolName}". Request a new tool call and approval.`;
- return { settled: [await settleFailure(callId, toolName, message)], toolResults };
+ return { settled: [await settleFailure(request, message)], toolResults };
}
const recheck = await recheckApprovedCall(definition, {
abortSignal: input.abortSignal,
@@ -125,6 +152,7 @@ export async function runApprovedCalls(input: {
approvedTools: input.approvedTools,
});
if (recheck.denied) {
+ await reportSignIns(request, recheck.reason ?? TOOL_EXECUTION_DENIED_MESSAGE);
settled.push(
await rejectApprovedCall({
request,
@@ -220,6 +248,7 @@ export async function runApprovedCalls(input: {
},
};
settled.push(completed);
+ await reportSignIns(request, failure);
await input.publish(
createActionResultEvent({
...at,
@@ -244,7 +273,7 @@ export async function runApprovedCalls(input: {
}
logError(log, "approved tool failed", error, { toolName, toolCallId: callId });
return {
- settled: [await settleFailure(callId, toolName, toError(error))],
+ settled: [await settleFailure(request, toError(error))],
toolResults,
};
}
@@ -275,7 +304,7 @@ function interruptedResult(callId: string, toolName: string): ApprovedCallResult
/** Records an approved call refused by its final request-policy check. */
export async function rejectApprovedCall(input: {
- readonly request: InputRequest;
+ readonly request: ApprovedCall;
readonly reason?: string;
readonly position: {
readonly sequence: number;
diff --git a/packages/eve/src/harness/hitl/index.ts b/packages/eve/src/harness/hitl/index.ts
index 719b348f8..f2da537eb 100644
--- a/packages/eve/src/harness/hitl/index.ts
+++ b/packages/eve/src/harness/hitl/index.ts
@@ -88,17 +88,16 @@ export async function parkOnApprovals(
}
/**
- * A call the model step ran needs a sign-in: the calls that need it stop, and the model calls
- * them again once it completes.
+ * A call the model step ran needs a sign-in: the calls that need it stop and wait in their
+ * steps, and run once it completes.
*/
export async function stopForToolSignIn(
step: Step,
interrupt: NonNullable<ReturnType<typeof resolveInlineAuthorizationInterrupt>>,
): Promise<StepResult> {
- step.session = { ...step.session, history: validateHarnessModelMessages(interrupt.history) };
await step.apply(
requireSignIn(step.view(), {
- callIdsByName: interrupt.callIdsByName,
+ calls: interrupt.calls,
challenges: interrupt.challenges,
}),
step.session.history,
diff --git a/packages/eve/src/harness/hitl/intake.ts b/packages/eve/src/harness/hitl/intake.ts
index 3b097fde7..316af353c 100644
--- a/packages/eve/src/harness/hitl/intake.ts
+++ b/packages/eve/src/harness/hitl/intake.ts
@@ -7,9 +7,11 @@ import {
approvedCalls,
completeSignIn,
hold,
+ runnableCalls,
settle,
} from "#harness/session-machine/transitions.js";
-import type { SuspendedStep } from "#harness/session-machine/view.js";
+import type { SessionView, SuspendedStep } from "#harness/session-machine/view.js";
+import { signInAttemptKey } from "#harness/hitl/sign-ins.js";
import type { Step } from "#harness/step/context.js";
import { SessionLimitDeclinedError, throwIfTurnAborted } from "#harness/turn-cancellation.js";
import { bumpSessionRuntimeUsageLimits } from "#harness/turn-tag-state.js";
@@ -30,7 +32,7 @@ import { getApprovalAuditState, responderOfRequest, retireActiveCandidates } fro
import { coordinateApprovalDelivery } from "./coordinator.js";
import { deliver, resolveTypedApproval, turnInputOnly, withoutTurnInput } from "./delivery.js";
import { recheckApprovedCall } from "#harness/tools.js";
-import type { InputRequest } from "#shared/input.js";
+import type { ApprovedCall } from "#harness/session-machine/state.js";
import type { InstrumentationAttempt } from "#instrumentation/runtime.js";
import { activeTurnId } from "#harness/session-machine/view.js";
@@ -180,8 +182,9 @@ export async function acceptHumanInput(
}
// Approved calls run with the tools of the step that asked, including calls approved by an
// earlier delivery whose turn the budget stopped.
- for (const parked of step.view().turn.suspended) {
- if ((parked.approved?.length ?? 0) > 0) await restoreTools(parked);
+ const view = step.view();
+ for (const parked of view.turn.suspended) {
+ if (runnableCalls(view, parked).length > 0) await restoreTools(parked);
}
// What the answers resolved comes before the delivery's own input, as with the AI SDK: the model
// reads the approved calls' results and the denials first, and the input waits a step. A
@@ -192,7 +195,7 @@ export async function acceptHumanInput(
batch.inputs.every((entry) => entry.response !== undefined),
);
const deferred =
- answeredApprovals || approvedCalls(step.view().turn).length > 0
+ answeredApprovals || approvedCalls(step.view()).length > 0
? turnInputOnly(decision.input)
: undefined;
if (deferred !== undefined) await step.apply(deferInput(step.view(), deferred));
@@ -223,7 +226,7 @@ export async function admitApprovedWork(step: Step, work: ApprovedWork): Promise
/** Whether a suspended step holds approved calls that haven't run. */
export function hasApprovedWork(step: Step): boolean {
- return approvedCalls(step.view().turn).length > 0;
+ return approvedCalls(step.view()).length > 0;
}
/**
@@ -234,10 +237,11 @@ export function hasApprovedWork(step: Step): boolean {
export async function dispatchApprovedWorkflows(step: Step, work: ApprovedWork): Promise<boolean> {
const tasks: RuntimeWorkflowTaskRequest[] = [];
const denied: ApprovedCallResult[] = [];
- for (const parked of step.view().turn.suspended) {
+ const view = step.view();
+ for (const parked of view.turn.suspended) {
const tools = work.toolsOf(parked);
- const requests: InputRequest[] = [];
- for (const request of parked.approved ?? []) {
+ const requests: ApprovedCall[] = [];
+ for (const request of runnableCalls(view, parked)) {
const tool = tools.get(request.action.toolName);
if (tool?.workflowId === undefined) continue;
const recheck = await recheckApprovedCall(tool, {
@@ -282,6 +286,22 @@ export async function dispatchApprovedWorkflows(step: Step, work: ApprovedWork):
return true;
}
+/**
+ * Calls a sign-in stopped can share it, and it reports once: with the first call to run it, unless
+ * a call that ran earlier reported it (the event log says it is no longer required). `claimed`
+ * holds the sign-ins the calls in this batch already took.
+ */
+function reportingOnce(view: SessionView, call: ApprovedCall, claimed: Set<string>): ApprovedCall {
+ const signIns = (call.signIns ?? []).filter((challenge) => {
+ const key = signInAttemptKey(challenge);
+ const required = view.projection.authorizations[challenge.attemptId ?? challenge.name]?.status;
+ if (claimed.has(key) || required !== "required") return false;
+ claimed.add(key);
+ return true;
+ });
+ return { ...call, signIns };
+}
+
/**
* Runs approved local calls once the step has started and its budget allows it, as the AI SDK ran
* them before the model read their results: each with the tools of the step that asked, as
@@ -293,11 +313,13 @@ export async function runApprovedLocalCalls(
work: ApprovedWork,
setAttemptScope: (scope: InstrumentationAttempt | undefined) => void,
): Promise<StepResult | undefined> {
- const local = step.view().turn.suspended.map((parked) => {
+ const view = step.view();
+ const claimed = new Set<string>();
+ const local = view.turn.suspended.map((parked) => {
const tools = work.toolsOf(parked);
- const requests = (parked.approved ?? []).filter(
- (request) => tools.get(request.action.toolName)?.workflowId === undefined,
- );
+ const requests = runnableCalls(view, parked)
+ .filter((request) => tools.get(request.action.toolName)?.workflowId === undefined)
+ .map((request) => reportingOnce(view, request, claimed));
return { requests, tools };
});
const approved = local.flatMap(({ requests }) => requests);
@@ -341,13 +363,12 @@ export async function runApprovedLocalCalls(
// A cancellation ends the turn once the calls it cut short have their results.
throwIfTurnAborted(step.config.abortSignal);
const signIn = resolveInlineAuthorizationInterrupt({
- messages: [],
toolResults: executed.flatMap((run) => run.toolResults),
});
if (signIn === undefined) return undefined;
await step.apply(
requireSignIn(step.view(), {
- callIdsByName: signIn.callIdsByName,
+ calls: signIn.calls,
challenges: signIn.challenges,
}),
step.session.history,
diff --git a/packages/eve/src/harness/inline-tool-authorization.test.ts b/packages/eve/src/harness/inline-tool-authorization.test.ts
index f3b4271ed..c3142a3dd 100644
--- a/packages/eve/src/harness/inline-tool-authorization.test.ts
+++ b/packages/eve/src/harness/inline-tool-authorization.test.ts
@@ -1,4 +1,4 @@
-import type { ModelMessage, ToolSet, TypedToolResult } from "ai";
+import type { ToolSet, TypedToolResult } from "ai";
import { describe, expect, it } from "vitest";
import { requestAuthorization } from "#harness/authorization.js";
@@ -22,83 +22,32 @@ function authorizationResult(toolCallId: string, attemptId: string) {
} satisfies TypedToolResult<ToolSet>;
}
-const interruptedCall = {
- input: { action: "authorize" },
- toolCallId: "call-auth",
- toolName: "protected_action",
- type: "tool-call" as const,
-};
-
-const siblingCall = {
- input: { action: "complete" },
- toolCallId: "call-sibling",
- toolName: "protected_action",
- type: "tool-call" as const,
-};
-
-const siblingResult = {
- input: siblingCall.input,
- output: { type: "text" as const, value: "completed" },
- toolCallId: siblingCall.toolCallId,
- toolName: siblingCall.toolName,
- type: "tool-result" as const,
-};
-
-function modelResult(result: ReturnType<typeof authorizationResult>) {
- return {
- input: result.input,
- output: { type: "text" as const, value: "Authorization required." },
- toolCallId: result.toolCallId,
- toolName: result.toolName,
- type: "tool-result" as const,
- };
-}
-
describe("resolveInlineAuthorizationInterrupt", () => {
- it("removes an interrupted call, its result, and its orphaned assistant text", () => {
- const messages: ModelMessage[] = [
- {
- content: [{ text: "I'll authorize this action.", type: "text" }, interruptedCall],
- role: "assistant",
- },
- {
- content: [modelResult(authorizationResult(interruptedCall.toolCallId, "attempt-1"))],
- role: "tool",
- },
- ];
-
- expect(
- resolveInlineAuthorizationInterrupt({
- messages,
- toolResults: [authorizationResult(interruptedCall.toolCallId, "attempt-1")],
- }),
- ).toMatchObject({
- challenges: [{ attemptId: "attempt-1" }],
- history: [],
+ it("names the calls each sign-in stopped, and ignores calls that ran", () => {
+ const authorization = authorizationResult("call-auth", "attempt-1");
+ const sibling = {
+ input: { action: "complete" },
+ output: { type: "text" as const, value: "completed" },
+ toolCallId: "call-sibling",
+ toolName: "protected_action",
+ type: "tool-result" as const,
+ };
+
+ const interrupt = resolveInlineAuthorizationInterrupt({
+ toolResults: [authorization, sibling],
});
- });
- it("preserves protocol-complete sibling calls and results", () => {
- const authorization = authorizationResult(interruptedCall.toolCallId, "attempt-1");
- const messages: ModelMessage[] = [
+ expect(interrupt?.challenges).toMatchObject([{ attemptId: "attempt-1" }]);
+ expect(interrupt?.calls).toEqual([
{
- content: [interruptedCall, siblingCall],
- role: "assistant",
+ action: {
+ callId: "call-auth",
+ input: { action: "authorize" },
+ kind: "tool-call",
+ toolName: "protected_action",
+ },
+ connections: ["protected_action"],
},
- {
- content: [modelResult(authorization), siblingResult],
- role: "tool",
- },
- ];
-
- expect(
- resolveInlineAuthorizationInterrupt({
- messages,
- toolResults: [authorization, siblingResult],
- })?.history,
- ).toEqual([
- { content: [siblingCall], role: "assistant" },
- { content: [siblingResult], role: "tool" },
]);
});
@@ -107,10 +56,7 @@ describe("resolveInlineAuthorizationInterrupt", () => {
const latest = authorizationResult("call-latest", "attempt-2");
expect(
- resolveInlineAuthorizationInterrupt({
- messages: [],
- toolResults: [first, latest],
- })?.challenges,
+ resolveInlineAuthorizationInterrupt({ toolResults: [first, latest] })?.challenges,
).toMatchObject([{ attemptId: "attempt-2" }]);
});
});
diff --git a/packages/eve/src/harness/inline-tool-authorization.ts b/packages/eve/src/harness/inline-tool-authorization.ts
index f574982ff..b372c1b13 100644
--- a/packages/eve/src/harness/inline-tool-authorization.ts
+++ b/packages/eve/src/harness/inline-tool-authorization.ts
@@ -1,4 +1,4 @@
-import type { ModelMessage, ToolSet, TypedToolResult } from "ai";
+import type { ToolSet, TypedToolResult } from "ai";
import { contextStorage } from "#context/container.js";
import {
@@ -8,6 +8,8 @@ import {
} from "#harness/authorization.js";
import { resolveActiveAuthorizationChallenges } from "#harness/hitl/sign-ins.js";
import { readToolInterrupt } from "#harness/tool-interrupts.js";
+import { createRuntimeToolCallActionFromToolCall } from "#harness/tool-call-action.js";
+import type { RuntimeToolCallActionRequest } from "#shared/action-types.js";
/** Returns whether an inline tool result represents a pending authorization interrupt. */
export function isInlineAuthorizationToolResult(toolResult: TypedToolResult<ToolSet>): boolean {
@@ -17,44 +19,42 @@ export function isInlineAuthorizationToolResult(toolResult: TypedToolResult<Tool
);
}
-/**
- * Resolves authorization interrupts and keeps only protocol-complete sibling
- * calls from the response that produced them.
- */
+/** A call a sign-in stopped, as the entry it ran, and the connections it needs. */
+export interface StoppedCall {
+ readonly action: RuntimeToolCallActionRequest;
+ readonly connections: readonly string[];
+}
+
+/** Resolves the authorization interrupts of a run's tool results. */
export function resolveInlineAuthorizationInterrupt(input: {
- readonly messages: readonly ModelMessage[];
readonly toolResults: readonly TypedToolResult<ToolSet>[] | undefined;
}):
| {
readonly challenges: AuthorizationSignal["challenges"];
- /** The calls each sign-in stopped, by the challenge's connection name. */
- readonly callIdsByName: ReadonlyMap<string, readonly string[]>;
- readonly history: ModelMessage[];
+ /** The calls sign-ins stopped, each with the connections it needs. */
+ readonly calls: readonly StoppedCall[];
}
| undefined {
const signals: AuthorizationSignal[] = [];
- const interruptedCallIds = new Set<string>();
- const callIdsByName = new Map<string, string[]>();
+ const calls: StoppedCall[] = [];
for (const toolResult of input.toolResults ?? []) {
const signal = readAuthorizationSignal(toolResult);
if (signal === undefined) continue;
signals.push(signal);
- interruptedCallIds.add(toolResult.toolCallId);
- for (const challenge of signal.challenges) {
- const callIds = callIdsByName.get(challenge.name) ?? [];
- callIdsByName.set(challenge.name, [...callIds, toolResult.toolCallId]);
- }
+ calls.push({
+ action: createRuntimeToolCallActionFromToolCall({ toolCall: toolResult }),
+ connections: signal.challenges.map((challenge) => challenge.name),
+ });
}
if (signals.length === 0) return undefined;
return {
- callIdsByName,
+ calls,
challenges: resolveActiveAuthorizationChallenges(
signals.flatMap((signal) => signal.challenges),
),
- history: withoutCalls(input.messages, interruptedCallIds),
};
}
@@ -66,33 +66,3 @@ function readAuthorizationSignal(
if (stashed !== undefined && isAuthorizationSignal(stashed)) return stashed;
return isAuthorizationSignal(toolResult.output) ? toolResult.output : undefined;
}
-
-/** Drops the given calls and their results, keeping protocol-complete sibling calls. */
-export function withoutCalls(
- messages: readonly ModelMessage[],
- interruptedCallIds: ReadonlySet<string>,
-): ModelMessage[] {
- const projected = messages.flatMap((message): ModelMessage[] => {
- if (message.role === "assistant" && Array.isArray(message.content)) {
- const interrupted = message.content.some(
- (part) => part.type === "tool-call" && interruptedCallIds.has(part.toolCallId),
- );
- const content = message.content.filter(
- (part) => part.type !== "tool-call" || !interruptedCallIds.has(part.toolCallId),
- );
- const hasSiblingCall = content.some((part) => part.type === "tool-call");
- return content.length === 0 || (interrupted && !hasSiblingCall)
- ? []
- : [{ ...message, content }];
- }
- if (message.role === "tool") {
- const content = message.content.filter(
- (part) => part.type !== "tool-result" || !interruptedCallIds.has(part.toolCallId),
- );
- return content.length === 0 ? [] : [{ ...message, content }];
- }
- return [message];
- });
-
- return projected;
-}
diff --git a/packages/eve/src/harness/session-machine/machine.test.ts b/packages/eve/src/harness/session-machine/machine.test.ts
index 8e571f7f9..457d4c127 100644
--- a/packages/eve/src/harness/session-machine/machine.test.ts
+++ b/packages/eve/src/harness/session-machine/machine.test.ts
@@ -22,17 +22,22 @@ import {
parkOnApprovals as parkOnApprovalsTransition,
requestLimit,
requireSignIn,
+ withdrawSignIns,
type ResponsePolicyPass,
} from "#harness/hitl/approvals.js";
import { deliver, resolveTypedApproval } from "#harness/hitl/delivery.js";
import { applyTransition, sessionView, type Transition } from "./commit.js";
import { saveProjection } from "./current.js";
import {
+ CANCELLED_CALL_RESULT,
cancel,
+ awaitSignIns,
clear,
completeSignIn,
hold,
+ matchSignIns,
receive,
+ runnableCalls,
relay,
settle,
suspendStep,
@@ -116,6 +121,14 @@ function callMessage(...callIds: string[]): ModelMessage {
};
}
+/** A call a sign-in for `connection` stopped. */
+function stopped(callId: string, connection: string) {
+ return {
+ action: { callId, input: { service: "api" }, kind: "tool-call" as const, toolName: "deploy" },
+ connections: [connection],
+ };
+}
+
function result(callId: string, value = "ok"): SettledCall {
return {
part: {
@@ -419,7 +432,7 @@ describe("session machine", () => {
await machine.apply(
requireSignIn(machine.view(), {
- callIdsByName: new Map([["google", ["call-1"]]]),
+ calls: [stopped("call-1", "google")],
challenges: [challenge],
}),
);
@@ -436,6 +449,105 @@ describe("session machine", () => {
]);
});
+ it("runs each call a sign-in stopped once its own sign-ins call back, and fails the rest when steered", async () => {
+ const machine = createMachine();
+ await machine.apply(receive(machine.view(), { message: "Read both." }));
+ await machine.apply(startStep(machine.view(), { modelId: "test-model" }));
+ const challenge = (name: string, attemptId: string): AuthorizationChallenge => ({
+ attemptId,
+ challenge: { instructions: `Sign in to ${name}.` },
+ hookUrl: `https://example.test/${attemptId}`,
+ name,
+ });
+ const google = challenge("google", "attempt-google");
+ const slack = challenge("slack", "attempt-slack");
+ await machine.apply(
+ suspendStep(machine.view(), {
+ event: { ...machine.position() },
+ messages: [callMessage("call-1", "call-2")],
+ tasks: [],
+ }),
+ );
+ await machine.apply(
+ requireSignIn(machine.view(), {
+ calls: [stopped("call-1", "google"), stopped("call-2", "slack")],
+ challenges: [google, slack],
+ }),
+ );
+ const waiting = () => suspendedSteps(machine.session.state)[0];
+ const runnable = () =>
+ runnableCalls(machine.view(), waiting()!).map((call) => call.action.callId);
+ expect(waiting()?.approved?.map((call) => call.action.callId)).toEqual(["call-1", "call-2"]);
+ expect(runnable()).toEqual([]);
+ expect(machine.session.history).toEqual([]);
+
+ // The first sign-in calls back: its call runs, and the call reports it, not the callback.
+ await machine.apply(matchSignIns(machine.view(), { attemptIds: ["attempt-google"] }));
+ const before = machine.events.length;
+ await machine.apply(completeSignIn(machine.view(), { completions: [google] }));
+ expect(machine.eventsSince(before)).toEqual([]);
+ expect(runnable()).toEqual(["call-1"]);
+
+ // A new message ends the second sign-in, and the call that waited for it fails with why.
+ await machine.apply(
+ withdrawSignIns(machine.view(), "Cancelled because a new message arrived."),
+ );
+ expect(machine.eventsSince(before).map((event) => event.type)).toEqual([
+ "authorization.completed",
+ "action.result",
+ ]);
+ expect(machine.projection.authorizations["attempt-slack"]?.status).toBe("declined");
+ expect(waiting()?.approved?.map((call) => call.action.callId)).toEqual(["call-1"]);
+ expect(runnable()).toEqual(["call-1"]);
+ });
+
+ it("keeps the request of an approved call that a sign-in stops again", async () => {
+ const machine = createMachine();
+ await machine.apply(receive(machine.view(), { message: "Deploy." }));
+ await machine.apply(startStep(machine.view(), { modelId: "test-model" }));
+ const call = stopped("call-1", "google");
+ const step = {
+ event: { ...machine.position() },
+ messages: [callMessage("call-1")],
+ requests: [],
+ tasks: [],
+ approved: [{ action: call.action, requestId: "approval-1" }],
+ };
+ const next = awaitSignIns(step, { stopped: [call], challenges: [] });
+ expect(next.approved?.map((approved) => approved.requestId)).toEqual(["approval-1"]);
+ });
+
+ it("cancels the call a sign-in stopped with its turn, answering it as cancelled", async () => {
+ const machine = createMachine();
+ await machine.apply(receive(machine.view(), { message: "Read my calendar." }));
+ await machine.apply(startStep(machine.view(), { modelId: "test-model" }));
+ const challenge: AuthorizationChallenge = {
+ attemptId: "attempt-1",
+ challenge: { instructions: "Sign in to Google." },
+ hookUrl: "https://example.test/callback",
+ name: "google",
+ };
+ await machine.apply(
+ suspendStep(machine.view(), {
+ event: { ...machine.position() },
+ messages: [callMessage("call-1")],
+ tasks: [],
+ }),
+ );
+ await machine.apply(
+ requireSignIn(machine.view(), {
+ calls: [stopped("call-1", "google")],
+ challenges: [challenge],
+ }),
+ );
+
+ await machine.apply(cancel(machine.view()));
+
+ expect(machine.projection.authorizations["attempt-1"]?.status).toBe("declined");
+ expect(suspendedSteps(machine.session.state)).toEqual([]);
+ expect(JSON.stringify(machine.session.history)).toContain(CANCELLED_CALL_RESULT);
+ });
+
it("retains a child's request coordinates while holding the parent turn", async () => {
const machine = createMachine();
await machine.apply(receive(machine.view(), { message: "Ask the researcher." }));
diff --git a/packages/eve/src/harness/session-machine/state.ts b/packages/eve/src/harness/session-machine/state.ts
index 3c167153c..ceb7ad686 100644
--- a/packages/eve/src/harness/session-machine/state.ts
+++ b/packages/eve/src/harness/session-machine/state.ts
@@ -1,5 +1,6 @@
import type { ModelMessage } from "ai";
+import type { AuthorizationChallenge } from "#harness/authorization.js";
import type { SessionAuthContext } from "#channel/types.js";
import type { SessionStateMap, StepInput } from "#harness/types.js";
import type { RuntimeWorkflowTaskRequest } from "#shared/action-types.js";
@@ -19,7 +20,17 @@ export interface StepCoordinates {
}
/**
- * A model step whose calls can't all settle yet: some await an approval, others the runtime.
+ * A call that runs on the turn's next model call, whatever asked for it: a person approved it, or
+ * the sign-ins it waited on called back.
+ */
+export type ApprovedCall = Pick<InputRequest, "action" | "requestId"> & {
+ /** The sign-ins the call stopped on: its outcome settles them. */
+ readonly signIns?: readonly AuthorizationChallenge[];
+};
+
+/**
+ * A model step whose calls can't all settle yet: some await an approval or a sign-in, others the
+ * runtime.
* Its response stays out of history until every call it made has a result there, so a call
* never reaches the model without its result.
*/
@@ -31,9 +42,11 @@ export interface SuspendedStep {
readonly requests: readonly InputRequest[];
/**
* Calls a person approved that haven't run. They run when the turn next calls the model, after
- * its step starts and its budget allows it, as calls the model makes do.
+ * its step starts and its budget allows it, as calls the model makes do. A call a sign-in
+ * stopped is one of them, once the sign-ins it lists have called back, so the call that asked
+ * runs, rather than the model deciding again.
*/
- readonly approved?: readonly InputRequest[];
+ readonly approved?: readonly ApprovedCall[];
/** Workflow and agent calls the runtime runs for the step. */
readonly tasks: readonly RuntimeWorkflowTaskRequest[];
readonly responseAuthRequiredRequestIds?: readonly string[];
diff --git a/packages/eve/src/harness/session-machine/transitions.ts b/packages/eve/src/harness/session-machine/transitions.ts
index 3b61a5fa4..d2843d640 100644
--- a/packages/eve/src/harness/session-machine/transitions.ts
+++ b/packages/eve/src/harness/session-machine/transitions.ts
@@ -45,14 +45,13 @@ import {
type SessionProjection,
} from "#protocol/session-projection.js";
import type { RuntimeActionResult } from "#shared/action-types.js";
-import type { InputRequest } from "#shared/input.js";
import type { JsonObject, JsonValue } from "#shared/json.js";
import type { TokenUsage } from "#shared/token-usage.js";
import type { Transition } from "./commit.js";
import { inputWithdrawn, signInWithdrawn } from "./events.js";
import { signInAttemptKey } from "#harness/hitl/sign-ins.js";
import { toProxyInputRequestEntries, type WorkflowAskRoute } from "#harness/hitl/relays.js";
-import type { StepCoordinates, SuspendedStep, TurnState } from "./state.js";
+import type { ApprovedCall, StepCoordinates, SuspendedStep, TurnState } from "./state.js";
import {
activeTurnId,
answeredCallIds,
@@ -355,9 +354,20 @@ export function withoutApproved(step: SuspendedStep, callIds: ReadonlySet<string
return approved.length === 0 ? rest : { ...rest, approved };
}
-/** Approved calls of suspended steps that haven't run. */
-export function approvedCalls(turn: TurnState): readonly InputRequest[] {
- return turn.suspended.flatMap((step) => step.approved ?? []);
+/**
+ * The approved calls of `step` that can run. A call a sign-in stopped is approved too, but waits
+ * until every sign-in it stopped on has called back.
+ */
+export function runnableCalls(view: SessionView, step: SuspendedStep): readonly ApprovedCall[] {
+ const waiting = new Set(view.signIns.map(signInAttemptKey));
+ return (step.approved ?? []).filter((call) =>
+ (call.signIns ?? []).every((challenge) => !waiting.has(signInAttemptKey(challenge))),
+ );
+}
+
+/** Approved calls of suspended steps that can run and haven't. */
+export function approvedCalls(view: SessionView): readonly ApprovedCall[] {
+ return view.turn.suspended.flatMap((step) => runnableCalls(view, step));
}
export function suspend(turn: TurnState, step: SuspendedStep): TurnState {
@@ -406,6 +416,20 @@ export function withResult(
return next;
}
+/** The messages without the results of the calls in `callIds`, which stay as unanswered calls. */
+function withoutResults(
+ messages: readonly ModelMessage[],
+ callIds: ReadonlySet<string>,
+): ModelMessage[] {
+ return messages.flatMap((message): ModelMessage[] => {
+ if (message.role !== "tool") return [message];
+ const content = message.content.filter(
+ (part) => part.type !== "tool-result" || !callIds.has(part.toolCallId),
+ );
+ return content.length === 0 ? [] : [{ ...message, content }];
+ });
+}
+
/** The call with this id, and the index of the message that made it. */
function findCall(
messages: readonly ModelMessage[],
@@ -637,7 +661,8 @@ export function runSignIn(
/**
* Sign-in callbacks matched the attempts `attemptIds` name: the session stops waiting on them.
- * Their completions report once the turn they resume runs (`completeSignIn`).
+ * A call they stopped runs once every sign-in it waits on has called back (`runnableCalls`); any
+ * other completion reports once the turn it resumes runs (`completeSignIn`).
*/
export function matchSignIns(
view: SessionView,
@@ -650,27 +675,79 @@ export function matchSignIns(
};
}
+/** The sign-ins calls stopped on, which the calls report when they settle. */
+export function callSignIns(turn: TurnState): readonly AuthorizationChallenge[] {
+ return turn.suspended.flatMap((step) =>
+ (step.approved ?? []).flatMap((call) => call.signIns ?? []),
+ );
+}
+
+/**
+ * The `stopped` calls wait for a sign-in: their results leave the step, and they stay approved,
+ * to run once the sign-ins call back, rather than the model asking for them again.
+ */
+export function awaitSignIns(
+ step: SuspendedStep,
+ input: {
+ readonly stopped: readonly {
+ readonly action: ApprovedCall["action"];
+ readonly connections: readonly string[];
+ }[];
+ readonly challenges: readonly AuthorizationChallenge[];
+ },
+): SuspendedStep {
+ const ids = stepCallIds(step);
+ const stopped = input.stopped.filter(({ action }) => ids.has(action.callId));
+ if (stopped.length === 0) return step;
+ const calls = stopped.map(({ action, connections }): ApprovedCall => {
+ // A call approved before keeps its request, and the sign-ins it already waited on report with
+ // it, beside the ones it waits on now.
+ const earlier = step.approved?.find((approved) => approved.action.callId === action.callId);
+ return {
+ action,
+ requestId: earlier?.requestId ?? action.callId,
+ signIns: [
+ ...(earlier?.signIns ?? []),
+ ...input.challenges.filter((challenge) => connections.includes(challenge.name)),
+ ],
+ };
+ });
+ const callIds = new Set(calls.map((call) => call.action.callId));
+ return {
+ ...step,
+ messages: withoutResults(step.messages, callIds),
+ approved: [
+ ...(step.approved ?? []).filter((call) => !callIds.has(call.action.callId)),
+ ...calls,
+ ],
+ };
+}
+
/** Sign-in callbacks arrived: each completion is reported at the coordinates of the turn that asked. */
export function completeSignIn(
view: SessionView,
input: { readonly completions: readonly AuthorizationChallenge[] },
): Transition {
const position = at(view.projection);
+ // A call a sign-in stopped reports it when the call settles.
+ const reportedByCalls = new Set(callSignIns(view.turn).map(signInAttemptKey));
return unchanged(
view,
- input.completions.map((challenge) => {
- const attempt =
- challenge.attemptId === undefined
- ? undefined
- : view.projection.authorizations[challenge.attemptId];
- return createAuthorizationCompletedEvent({
- ...authorizationEventFields(challenge),
- outcome: "authorized",
- sequence: attempt?.sequence ?? position.sequence,
- stepIndex: attempt?.stepIndex ?? position.stepIndex,
- turnId: attempt?.turnId ?? position.turnId,
- });
- }),
+ input.completions
+ .filter((challenge) => !reportedByCalls.has(signInAttemptKey(challenge)))
+ .map((challenge) => {
+ const attempt =
+ challenge.attemptId === undefined
+ ? undefined
+ : view.projection.authorizations[challenge.attemptId];
+ return createAuthorizationCompletedEvent({
+ ...authorizationEventFields(challenge),
+ outcome: "authorized",
+ sequence: attempt?.sequence ?? position.sequence,
+ stepIndex: attempt?.stepIndex ?? position.stepIndex,
+ turnId: attempt?.turnId ?? position.turnId,
+ });
+ }),
);
}
diff --git a/packages/eve/src/harness/step/after-model.ts b/packages/eve/src/harness/step/after-model.ts
index 69ce18296..a4e15bafa 100644
--- a/packages/eve/src/harness/step/after-model.ts
+++ b/packages/eve/src/harness/step/after-model.ts
@@ -140,13 +140,12 @@ export async function handleStepResult(step: Step, input: ModelResponse): Promis
.filter((toolCall) => isWorkflowTool(input.catalog.get(toolCall.toolName)));
const authorizationInterrupt = resolveInlineAuthorizationInterrupt({
- messages: [...promptMessages, ...responseMessages],
toolResults: result.toolResults,
});
// --- Park on approvals or runtime calls ----------------------------------
- if (workflowToolCalls.length > 0 || approvalRequests.length > 0) {
+ if (workflowToolCalls.length > 0 || approvalRequests.length > 0 || authorizationInterrupt) {
const { sequence, stepIndex, turnId } = step.position();
const dispatched = collectWorkflowCalls({
session: step.session,
@@ -174,31 +173,17 @@ export async function handleStepResult(step: Step, input: ModelResponse): Promis
: undefined,
});
// Approval only precedes sign-in for the same call, not an executing sibling.
- if (authorizationInterrupt) {
- return stopForToolSignIn(step, {
- ...authorizationInterrupt,
- history: step.session.history,
- });
- }
+ if (authorizationInterrupt) return stopForToolSignIn(step, authorizationInterrupt);
return parkedResult;
}
await step.apply(suspendStep(step.view(), parked));
- if (authorizationInterrupt) {
- return stopForToolSignIn(step, {
- ...authorizationInterrupt,
- history: step.session.history,
- });
- }
+ if (authorizationInterrupt) return stopForToolSignIn(step, authorizationInterrupt);
if (dispatched.workflowRequests.some((task) => task.entry.entryPoint === "execute")) {
await step.apply(hold(step.view(), { on: "tasks" }));
}
return { next: null, session: step.session };
}
- // --- Park on authorization request ------------------------------------------
-
- if (authorizationInterrupt) return stopForToolSignIn(step, authorizationInterrupt);
-
// --- Continue or terminate ------------------------------------------------
// History grows by append only; nothing rewrites earlier messages mid-turn,
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index 07797de30..db4bee450 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -62,7 +62,7 @@ import { applyTransition, sessionView } from "#harness/session-machine/commit.js
import { requireSignIn } from "#harness/hitl/approvals.js";
import { createAuthorizationRequiredEvent } from "#protocol/message.js";
import { finishTurn, ownOpenRequestIds } from "#harness/session-machine/transitions.js";
-import { runtimeWait, storedProjection } from "#harness/session-machine/view.js";
+import { answeredCallIds, runtimeWait, storedProjection } from "#harness/session-machine/view.js";
import {
foldingHandler,
parkedSteps,
@@ -6459,17 +6459,13 @@ describe("createToolLoopHarness", () => {
const parked = parkedSteps(result.session);
expect(parked).toHaveLength(1);
expect(parked[0]?.requests).toEqual([]);
- expect(parked[0]?.messages).toEqual([{ content: [delegateCall], role: "assistant" }]);
- for (const message of result.session.history) {
- if (!Array.isArray(message.content)) continue;
- expect(
- message.content.filter(
- (part) =>
- (part.type === "tool-call" || part.type === "tool-result") &&
- part.toolCallId === signInCall.toolCallId,
- ),
- ).toHaveLength(0);
- }
+ // The call the sign-in stopped waits in the step, unanswered, to run once it completes.
+ expect(parked[0]?.messages).toEqual([
+ { content: [delegateCall, signInCall], role: "assistant" },
+ ]);
+ expect(parked[0]?.approved?.map((call) => call.action.callId)).toEqual([
+ signInCall.toolCallId,
+ ]);
expect(ToolLoopAgent).toHaveBeenCalledTimes(1);
});
@@ -6576,42 +6572,11 @@ describe("createToolLoopHarness", () => {
// A sign-in interrupt is not a tool result on the v26 stream.
const actionResults = events.filter((event) => event.type === "action.result");
expect(actionResults).toHaveLength(0);
- expect(
- result.session.history.flatMap((message) =>
- message.role === "tool"
- ? message.content.filter(
- (part) => part.type === "tool-result" && part.toolCallId === "call-1",
- )
- : [],
- ),
- ).toHaveLength(0);
- expect(
- result.session.history.flatMap((message) =>
- message.role === "assistant" && Array.isArray(message.content)
- ? message.content.filter(
- (part) => part.type === "tool-call" && part.toolCallId === "call-1",
- )
- : [],
- ),
- ).toHaveLength(0);
- expect(
- result.session.history.flatMap((message) =>
- message.role === "tool"
- ? message.content.filter(
- (part) => part.type === "tool-result" && part.toolCallId === "call-2",
- )
- : [],
- ),
- ).toHaveLength(1);
- expect(
- result.session.history.flatMap((message) =>
- message.role === "assistant" && Array.isArray(message.content)
- ? message.content.filter(
- (part) => part.type === "tool-call" && part.toolCallId === "call-2",
- )
- : [],
- ),
- ).toHaveLength(1);
+ // The step waits whole: the sign-in's call has no result, so nothing reaches history yet.
+ const [parked] = parkedSteps(result.session);
+ expect(parked?.approved?.map((call) => call.action.callId)).toEqual(["call-1"]);
+ expect(answeredCallIds(parked?.messages ?? [])).toEqual(new Set(["call-2"]));
+ expect(JSON.stringify(result.session.history)).not.toContain("call-");
});
});