Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

test(eve): reproduce authorization.completed reporting authorized before completeAuthorization runs (#4761)

roboeve[bot]bd26a3f160aa
diff --git a/e2e/fixtures/agent-workflow-tools/evals/public-catalog.sign-in-runs-call.eval.ts b/e2e/fixtures/agent-workflow-tools/evals/public-catalog.sign-in-runs-call.eval.ts
new file mode 100644
index 000000000..7d7d0189b
--- /dev/null
+++ b/e2e/fixtures/agent-workflow-tools/evals/public-catalog.sign-in-runs-call.eval.ts
@@ -0,0 +1,75 @@
+import { defineEval } from "eve/evals";
+
+import { fixtureAuthorizationCallback } from "../agent/lib/fake-service.ts";
+
+const PROTECTED_TOOL = "public-catalog__list_orders";
+
+/** Why this eval is skipped: it fails until a session-path sign-in runs its call itself. */
+const KNOWN_RED =
+ "Fails today: after sign-in the session path drops the waiting call and asks the model again. Remove this skip with the change that runs the call itself.";
+
+export default defineEval({
+ description:
+ "After the user signs in for a protected MCP tool, the call that was waiting runs without asking the model again: no model call between authorization.completed and the call's result, and the call is not requested twice.",
+ timeoutMs: 90_000,
+
+ async test(t) {
+ t.skip(KNOWN_RED);
+
+ // A fresh principal per run: the fixture provider remembers each user's token.
+ const alice = { "x-eve-forwarded-principal-id": `sign-in-alice-${crypto.randomUUID()}` };
+ const started = await t.send(
+ "PUBLIC-CATALOG-E2E Alice wants her public-catalog items and orders.",
+ { headers: alice },
+ );
+ const session = started.session;
+ started.expectOk();
+ started.event("authorization.required", { count: 1 });
+ started.event("turn.waiting", { count: 1 });
+
+ const required = started.events.find((event) => event.type === "authorization.required");
+ if (required?.type !== "authorization.required") {
+ throw new Error("The protected tool did not produce an authorization challenge.");
+ }
+ if (session.sessionId === undefined || session.state === undefined) {
+ throw new Error("The turn did not create a session.");
+ }
+ const resumed = t.target.watchTurn(session.sessionId, {
+ startIndex: session.state.streamIndex,
+ });
+ const callback = fixtureAuthorizationCallback(t.target.url, required.data.authorization?.url);
+ const response = await t.target.fetch(`${callback.pathname}${callback.search}`);
+ if (!response.ok) throw new Error(`Authorization callback failed (${response.status}).`);
+
+ const completed = await resumed.result();
+ completed.expectOk();
+ completed.noFailedActions();
+ completed.event("authorization.completed", { count: 1, data: { outcome: "authorized" } });
+ completed.calledTool(PROTECTED_TOOL, { count: 1 });
+ completed.eventsSatisfy(
+ "no model call between authorization.completed and the tool call's action.result",
+ (events) => {
+ const signedIn = events.findIndex((event) => event.type === "authorization.completed");
+ const result = events.findIndex((event) => event.type === "action.result");
+ return (
+ signedIn >= 0 &&
+ result > signedIn &&
+ !events.slice(signedIn, result).some((event) => event.type === "step.started")
+ );
+ },
+ );
+ completed.eventsSatisfy(
+ "the call that waited for sign-in is not requested again by the model",
+ (events) =>
+ !events.some(
+ (event) =>
+ event.type === "actions.requested" &&
+ event.data.actions.some((action) => JSON.stringify(action).includes(PROTECTED_TOOL)),
+ ),
+ );
+ completed.eventsSatisfy(
+ "one model call after sign-in, the one that reads the result",
+ (events) => events.filter((event) => event.type === "step.started").length === 1,
+ );
+ },
+});
diff --git a/packages/eve/src/execution/session/entry-authorization.integration.test.ts b/packages/eve/src/execution/session/entry-authorization.integration.test.ts
index b07372bbf..a687bccf0 100644
--- a/packages/eve/src/execution/session/entry-authorization.integration.test.ts
+++ b/packages/eve/src/execution/session/entry-authorization.integration.test.ts
@@ -18,7 +18,10 @@ import {
} from "#execution/session-inbox/address.js";
import { createToolExecuteWithAuth } from "#execution/tool-auth.js";
import { ROOT_COMPILED_AGENT_NODE_ID } from "#compiler/manifest.js";
-import { ConnectionAuthorizationRequiredError } from "#connections/errors.js";
+import {
+ ConnectionAuthorizationFailedError,
+ ConnectionAuthorizationRequiredError,
+} from "#connections/errors.js";
import type { MessageStreamEvent } from "#protocol/message.js";
import type { ToolContext } from "#tools/definition.js";
import type {
@@ -258,6 +261,103 @@ describe("workflowEntry integration", () => {
});
});
+ it.fails("reports a sign-in failed when completeAuthorization rejects its callback", async () => {
+ const { completeCalls, runtime } = await createWeatherAuthRuntime("workflow-entry-auth-denied");
+ const continuationToken = "http:workflow-entry-auth-denied";
+
+ await runtime.run(async () => {
+ const run = await startWeatherRun(continuationToken);
+ const stream = captureEvents(run);
+
+ try {
+ const firstTurn = await stream.nextUntil(
+ "initial auth-required event",
+ (event) => event.type === "authorization.required",
+ );
+ await stream.nextUntil("authorization hold", (event) => event.type === "turn.waiting");
+
+ // The provider redirects back with a denial, which completeAuthorization rejects.
+ await resumeHook(sessionInboxHookToken(sessionCommandHookToken(run.runId)), {
+ kind: "authorization-callback",
+ payloads: [
+ {
+ authorizationCallback: {
+ attemptId: authorizationAttemptId(firstTurn),
+ callback: { method: "GET", params: { error: "access_denied" } },
+ connectionName: "weather",
+ },
+ },
+ ],
+ });
+
+ const resumed = await stream.nextUntil(
+ "resumed held turn",
+ (event) => event.type === "session.waiting",
+ );
+ expect(completeCalls()).toBe(1);
+ expect(filterEventsByType(resumed, "authorization.completed")).toMatchObject([
+ { data: { name: "weather", outcome: "failed", reason: "access_denied" } },
+ ]);
+ } finally {
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ });
+
+ it.fails("runs the call a sign-in was for without asking the model again", async () => {
+ const { executions, runtime } = await createWeatherAuthRuntime("workflow-entry-auth-no-reask");
+ const continuationToken = "http:workflow-entry-auth-no-reask";
+
+ await runtime.run(async () => {
+ const run = await startWeatherRun(continuationToken);
+ const stream = captureEvents(run);
+
+ try {
+ const beforeSignIn = await stream.nextUntil(
+ "initial auth-required event",
+ (event) => event.type === "authorization.required",
+ );
+ await stream.nextUntil("authorization hold", (event) => event.type === "turn.waiting");
+ await resumeHook(sessionInboxHookToken(sessionCommandHookToken(run.runId)), {
+ kind: "authorization-callback",
+ payloads: [weatherCallback(authorizationAttemptId(beforeSignIn))],
+ });
+ const afterSignIn = await stream.nextUntil(
+ "turn after sign-in",
+ (event) => event.type === "session.waiting",
+ );
+
+ // The call that stopped for sign-in is the call that runs: the model is
+ // not asked to decide again between the sign-in and that call's result.
+ const completedAt = afterSignIn.findIndex(
+ (event) => event.type === "authorization.completed",
+ );
+ const resultAt = afterSignIn.findIndex((event) => event.type === "action.result");
+ const modelCallsBetween = afterSignIn
+ .slice(completedAt, resultAt)
+ .filter((event) => event.type === "step.started");
+ expect(
+ modelCallsBetween,
+ "model calls between authorization.completed and the tool call's action.result",
+ ).toHaveLength(0);
+
+ // One model call asks for the tool, one reads its result. Today a third
+ // call sits between the sign-in and the tool running.
+ const modelCalls = [...beforeSignIn, ...afterSignIn].filter(
+ (event) => event.type === "step.started",
+ );
+ expect(modelCalls, "model calls in the turn").toHaveLength(2);
+
+ // The tool runs once, with the arguments of the original call.
+ expect(executions()).toEqual([{ city: "Lisbon" }]);
+ } finally {
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ });
+
it("queues another person's message behind a held sign-in", async () => {
const { completeCalls, completedPrincipals, runtime } = await createWeatherAuthRuntime(
"workflow-entry-auth-queued",
@@ -456,6 +556,8 @@ describe("workflowEntry integration", () => {
});
interface WeatherAuthRuntime {
+ /** The input of each get_weather call that got its token and ran. */
+ executions(): readonly { city: string }[];
completeCalls(): number;
completedPrincipals(): readonly ConnectionPrincipal[];
runtime: Awaited<ReturnType<typeof createTestRuntime>>;
@@ -464,8 +566,8 @@ interface WeatherAuthRuntime {
/**
* A get_weather tool behind an interactive authorization: getToken always
* requires sign-in, and completeAuthorization mints `weather-token` from the
- * `oauth-code` callback. Shared by the callback-resume and
- * challenge-stays-open owner tests.
+ * `oauth-code` callback and rejects an `access_denied` one. Shared by the
+ * callback-resume and challenge-stays-open owner tests.
*/
async function createWeatherAuthRuntime(
agentName: string,
@@ -473,6 +575,7 @@ async function createWeatherAuthRuntime(
): Promise<WeatherAuthRuntime> {
let completeCalls = 0;
const completedPrincipals: ConnectionPrincipal[] = [];
+ const executions: { city: string }[] = [];
const weatherAuth: AuthorizationDefinition<{ nonce: string }> = {
principalType: "user",
async getToken(): Promise<TokenResult> {
@@ -491,6 +594,12 @@ async function createWeatherAuthRuntime(
async completeAuthorization({ callback, principal, resume }): Promise<TokenResult> {
completeCalls += 1;
completedPrincipals.push(principal);
+ if (callback.params.error === "access_denied") {
+ throw new ConnectionAuthorizationFailedError("weather", {
+ reason: "access_denied",
+ retryable: false,
+ });
+ }
expect(callback.params.code).toBe("oauth-code");
expect(resume).toEqual({ nonce: "weather-nonce" });
return { token: "weather-token" };
@@ -512,6 +621,7 @@ async function createWeatherAuthRuntime(
typeof (rawInput as { city?: unknown }).city === "string"
? (rawInput as { city: string }).city
: "Lisbon";
+ executions.push({ city });
return {
city,
condition: "Sunny",
@@ -551,6 +661,7 @@ async function createWeatherAuthRuntime(
return {
completeCalls: () => completeCalls,
completedPrincipals: () => completedPrincipals,
+ executions: () => executions,
runtime,
};
}