The Open Framework for Building Agents
fix(e2e): agent-tools - replace flaky timing assertions (#981)
Andrew Barba0ce73f6f38d1
diff --git a/e2e/fixtures/agent-tools/agent/tools/fanout-barrier.ts b/e2e/fixtures/agent-tools/agent/tools/fanout-barrier.tsnew file mode 100644index 000000000..b439ab2b4--- /dev/null+++ b/e2e/fixtures/agent-tools/agent/tools/fanout-barrier.ts@@ -0,0 +1,75 @@+import { defineTool } from "eve/tools";+import { z } from "zod";++const EXPECTED_CONCURRENT_CALLS = 10;+const BARRIER_TIMEOUT_MS = 15_000;++interface Barrier {+ arrived: number;+ departed: number;+ readonly released: Promise<void>;+ readonly release: () => void;+}++let activeBarrier: Barrier | undefined;++export default defineTool({+ description:+ "Test-only tool: waits for ten concurrent calls before releasing them. Only call when the user explicitly asks to use `fanout-barrier`.",+ inputSchema: z.object({+ label: z.string(),+ }),+ async execute(input) {+ const barrier = joinBarrier();++ try {+ await waitForRelease(barrier.released);+ return { concurrentCallsAtRelease: barrier.arrived, label: input.label };+ } finally {+ barrier.departed += 1;+ if (barrier.departed === barrier.arrived && activeBarrier === barrier) {+ activeBarrier = undefined;+ }+ }+ },+});++function joinBarrier(): Barrier {+ const barrier = activeBarrier ?? createBarrier();+ activeBarrier = barrier;++ if (barrier.arrived >= EXPECTED_CONCURRENT_CALLS) {+ throw new Error("fanout-barrier received more than ten concurrent calls");+ }++ barrier.arrived += 1;+ if (barrier.arrived === EXPECTED_CONCURRENT_CALLS) {+ barrier.release();+ }+ return barrier;+}++function createBarrier(): Barrier {+ let release!: () => void;+ const released = new Promise<void>((resolve) => {+ release = resolve;+ });+ return { arrived: 0, departed: 0, release, released };+}++async function waitForRelease(released: Promise<void>): Promise<void> {+ let timer: ReturnType<typeof setTimeout> | undefined;++ try {+ await Promise.race([+ released,+ new Promise<never>((_resolve, reject) => {+ timer = setTimeout(() => {+ reject(new Error("fanout-barrier timed out waiting for ten concurrent calls"));+ }, BARRIER_TIMEOUT_MS);+ }),+ ]);+ } finally {+ if (timer !== undefined) clearTimeout(timer);+ }+}diff --git a/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.tsindex 1a113391b..c6f256fa0 100644--- a/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts+++ b/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts@@ -1,13 +1,13 @@import { defineEval } from "eve/evals";import {- authoredFanoutExecutionsOverlap,FANOUT_SIZE,+ fanoutExecutionsReachBarrier,fanoutRequestsPrecedeFirstResult,fanoutRequestsUseExpectedLabels,} from "./fanout";-const TOOL_NAME = "streamed-action";+const TOOL_NAME = "fanout-barrier";const LABELS = ["fanout-authored-01","fanout-authored-02",@@ -43,8 +43,8 @@ export default defineEval({turn.eventsSatisfy("ten authored requests use their distinct labels", (events) =>fanoutRequestsUseExpectedLabels({ events, labels: LABELS, toolName: TOOL_NAME }),);- turn.eventsSatisfy("ten authored executions overlap", (events) =>- authoredFanoutExecutionsOverlap({ events, toolName: TOOL_NAME }),+ turn.eventsSatisfy("ten authored executions reach the concurrency barrier", (events) =>+ fanoutExecutionsReachBarrier({ events, toolName: TOOL_NAME }),);},});diff --git a/e2e/fixtures/agent-tools/evals/static-tools/fanout.ts b/e2e/fixtures/agent-tools/evals/static-tools/fanout.tsindex e6cc89953..84680cbaa 100644--- a/e2e/fixtures/agent-tools/evals/static-tools/fanout.ts+++ b/e2e/fixtures/agent-tools/evals/static-tools/fanout.ts@@ -51,33 +51,22 @@ export function fanoutRequestsUseExpectedLabels(input: {);}-/**- * The fixture tool holds each invocation open. Requiring every interval to- * overlap rules out a serialized executor without relying on a clock-skew- * threshold.- */-export function authoredFanoutExecutionsOverlap(input: {+/** A serialized executor cannot release the fixture tool's concurrency barrier. */+export function fanoutExecutionsReachBarrier(input: {readonly events: readonly HandleMessageStreamEvent[];readonly toolName: string;}): boolean {- const intervals: Array<{ readonly completedAt: number; readonly startedAt: number }> = [];+ const concurrentCallsAtRelease = input.events.flatMap((event) => {+ if (event.type !== "action.result" || event.data.result.kind !== "tool-result") return [];+ if (event.data.result.toolName !== input.toolName) return [];- for (const event of input.events) {- if (event.type !== "action.result" || event.data.result.kind !== "tool-result") continue;- if (event.data.result.toolName !== input.toolName) continue;-- const startedAt = readFiniteNumberField(event.data.result.output, "executionStartedAt");- const completedAt = readFiniteNumberField(event.data.result.output, "executionCompletedAt");- if (startedAt === undefined || completedAt === undefined || startedAt >= completedAt) {- return false;- }- intervals.push({ completedAt, startedAt });- }+ const count = readFiniteNumberField(event.data.result.output, "concurrentCallsAtRelease");+ return count === undefined ? [] : [count];+ });return (- intervals.length === FANOUT_SIZE &&- Math.max(...intervals.map((interval) => interval.startedAt)) <- Math.min(...intervals.map((interval) => interval.completedAt))+ concurrentCallsAtRelease.length === FANOUT_SIZE &&+ concurrentCallsAtRelease.every((count) => count === FANOUT_SIZE));}diff --git a/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.tsindex cfa5879bd..5664fb88e 100644--- a/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts+++ b/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts@@ -4,7 +4,9 @@ import { defineEval } from "eve/evals";const TOOL_NAME = "streamed-action";const LABEL = "streaming-e2e";-function streamedBeforeLocalExecution(events: readonly HandleMessageStreamEvent[]): boolean {+function streamedBeforeLocalExecutionCompletes(+ events: readonly HandleMessageStreamEvent[],+): boolean {const matchingRequests = events.flatMap((event) => {if (event.type !== "actions.requested") return [];@@ -32,9 +34,11 @@ function streamedBeforeLocalExecution(events: readonly HandleMessageStreamEvent[}const requestAt = parseTimestamp(request.event.meta?.at);- const executionStartedAt = readExecutionStartedAt(result.data.result.output);+ const executionCompletedAt = readExecutionCompletedAt(result.data.result.output);return (- requestAt !== undefined && executionStartedAt !== undefined && requestAt <= executionStartedAt+ requestAt !== undefined &&+ executionCompletedAt !== undefined &&+ requestAt < executionCompletedAt);}@@ -45,27 +49,27 @@ function parseTimestamp(value: string | undefined): number | undefined {return Number.isFinite(timestamp) ? timestamp : undefined;}-function readExecutionStartedAt(output: unknown): number | undefined {+function readExecutionCompletedAt(output: unknown): number | undefined {if (typeof output !== "object" ||output === null ||Array.isArray(output) ||- !("executionStartedAt" in output)+ !("executionCompletedAt" in output)) {return undefined;}- const executionStartedAt = output.executionStartedAt;- return typeof executionStartedAt === "number" && Number.isFinite(executionStartedAt)- ? executionStartedAt+ const executionCompletedAt = output.executionCompletedAt;+ return typeof executionCompletedAt === "number" && Number.isFinite(executionCompletedAt)+ ? executionCompletedAt: undefined;}-// The runtime stamps meta.at immediately before persisting an event. The tool-// records its start before waiting, so post-execution batch emission cannot-// satisfy this relation.+// The AI SDK can begin local execution just before its tool-call stream part is+// consumed. The tool waits before completing, so post-execution batch emission+// still cannot satisfy this relation.export default defineEval({- description: "Static tools smoke: a local action request streams before execution begins.",+ description: "Static tools smoke: a local action request streams while execution is pending.",async test(t) {const turn = await t.send(`Call the \`${TOOL_NAME}\` tool exactly once with label "${LABEL}". ` +@@ -78,6 +82,9 @@ export default defineEval({input: { label: LABEL },count: 1,});- turn.eventsSatisfy("local action request precedes execution", streamedBeforeLocalExecution);+ turn.eventsSatisfy(+ "local action request precedes execution completion",+ streamedBeforeLocalExecutionCompletes,+ );},});