Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(e2e): agent-tools - replace flaky timing assertions (#981)

Andrew Barba0ce73f6f38d1
diff --git a/e2e/fixtures/agent-tools/agent/tools/fanout-barrier.ts b/e2e/fixtures/agent-tools/agent/tools/fanout-barrier.ts
new file mode 100644
index 000000000..b439ab2b4
--- /dev/null
+++ b/e2e/fixtures/agent-tools/agent/tools/fanout-barrier.ts
@@ -0,0 +1,75 @@
+import { defineTool } from "eve/tools";
+import { z } from "zod";
+
+const EXPECTED_CONCURRENT_CALLS = 10;
+const BARRIER_TIMEOUT_MS = 15_000;
+
+interface Barrier {
+ arrived: number;
+ departed: number;
+ readonly released: Promise<void>;
+ readonly release: () => void;
+}
+
+let activeBarrier: Barrier | undefined;
+
+export default defineTool({
+ description:
+ "Test-only tool: waits for ten concurrent calls before releasing them. Only call when the user explicitly asks to use `fanout-barrier`.",
+ inputSchema: z.object({
+ label: z.string(),
+ }),
+ async execute(input) {
+ const barrier = joinBarrier();
+
+ try {
+ await waitForRelease(barrier.released);
+ return { concurrentCallsAtRelease: barrier.arrived, label: input.label };
+ } finally {
+ barrier.departed += 1;
+ if (barrier.departed === barrier.arrived && activeBarrier === barrier) {
+ activeBarrier = undefined;
+ }
+ }
+ },
+});
+
+function joinBarrier(): Barrier {
+ const barrier = activeBarrier ?? createBarrier();
+ activeBarrier = barrier;
+
+ if (barrier.arrived >= EXPECTED_CONCURRENT_CALLS) {
+ throw new Error("fanout-barrier received more than ten concurrent calls");
+ }
+
+ barrier.arrived += 1;
+ if (barrier.arrived === EXPECTED_CONCURRENT_CALLS) {
+ barrier.release();
+ }
+ return barrier;
+}
+
+function createBarrier(): Barrier {
+ let release!: () => void;
+ const released = new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ return { arrived: 0, departed: 0, release, released };
+}
+
+async function waitForRelease(released: Promise<void>): Promise<void> {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+
+ try {
+ await Promise.race([
+ released,
+ new Promise<never>((_resolve, reject) => {
+ timer = setTimeout(() => {
+ reject(new Error("fanout-barrier timed out waiting for ten concurrent calls"));
+ }, BARRIER_TIMEOUT_MS);
+ }),
+ ]);
+ } finally {
+ if (timer !== undefined) clearTimeout(timer);
+ }
+}
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts
index 1a113391b..c6f256fa0 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/fanout-authored.eval.ts
@@ -1,13 +1,13 @@
import { defineEval } from "eve/evals";
import {
- authoredFanoutExecutionsOverlap,
FANOUT_SIZE,
+ fanoutExecutionsReachBarrier,
fanoutRequestsPrecedeFirstResult,
fanoutRequestsUseExpectedLabels,
} from "./fanout";
-const TOOL_NAME = "streamed-action";
+const TOOL_NAME = "fanout-barrier";
const LABELS = [
"fanout-authored-01",
"fanout-authored-02",
@@ -43,8 +43,8 @@ export default defineEval({
turn.eventsSatisfy("ten authored requests use their distinct labels", (events) =>
fanoutRequestsUseExpectedLabels({ events, labels: LABELS, toolName: TOOL_NAME }),
);
- turn.eventsSatisfy("ten authored executions overlap", (events) =>
- authoredFanoutExecutionsOverlap({ events, toolName: TOOL_NAME }),
+ turn.eventsSatisfy("ten authored executions reach the concurrency barrier", (events) =>
+ fanoutExecutionsReachBarrier({ events, toolName: TOOL_NAME }),
);
},
});
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/fanout.ts b/e2e/fixtures/agent-tools/evals/static-tools/fanout.ts
index e6cc89953..84680cbaa 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/fanout.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/fanout.ts
@@ -51,33 +51,22 @@ export function fanoutRequestsUseExpectedLabels(input: {
);
}
-/**
- * The fixture tool holds each invocation open. Requiring every interval to
- * overlap rules out a serialized executor without relying on a clock-skew
- * threshold.
- */
-export function authoredFanoutExecutionsOverlap(input: {
+/** A serialized executor cannot release the fixture tool's concurrency barrier. */
+export function fanoutExecutionsReachBarrier(input: {
readonly events: readonly HandleMessageStreamEvent[];
readonly toolName: string;
}): boolean {
- const intervals: Array<{ readonly completedAt: number; readonly startedAt: number }> = [];
+ const concurrentCallsAtRelease = input.events.flatMap((event) => {
+ if (event.type !== "action.result" || event.data.result.kind !== "tool-result") return [];
+ if (event.data.result.toolName !== input.toolName) return [];
- for (const event of input.events) {
- if (event.type !== "action.result" || event.data.result.kind !== "tool-result") continue;
- if (event.data.result.toolName !== input.toolName) continue;
-
- const startedAt = readFiniteNumberField(event.data.result.output, "executionStartedAt");
- const completedAt = readFiniteNumberField(event.data.result.output, "executionCompletedAt");
- if (startedAt === undefined || completedAt === undefined || startedAt >= completedAt) {
- return false;
- }
- intervals.push({ completedAt, startedAt });
- }
+ const count = readFiniteNumberField(event.data.result.output, "concurrentCallsAtRelease");
+ return count === undefined ? [] : [count];
+ });
return (
- intervals.length === FANOUT_SIZE &&
- Math.max(...intervals.map((interval) => interval.startedAt)) <
- Math.min(...intervals.map((interval) => interval.completedAt))
+ concurrentCallsAtRelease.length === FANOUT_SIZE &&
+ concurrentCallsAtRelease.every((count) => count === FANOUT_SIZE)
);
}
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts
index cfa5879bd..5664fb88e 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/streamed-action.eval.ts
@@ -4,7 +4,9 @@ import { defineEval } from "eve/evals";
const TOOL_NAME = "streamed-action";
const LABEL = "streaming-e2e";
-function streamedBeforeLocalExecution(events: readonly HandleMessageStreamEvent[]): boolean {
+function streamedBeforeLocalExecutionCompletes(
+ events: readonly HandleMessageStreamEvent[],
+): boolean {
const matchingRequests = events.flatMap((event) => {
if (event.type !== "actions.requested") return [];
@@ -32,9 +34,11 @@ function streamedBeforeLocalExecution(events: readonly HandleMessageStreamEvent[
}
const requestAt = parseTimestamp(request.event.meta?.at);
- const executionStartedAt = readExecutionStartedAt(result.data.result.output);
+ const executionCompletedAt = readExecutionCompletedAt(result.data.result.output);
return (
- requestAt !== undefined && executionStartedAt !== undefined && requestAt <= executionStartedAt
+ requestAt !== undefined &&
+ executionCompletedAt !== undefined &&
+ requestAt < executionCompletedAt
);
}
@@ -45,27 +49,27 @@ function parseTimestamp(value: string | undefined): number | undefined {
return Number.isFinite(timestamp) ? timestamp : undefined;
}
-function readExecutionStartedAt(output: unknown): number | undefined {
+function readExecutionCompletedAt(output: unknown): number | undefined {
if (
typeof output !== "object" ||
output === null ||
Array.isArray(output) ||
- !("executionStartedAt" in output)
+ !("executionCompletedAt" in output)
) {
return undefined;
}
- const executionStartedAt = output.executionStartedAt;
- return typeof executionStartedAt === "number" && Number.isFinite(executionStartedAt)
- ? executionStartedAt
+ const executionCompletedAt = output.executionCompletedAt;
+ return typeof executionCompletedAt === "number" && Number.isFinite(executionCompletedAt)
+ ? executionCompletedAt
: undefined;
}
-// The runtime stamps meta.at immediately before persisting an event. The tool
-// records its start before waiting, so post-execution batch emission cannot
-// satisfy this relation.
+// The AI SDK can begin local execution just before its tool-call stream part is
+// consumed. The tool waits before completing, so post-execution batch emission
+// still cannot satisfy this relation.
export default defineEval({
- description: "Static tools smoke: a local action request streams before execution begins.",
+ description: "Static tools smoke: a local action request streams while execution is pending.",
async test(t) {
const turn = await t.send(
`Call the \`${TOOL_NAME}\` tool exactly once with label "${LABEL}". ` +
@@ -78,6 +82,9 @@ export default defineEval({
input: { label: LABEL },
count: 1,
});
- turn.eventsSatisfy("local action request precedes execution", streamedBeforeLocalExecution);
+ turn.eventsSatisfy(
+ "local action request precedes execution completion",
+ streamedBeforeLocalExecutionCompletes,
+ );
},
});