The Open Framework for Building Agents
feat(eve): eval assertions - support count predicates (#647)
Andrew Barba89cd2d634563
diff --git a/.changeset/flexible-eval-counts.md b/.changeset/flexible-eval-counts.mdnew file mode 100644index 000000000..080c5eb80--- /dev/null+++ b/.changeset/flexible-eval-counts.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Eval assertion `count` options now accept predicates, allowing ranges such as “at least two” while preserving exact numeric counts.diff --git a/docs/evals/assertions.mdx b/docs/evals/assertions.mdxindex d5f15eeba..924770c9d 100644--- a/docs/evals/assertions.mdx+++ b/docs/evals/assertions.mdx@@ -77,13 +77,15 @@ Pick the cheapest builder that captures what "correct" means. When exact match i## The matcher mini-language-`t.calledTool` and `t.calledSubagent` take matcher objects. Tools accept `{ input, output, status, count }`; subagents accept `{ remoteUrl, output, status, count }`. Calls match `status: "completed"` by default; use `"pending"`, `"failed"`, or `"rejected"` explicitly for lifecycle checks. `count` is the exact number of calls matching every supplied constraint.+`t.calledTool` and `t.calledSubagent` take matcher objects. Tools accept `{ input, output, status, count }`; subagents accept `{ remoteUrl, output, status, count }`. Calls match `status: "completed"` by default; use `"pending"`, `"failed"`, or `"rejected"` explicitly for lifecycle checks. A numeric `count` requires an exact number of calls matching every supplied constraint. Use a predicate for ranges or other custom count requirements; it receives the observed number of matching calls.Matcher values accept a literal (objects partial-deep-match), a RegExp, or a predicate function that returns a boolean:```tst.calledTool("bash", { input: { command: /^pwd/ }, count: 1 });+t.calledTool("echo", { count: (count) => count >= 2 });+t.calledTool("echo", { output: (value) => String(value).includes(marker) });parked.calledTool("guarded", { status: "pending", count: 1 });@@ -108,7 +110,7 @@ const request = session.requireInputRequest({Beyond the raw `t.events` stream, the runner derives typed facts the assertions read: tool calls (name, input, output, lifecycle status), subagent calls, and HITL input requests. A turn that leaves the session open for a next message is the normal end state of a successful turn; parking on unanswered HITL input is tracked separately.-Typed event matching covers presence, absence, exact counts, partial event data, and ordering:+Typed event matching covers presence, absence, numeric or predicate counts, partial event data, and ordering:```tsturn.notEvent("result.completed");diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.tsindex 53c4dfd71..32c82db01 100644--- a/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts+++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts@@ -25,7 +25,7 @@ export default defineEval({t.succeeded();t.calledTool(ECHO_TOOL, {output: { token: DYNAMIC_ECHO_TOKEN },- count: 2,+ count: (count) => count >= 2,});},});diff --git a/packages/eve/src/evals/assertions/run.test.ts b/packages/eve/src/evals/assertions/run.test.tsindex 538060b37..2032b66c9 100644--- a/packages/eve/src/evals/assertions/run.test.ts+++ b/packages/eve/src/evals/assertions/run.test.ts@@ -162,6 +162,28 @@ describe("run assertions", () => {expect((await Run.calledTool("missing").evaluate(result)).score).toBe(0);});+ it("calledTool accepts a predicate over the matching call count", async () => {+ const result = makeResult({+ derived: {+ toolCalls: [completedToolCall("get_weather"), completedToolCall("get_weather")],+ toolCallCount: 2,+ },+ });++ expect(+ (await Run.calledTool("get_weather", { count: (count) => count >= 2 }).evaluate(result))+ .score,+ ).toBe(1);+ expect(+ (await Run.calledTool("missing", { count: (count) => count === 0 }).evaluate(result)).score,+ ).toBe(1);+ const failed = await Run.calledTool("get_weather", {+ count: (count) => count > 2,+ }).evaluate(result);+ expect(failed.score).toBe(0);+ expect(failed.message).toContain("count predicate");+ });+it("calledTool defaults to completed while notCalledTool rejects every lifecycle state", async () => {const result = makeResult({derived: {@@ -180,6 +202,9 @@ describe("run assertions", () => {it("validates exact-count options", () => {expect(() => Run.calledTool("search", { count: -1 })).toThrow(/non-negative integer/);+ expect(() => Run.calledTool("search", { count: "1" as never })).toThrow(+ /non-negative integer or predicate/,+ );expect(() => Run.calledSubagent("child", { count: 1.5 })).toThrow(/non-negative integer/);expect(() => Run.typedEvent({ type: "turn.completed", count: Number.NaN })).toThrow(/non-negative integer/,@@ -208,6 +233,17 @@ describe("run assertions", () => {expect((await Run.calledSubagent("child", { status: "completed", count: 2 }).evaluate(result)).score,).toBe(0);+ expect(+ (+ await Run.calledSubagent("child", {+ count: (count) => count >= 4,+ status: "pending",+ }).evaluate(result)+ ).score,+ ).toBe(0);+ expect(+ (await Run.calledSubagent("child", { count: (count) => count === 1 }).evaluate(result)).score,+ ).toBe(1);});it("toolOrder checks request order", async () => {@@ -291,6 +327,15 @@ describe("run assertions", () => {}).evaluate(result)).score,).toBe(1);+ expect(+ (+ await Run.typedEvent({+ type: "subagent.called",+ data: { name: "child" },+ count: (count) => count >= 2,+ }).evaluate(result)+ ).score,+ ).toBe(1);expect((await Run.notEvent({ type: "turn.failed" }).evaluate(result)).score).toBe(1);expect((@@ -300,6 +345,22 @@ describe("run assertions", () => {]).evaluate(result)).score,).toBe(1);+ expect(+ (+ await Run.eventOrder([+ {+ type: "subagent.called",+ data: { name: "child" },+ count: (count) => count >= 2,+ },+ {+ type: "subagent.completed",+ data: { subagentName: "child" },+ count: (count) => count >= 1,+ },+ ]).evaluate(result)+ ).score,+ ).toBe(1);expect((await Run.eventsSatisfy(diff --git a/packages/eve/src/evals/assertions/run.ts b/packages/eve/src/evals/assertions/run.tsindex 28313d02e..caeef0faa 100644--- a/packages/eve/src/evals/assertions/run.ts+++ b/packages/eve/src/evals/assertions/run.ts@@ -6,6 +6,7 @@ import {subagentCallMatches,testRegExp,toolCallMatches,+ type EveEvalCountMatcher,type EveEvalSkillLoadMatchOptions,type EveEvalSubagentCallMatchOptions,type EveEvalToolCallMatchOptions,@@ -85,7 +86,8 @@ export function messageIncludes(token: string | RegExp): RunAssertion {/*** Asserts a completed tool call with `name` happened. Options constrain the* call further: `input` partial-deep-matches, `output` matches the result,- * `status` overrides the lifecycle state, and `count` requires an exact count.+ * `status` overrides the lifecycle state, and `count` constrains the number of+ * matches with either an exact number or a predicate.*/export function calledTool(name: string, options: EveEvalToolCallMatchOptions = {}): RunAssertion {validateCount(options.count);@@ -94,18 +96,21 @@ export function calledTool(name: string, options: EveEvalToolCallMatchOptions =evaluate(result) {const named = result.derived.toolCalls.filter((call) => call.name === name);const matching = named.filter((call) => toolCallMatches(call, options));- const passed =- options.count !== undefined ? matching.length === options.count : matching.length > 0;+ const passed = matchesCount(options.count, matching.length);if (passed) return { score: 1, metadata: { matchingCalls: matching.length } };const observed =named.length > 0? `observed ${name} calls: ${named.map((call) => truncate(JSON.stringify(call.input))).join(", ")}`: `observed tools: [${result.derived.toolCalls.map((call) => call.name).join(", ")}]`;- const expectation =- options.count !== undefined- ? `expected exactly ${options.count} matching call(s), found ${matching.length}`- : `expected a matching call to "${name}"`;+ let expectation: string;+ if (options.count === undefined) {+ expectation = `expected a matching call to "${name}"`;+ } else if (typeof options.count === "number") {+ expectation = `expected exactly ${options.count} matching call(s), found ${matching.length}`;+ } else {+ expectation = `expected matching call count to satisfy the count predicate, found ${matching.length}`;+ }return fail(`${expectation}; ${observed}`);},};@@ -233,8 +238,7 @@ export function calledSubagent(evaluate(result) {const named = result.derived.subagentCalls.filter((call) => call.name === name);const matching = named.filter((call) => subagentCallMatches(call, options));- const passed =- options.count === undefined ? matching.length > 0 : matching.length === options.count;+ const passed = matchesCount(options.count, matching.length);if (passed) return { score: 1, metadata: { matchingCalls: matching.length } };if (named.length === 0) {@@ -270,17 +274,16 @@ export function eventsSatisfy(};}-/** Asserts a typed stream event exists, optionally with an exact count. */+/** Asserts a typed stream event exists, optionally constraining its count. */export function typedEvent(matcher: EveEvalEventMatch): RunAssertion {validateCount(matcher.count);return {name: `event(${matcher.type})`,evaluate(result) {const matching = result.events.filter((entry) => eventMatches(entry, matcher));- const passed =- matcher.count === undefined ? matching.length > 0 : matching.length === matcher.count;+ const passed = matchesCount(matcher.count, matching.length);if (passed) return { score: 1, metadata: { matchingEvents: matching.length } };- const expected = matcher.count === undefined ? "at least one" : `exactly ${matcher.count}`;+ const expected = describeCount(matcher.count);return fail(`expected ${expected} matching ${matcher.type} event(s), found ${matching.length}; observed: [${result.events.map((entry) => entry.type).join(", ")}]`,);@@ -312,11 +315,9 @@ export function eventOrder(matchers: readonly EveEvalEventMatch[]): RunAssertionconst indexes = result.events.flatMap((entry, index) =>eventMatches(entry, matcher) ? [index] : [],);- const countPassed =- matcher.count === undefined ? indexes.length > 0 : indexes.length === matcher.count;+ const countPassed = matchesCount(matcher.count, indexes.length);if (!countPassed) {- const expected =- matcher.count === undefined ? "at least one" : `exactly ${matcher.count}`;+ const expected = describeCount(matcher.count);return fail(`expected ${expected} matching ${matcher.type} event(s), found ${indexes.length}`,);@@ -433,9 +434,22 @@ function formatUnknown(value: unknown): string | undefined {}}-function validateCount(count: number | undefined): void {- if (count !== undefined && (!Number.isInteger(count) || count < 0)) {- throw new TypeError(`Assertion count must be a non-negative integer; received ${count}.`);+function matchesCount(matcher: EveEvalCountMatcher | undefined, count: number): boolean {+ if (matcher === undefined) return count > 0;+ return typeof matcher === "function" ? matcher(count) : count === matcher;+}++function describeCount(matcher: EveEvalCountMatcher | undefined): string {+ if (matcher === undefined) return "at least one";+ return typeof matcher === "function" ? "a count satisfying the predicate" : `exactly ${matcher}`;+}++function validateCount(count: unknown): void {+ if (count === undefined || typeof count === "function") return;+ if (typeof count !== "number" || !Number.isInteger(count) || count < 0) {+ throw new TypeError(+ `Assertion count must be a non-negative integer or predicate; received ${String(count)}.`,+ );}}diff --git a/packages/eve/src/evals/index.ts b/packages/eve/src/evals/index.tsindex a27fd0ee7..29070843e 100644--- a/packages/eve/src/evals/index.ts+++ b/packages/eve/src/evals/index.ts@@ -15,6 +15,7 @@ export type { RuntimeIdentity } from "#protocol/message.js";export type { InputRequest } from "#runtime/input/types.js";export type {+ EveEvalCountMatcher,EveEvalEventMatch,EveEvalInputRequestMatchOptions,EveEvalValueMatcher,diff --git a/packages/eve/src/evals/match.ts b/packages/eve/src/evals/match.tsindex e4b558324..0fe7778a8 100644--- a/packages/eve/src/evals/match.ts+++ b/packages/eve/src/evals/match.ts@@ -17,6 +17,12 @@ import type { EveEvalSubagentCall, EveEvalToolCall } from "#evals/types.js";*/export type EveEvalValueMatcher<T = JsonValue | undefined> = EveEvalDeepMatcher<T>;+/**+ * Constraint over an observed assertion count. A number requires an exact+ * count; a predicate receives the observed count and returns its verdict.+ */+export type EveEvalCountMatcher = number | ((count: number) => boolean);+type EveEvalDeepMatcher<T> =| RegExp| ((value: T) => boolean)@@ -37,8 +43,8 @@ export interface EveEvalToolCallMatchOptions {readonly output?: EveEvalValueMatcher;/** Required lifecycle outcome. Defaults to `"completed"`. */readonly status?: EveEvalToolCall["status"];- /** Exact number of matching calls required. Defaults to "at least one". */- readonly count?: number;+ /** Constraint over the matching call count. Defaults to "at least one". */+ readonly count?: EveEvalCountMatcher;}/**@@ -58,8 +64,8 @@ export interface EveEvalSubagentCallMatchOptions {readonly output?: EveEvalValueMatcher;/** Required lifecycle outcome. Defaults to `"completed"`. */readonly status?: EveEvalSubagentCall["status"];- /** Exact number of matching delegations required. Defaults to "at least one". */- readonly count?: number;+ /** Constraint over the matching delegation count. Defaults to "at least one". */+ readonly count?: EveEvalCountMatcher;}/** Constraints accepted by `requireInputRequest`. */@@ -89,8 +95,8 @@ export type EveEvalEventMatch<? TData: never>;- /** Exact number of matching events required. Defaults to "at least one". */- readonly count?: number;+ /** Constraint over the matching event count. Defaults to "at least one". */+ readonly count?: EveEvalCountMatcher;}: never;