Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(eve): eval assertions - support count predicates (#647)

Andrew Barba89cd2d634563
diff --git a/.changeset/flexible-eval-counts.md b/.changeset/flexible-eval-counts.md
new file mode 100644
index 000000000..080c5eb80
--- /dev/null
+++ b/.changeset/flexible-eval-counts.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Eval assertion `count` options now accept predicates, allowing ranges such as “at least two” while preserving exact numeric counts.
diff --git a/docs/evals/assertions.mdx b/docs/evals/assertions.mdx
index d5f15eeba..924770c9d 100644
--- a/docs/evals/assertions.mdx
+++ b/docs/evals/assertions.mdx
@@ -77,13 +77,15 @@ Pick the cheapest builder that captures what "correct" means. When exact match i
## The matcher mini-language
-`t.calledTool` and `t.calledSubagent` take matcher objects. Tools accept `{ input, output, status, count }`; subagents accept `{ remoteUrl, output, status, count }`. Calls match `status: "completed"` by default; use `"pending"`, `"failed"`, or `"rejected"` explicitly for lifecycle checks. `count` is the exact number of calls matching every supplied constraint.
+`t.calledTool` and `t.calledSubagent` take matcher objects. Tools accept `{ input, output, status, count }`; subagents accept `{ remoteUrl, output, status, count }`. Calls match `status: "completed"` by default; use `"pending"`, `"failed"`, or `"rejected"` explicitly for lifecycle checks. A numeric `count` requires an exact number of calls matching every supplied constraint. Use a predicate for ranges or other custom count requirements; it receives the observed number of matching calls.
Matcher values accept a literal (objects partial-deep-match), a RegExp, or a predicate function that returns a boolean:
```ts
t.calledTool("bash", { input: { command: /^pwd/ }, count: 1 });
+t.calledTool("echo", { count: (count) => count >= 2 });
+
t.calledTool("echo", { output: (value) => String(value).includes(marker) });
parked.calledTool("guarded", { status: "pending", count: 1 });
@@ -108,7 +110,7 @@ const request = session.requireInputRequest({
Beyond the raw `t.events` stream, the runner derives typed facts the assertions read: tool calls (name, input, output, lifecycle status), subagent calls, and HITL input requests. A turn that leaves the session open for a next message is the normal end state of a successful turn; parking on unanswered HITL input is tracked separately.
-Typed event matching covers presence, absence, exact counts, partial event data, and ordering:
+Typed event matching covers presence, absence, numeric or predicate counts, partial event data, and ordering:
```ts
turn.notEvent("result.completed");
diff --git a/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts b/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts
index 53c4dfd71..32c82db01 100644
--- a/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/dynamic-tools/multi-turn.eval.ts
@@ -25,7 +25,7 @@ export default defineEval({
t.succeeded();
t.calledTool(ECHO_TOOL, {
output: { token: DYNAMIC_ECHO_TOKEN },
- count: 2,
+ count: (count) => count >= 2,
});
},
});
diff --git a/packages/eve/src/evals/assertions/run.test.ts b/packages/eve/src/evals/assertions/run.test.ts
index 538060b37..2032b66c9 100644
--- a/packages/eve/src/evals/assertions/run.test.ts
+++ b/packages/eve/src/evals/assertions/run.test.ts
@@ -162,6 +162,28 @@ describe("run assertions", () => {
expect((await Run.calledTool("missing").evaluate(result)).score).toBe(0);
});
+ it("calledTool accepts a predicate over the matching call count", async () => {
+ const result = makeResult({
+ derived: {
+ toolCalls: [completedToolCall("get_weather"), completedToolCall("get_weather")],
+ toolCallCount: 2,
+ },
+ });
+
+ expect(
+ (await Run.calledTool("get_weather", { count: (count) => count >= 2 }).evaluate(result))
+ .score,
+ ).toBe(1);
+ expect(
+ (await Run.calledTool("missing", { count: (count) => count === 0 }).evaluate(result)).score,
+ ).toBe(1);
+ const failed = await Run.calledTool("get_weather", {
+ count: (count) => count > 2,
+ }).evaluate(result);
+ expect(failed.score).toBe(0);
+ expect(failed.message).toContain("count predicate");
+ });
+
it("calledTool defaults to completed while notCalledTool rejects every lifecycle state", async () => {
const result = makeResult({
derived: {
@@ -180,6 +202,9 @@ describe("run assertions", () => {
it("validates exact-count options", () => {
expect(() => Run.calledTool("search", { count: -1 })).toThrow(/non-negative integer/);
+ expect(() => Run.calledTool("search", { count: "1" as never })).toThrow(
+ /non-negative integer or predicate/,
+ );
expect(() => Run.calledSubagent("child", { count: 1.5 })).toThrow(/non-negative integer/);
expect(() => Run.typedEvent({ type: "turn.completed", count: Number.NaN })).toThrow(
/non-negative integer/,
@@ -208,6 +233,17 @@ describe("run assertions", () => {
expect(
(await Run.calledSubagent("child", { status: "completed", count: 2 }).evaluate(result)).score,
).toBe(0);
+ expect(
+ (
+ await Run.calledSubagent("child", {
+ count: (count) => count >= 4,
+ status: "pending",
+ }).evaluate(result)
+ ).score,
+ ).toBe(0);
+ expect(
+ (await Run.calledSubagent("child", { count: (count) => count === 1 }).evaluate(result)).score,
+ ).toBe(1);
});
it("toolOrder checks request order", async () => {
@@ -291,6 +327,15 @@ describe("run assertions", () => {
}).evaluate(result)
).score,
).toBe(1);
+ expect(
+ (
+ await Run.typedEvent({
+ type: "subagent.called",
+ data: { name: "child" },
+ count: (count) => count >= 2,
+ }).evaluate(result)
+ ).score,
+ ).toBe(1);
expect((await Run.notEvent({ type: "turn.failed" }).evaluate(result)).score).toBe(1);
expect(
(
@@ -300,6 +345,22 @@ describe("run assertions", () => {
]).evaluate(result)
).score,
).toBe(1);
+ expect(
+ (
+ await Run.eventOrder([
+ {
+ type: "subagent.called",
+ data: { name: "child" },
+ count: (count) => count >= 2,
+ },
+ {
+ type: "subagent.completed",
+ data: { subagentName: "child" },
+ count: (count) => count >= 1,
+ },
+ ]).evaluate(result)
+ ).score,
+ ).toBe(1);
expect(
(
await Run.eventsSatisfy(
diff --git a/packages/eve/src/evals/assertions/run.ts b/packages/eve/src/evals/assertions/run.ts
index 28313d02e..caeef0faa 100644
--- a/packages/eve/src/evals/assertions/run.ts
+++ b/packages/eve/src/evals/assertions/run.ts
@@ -6,6 +6,7 @@ import {
subagentCallMatches,
testRegExp,
toolCallMatches,
+ type EveEvalCountMatcher,
type EveEvalSkillLoadMatchOptions,
type EveEvalSubagentCallMatchOptions,
type EveEvalToolCallMatchOptions,
@@ -85,7 +86,8 @@ export function messageIncludes(token: string | RegExp): RunAssertion {
/**
* Asserts a completed tool call with `name` happened. Options constrain the
* call further: `input` partial-deep-matches, `output` matches the result,
- * `status` overrides the lifecycle state, and `count` requires an exact count.
+ * `status` overrides the lifecycle state, and `count` constrains the number of
+ * matches with either an exact number or a predicate.
*/
export function calledTool(name: string, options: EveEvalToolCallMatchOptions = {}): RunAssertion {
validateCount(options.count);
@@ -94,18 +96,21 @@ export function calledTool(name: string, options: EveEvalToolCallMatchOptions =
evaluate(result) {
const named = result.derived.toolCalls.filter((call) => call.name === name);
const matching = named.filter((call) => toolCallMatches(call, options));
- const passed =
- options.count !== undefined ? matching.length === options.count : matching.length > 0;
+ const passed = matchesCount(options.count, matching.length);
if (passed) return { score: 1, metadata: { matchingCalls: matching.length } };
const observed =
named.length > 0
? `observed ${name} calls: ${named.map((call) => truncate(JSON.stringify(call.input))).join(", ")}`
: `observed tools: [${result.derived.toolCalls.map((call) => call.name).join(", ")}]`;
- const expectation =
- options.count !== undefined
- ? `expected exactly ${options.count} matching call(s), found ${matching.length}`
- : `expected a matching call to "${name}"`;
+ let expectation: string;
+ if (options.count === undefined) {
+ expectation = `expected a matching call to "${name}"`;
+ } else if (typeof options.count === "number") {
+ expectation = `expected exactly ${options.count} matching call(s), found ${matching.length}`;
+ } else {
+ expectation = `expected matching call count to satisfy the count predicate, found ${matching.length}`;
+ }
return fail(`${expectation}; ${observed}`);
},
};
@@ -233,8 +238,7 @@ export function calledSubagent(
evaluate(result) {
const named = result.derived.subagentCalls.filter((call) => call.name === name);
const matching = named.filter((call) => subagentCallMatches(call, options));
- const passed =
- options.count === undefined ? matching.length > 0 : matching.length === options.count;
+ const passed = matchesCount(options.count, matching.length);
if (passed) return { score: 1, metadata: { matchingCalls: matching.length } };
if (named.length === 0) {
@@ -270,17 +274,16 @@ export function eventsSatisfy(
};
}
-/** Asserts a typed stream event exists, optionally with an exact count. */
+/** Asserts a typed stream event exists, optionally constraining its count. */
export function typedEvent(matcher: EveEvalEventMatch): RunAssertion {
validateCount(matcher.count);
return {
name: `event(${matcher.type})`,
evaluate(result) {
const matching = result.events.filter((entry) => eventMatches(entry, matcher));
- const passed =
- matcher.count === undefined ? matching.length > 0 : matching.length === matcher.count;
+ const passed = matchesCount(matcher.count, matching.length);
if (passed) return { score: 1, metadata: { matchingEvents: matching.length } };
- const expected = matcher.count === undefined ? "at least one" : `exactly ${matcher.count}`;
+ const expected = describeCount(matcher.count);
return fail(
`expected ${expected} matching ${matcher.type} event(s), found ${matching.length}; observed: [${result.events.map((entry) => entry.type).join(", ")}]`,
);
@@ -312,11 +315,9 @@ export function eventOrder(matchers: readonly EveEvalEventMatch[]): RunAssertion
const indexes = result.events.flatMap((entry, index) =>
eventMatches(entry, matcher) ? [index] : [],
);
- const countPassed =
- matcher.count === undefined ? indexes.length > 0 : indexes.length === matcher.count;
+ const countPassed = matchesCount(matcher.count, indexes.length);
if (!countPassed) {
- const expected =
- matcher.count === undefined ? "at least one" : `exactly ${matcher.count}`;
+ const expected = describeCount(matcher.count);
return fail(
`expected ${expected} matching ${matcher.type} event(s), found ${indexes.length}`,
);
@@ -433,9 +434,22 @@ function formatUnknown(value: unknown): string | undefined {
}
}
-function validateCount(count: number | undefined): void {
- if (count !== undefined && (!Number.isInteger(count) || count < 0)) {
- throw new TypeError(`Assertion count must be a non-negative integer; received ${count}.`);
+function matchesCount(matcher: EveEvalCountMatcher | undefined, count: number): boolean {
+ if (matcher === undefined) return count > 0;
+ return typeof matcher === "function" ? matcher(count) : count === matcher;
+}
+
+function describeCount(matcher: EveEvalCountMatcher | undefined): string {
+ if (matcher === undefined) return "at least one";
+ return typeof matcher === "function" ? "a count satisfying the predicate" : `exactly ${matcher}`;
+}
+
+function validateCount(count: unknown): void {
+ if (count === undefined || typeof count === "function") return;
+ if (typeof count !== "number" || !Number.isInteger(count) || count < 0) {
+ throw new TypeError(
+ `Assertion count must be a non-negative integer or predicate; received ${String(count)}.`,
+ );
}
}
diff --git a/packages/eve/src/evals/index.ts b/packages/eve/src/evals/index.ts
index a27fd0ee7..29070843e 100644
--- a/packages/eve/src/evals/index.ts
+++ b/packages/eve/src/evals/index.ts
@@ -15,6 +15,7 @@ export type { RuntimeIdentity } from "#protocol/message.js";
export type { InputRequest } from "#runtime/input/types.js";
export type {
+ EveEvalCountMatcher,
EveEvalEventMatch,
EveEvalInputRequestMatchOptions,
EveEvalValueMatcher,
diff --git a/packages/eve/src/evals/match.ts b/packages/eve/src/evals/match.ts
index e4b558324..0fe7778a8 100644
--- a/packages/eve/src/evals/match.ts
+++ b/packages/eve/src/evals/match.ts
@@ -17,6 +17,12 @@ import type { EveEvalSubagentCall, EveEvalToolCall } from "#evals/types.js";
*/
export type EveEvalValueMatcher<T = JsonValue | undefined> = EveEvalDeepMatcher<T>;
+/**
+ * Constraint over an observed assertion count. A number requires an exact
+ * count; a predicate receives the observed count and returns its verdict.
+ */
+export type EveEvalCountMatcher = number | ((count: number) => boolean);
+
type EveEvalDeepMatcher<T> =
| RegExp
| ((value: T) => boolean)
@@ -37,8 +43,8 @@ export interface EveEvalToolCallMatchOptions {
readonly output?: EveEvalValueMatcher;
/** Required lifecycle outcome. Defaults to `"completed"`. */
readonly status?: EveEvalToolCall["status"];
- /** Exact number of matching calls required. Defaults to "at least one". */
- readonly count?: number;
+ /** Constraint over the matching call count. Defaults to "at least one". */
+ readonly count?: EveEvalCountMatcher;
}
/**
@@ -58,8 +64,8 @@ export interface EveEvalSubagentCallMatchOptions {
readonly output?: EveEvalValueMatcher;
/** Required lifecycle outcome. Defaults to `"completed"`. */
readonly status?: EveEvalSubagentCall["status"];
- /** Exact number of matching delegations required. Defaults to "at least one". */
- readonly count?: number;
+ /** Constraint over the matching delegation count. Defaults to "at least one". */
+ readonly count?: EveEvalCountMatcher;
}
/** Constraints accepted by `requireInputRequest`. */
@@ -89,8 +95,8 @@ export type EveEvalEventMatch<
? TData
: never
>;
- /** Exact number of matching events required. Defaults to "at least one". */
- readonly count?: number;
+ /** Constraint over the matching event count. Defaults to "at least one". */
+ readonly count?: EveEvalCountMatcher;
}
: never;