Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat(eve): expose assertion scorer errors (#3659)

Ben Pankowa3b795b8973a
diff --git a/.changeset/tidy-evals-error-status.md b/.changeset/tidy-evals-error-status.md
new file mode 100644
index 000000000..6db983c84
--- /dev/null
+++ b/.changeset/tidy-evals-error-status.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Export an `errored` flag on eval assertion results so scorer failures can be distinguished from legitimate zero scores.
diff --git a/packages/eve/src/evals/assertions/collector.ts b/packages/eve/src/evals/assertions/collector.ts
index 74afb64fc..4e00a0053 100644
--- a/packages/eve/src/evals/assertions/collector.ts
+++ b/packages/eve/src/evals/assertions/collector.ts
@@ -32,7 +32,7 @@ interface MutableEntry {
message?: string;
metadata?: Readonly<Record<string, unknown>>;
/** A model/value assertion that threw — a hard failure regardless of severity. */
- failed: boolean;
+ errored: boolean;
}
/**
@@ -66,7 +66,7 @@ export class AssertionCollector {
spec,
selectSubject,
score: 0,
- failed: false,
+ errored: false,
};
this.#entries.push(entry);
return makeHandle(entry);
@@ -86,7 +86,7 @@ export class AssertionCollector {
threshold: input.threshold,
kind: "resolved",
score: 0,
- failed: false,
+ errored: false,
};
this.#entries.push(entry);
@@ -107,10 +107,10 @@ export class AssertionCollector {
score: input.outcome.score,
message: input.outcome.message,
metadata: input.outcome.metadata,
- failed: false,
+ errored: false,
};
this.#entries.push(entry);
- return computePassed(entry.severity, entry.threshold, entry.score, entry.failed);
+ return computePassed(entry.severity, entry.threshold, entry.score, entry.errored);
}
/** Record and await a required value assertion, returning whether it passed. */
@@ -126,11 +126,11 @@ export class AssertionCollector {
threshold: input.threshold,
kind: "resolved",
score: 0,
- failed: false,
+ errored: false,
};
this.#entries.push(entry);
await settleEntry(entry, input.score);
- return computePassed(entry.severity, entry.threshold, entry.score, entry.failed);
+ return computePassed(entry.severity, entry.threshold, entry.score, entry.errored);
}
/**
@@ -153,7 +153,8 @@ export class AssertionCollector {
score: entry.score,
severity: entry.severity,
threshold: entry.threshold,
- passed: computePassed(entry.severity, entry.threshold, entry.score, entry.failed),
+ passed: computePassed(entry.severity, entry.threshold, entry.score, entry.errored),
+ errored: entry.errored,
message: entry.message,
metadata: entry.metadata,
});
@@ -171,9 +172,9 @@ function computePassed(
severity: AssertionSeverity,
threshold: number | undefined,
score: number,
- failed: boolean,
+ errored: boolean,
): boolean {
- if (failed) return false;
+ if (errored) return false;
const min = threshold ?? (severity === "gate" ? 1 : undefined);
return min === undefined || score >= min;
}
@@ -194,7 +195,7 @@ async function settleEntry(
entry.severity = "gate";
entry.threshold = undefined;
entry.message = toErrorMessage(error);
- entry.failed = true;
+ entry.errored = true;
}
}
diff --git a/packages/eve/src/evals/judge.integration.test.ts b/packages/eve/src/evals/judge.integration.test.ts
index ec922788b..0ee4cbac5 100644
--- a/packages/eve/src/evals/judge.integration.test.ts
+++ b/packages/eve/src/evals/judge.integration.test.ts
@@ -50,7 +50,12 @@ describe("evaluation-backed judge runner", () => {
t.judge("Answers the request").atLeast(0.9);
});
expect(factory).toHaveBeenCalledWith("typesafe-ai/jev");
- expect(outcome.assertions[0]).toMatchObject({ score: 0.8, severity: "soft", passed: false });
+ expect(outcome.assertions[0]).toMatchObject({
+ score: 0.8,
+ severity: "soft",
+ passed: false,
+ errored: false,
+ });
expect(computeEvalVerdict(outcome)).toBe("scored");
});
@@ -117,7 +122,12 @@ describe("evaluation-backed judge runner", () => {
expect(evaluator.doEvaluate).not.toHaveBeenCalled();
expect(outcome.assertions).toHaveLength(batch ? 2 : 1);
for (const assertion of outcome.assertions) {
- expect(assertion).toMatchObject({ score: 0, severity: "gate", passed: false });
+ expect(assertion).toMatchObject({
+ score: 0,
+ severity: "gate",
+ passed: false,
+ errored: true,
+ });
expect(assertion.message).toContain(
"score criteria must contain at least two ordered levels",
);
@@ -159,7 +169,9 @@ describe("evaluation-backed judge runner", () => {
);
expect(outcome.skipReason).toBeUndefined();
expect(outcome.assertions).toHaveLength(2);
- expect(outcome.assertions.every((a) => !a.passed && a.severity === "gate")).toBe(true);
+ expect(outcome.assertions.every((a) => !a.passed && a.severity === "gate" && a.errored)).toBe(
+ true,
+ );
expect(computeEvalVerdict(outcome)).toBe("failed");
},
);
diff --git a/packages/eve/src/evals/runner/artifacts.test.ts b/packages/eve/src/evals/runner/artifacts.test.ts
index a1976f925..587ad8ab5 100644
--- a/packages/eve/src/evals/runner/artifacts.test.ts
+++ b/packages/eve/src/evals/runner/artifacts.test.ts
@@ -79,6 +79,7 @@ describe("eval artifacts", () => {
},
name: "judge.boolean",
passed: false,
+ errored: false,
score: 0,
severity: "soft",
threshold: 0.8,
@@ -138,6 +139,7 @@ function judgedSummary(): EveEvalRunSummary {
severity: "soft",
threshold: 0.8,
passed: false,
+ errored: false,
message: 'state: {"input":"Name the source.","output":"No source."}',
metadata: {
question: { type: "boolean", instructions: "cites a source" },
diff --git a/packages/eve/src/evals/runner/artifacts.ts b/packages/eve/src/evals/runner/artifacts.ts
index 73a536c72..8d6a85482 100644
--- a/packages/eve/src/evals/runner/artifacts.ts
+++ b/packages/eve/src/evals/runner/artifacts.ts
@@ -70,6 +70,7 @@ function buildSummaryArtifact(summary: EveEvalRunSummary): Record<string, unknow
severity: a.severity,
threshold: a.threshold,
passed: a.passed,
+ errored: a.errored,
message: a.message,
metadata: a.metadata,
})),
diff --git a/packages/eve/src/evals/runner/execute-eval.test.ts b/packages/eve/src/evals/runner/execute-eval.test.ts
index c800f49fb..63ab2bf63 100644
--- a/packages/eve/src/evals/runner/execute-eval.test.ts
+++ b/packages/eve/src/evals/runner/execute-eval.test.ts
@@ -31,11 +31,18 @@ function taskOutcome(assertions: readonly AssertionResult[] = [], error?: string
}
function gate(passed: boolean): AssertionResult {
- return { name: "gate", score: passed ? 1 : 0, severity: "gate", passed };
+ return { name: "gate", score: passed ? 1 : 0, severity: "gate", passed, errored: false };
}
function softMiss(): AssertionResult {
- return { name: "soft", score: 0.5, severity: "soft", threshold: 0.9, passed: false };
+ return {
+ name: "soft",
+ score: 0.5,
+ severity: "soft",
+ threshold: 0.9,
+ passed: false,
+ errored: false,
+ };
}
function createEval(overrides: Partial<EveEval> = {}): EveEval {
diff --git a/packages/eve/src/evals/runner/reporters/assertion-diagnostics.test.ts b/packages/eve/src/evals/runner/reporters/assertion-diagnostics.test.ts
index 6d64d4daa..4b298431b 100644
--- a/packages/eve/src/evals/runner/reporters/assertion-diagnostics.test.ts
+++ b/packages/eve/src/evals/runner/reporters/assertion-diagnostics.test.ts
@@ -12,6 +12,7 @@ describe("assertion failure diagnostics", () => {
message: [`prompt: ${"x".repeat(300)}`, "one", "two", "three", "four", "five"].join("\n"),
name: "judge.boolean [citation]",
passed: false,
+ errored: false,
score: 0.42,
severity: "soft",
threshold: 0.8,
diff --git a/packages/eve/src/evals/runner/reporters/braintrust.test.ts b/packages/eve/src/evals/runner/reporters/braintrust.test.ts
index 2ddcb1ea5..b3e8867f3 100644
--- a/packages/eve/src/evals/runner/reporters/braintrust.test.ts
+++ b/packages/eve/src/evals/runner/reporters/braintrust.test.ts
@@ -66,8 +66,15 @@ function makeEvalResult(overrides: Partial<EveEvalResult> = {}): EveEvalResult {
traceContexts: [],
},
assertions: [
- { name: "succeeded", score: 1, severity: "gate", passed: true },
- { name: "similarity", score: 1, severity: "soft", threshold: 0.6, passed: true },
+ { name: "succeeded", score: 1, severity: "gate", passed: true, errored: false },
+ {
+ name: "similarity",
+ score: 1,
+ severity: "soft",
+ threshold: 0.6,
+ passed: true,
+ errored: false,
+ },
],
verdict: "passed",
startedAt: "2026-01-01T00:00:00.000Z",
@@ -146,6 +153,7 @@ describe("Braintrust", () => {
},
name: "judge.boolean",
passed: false,
+ errored: false,
score: 0,
severity: "soft",
threshold: 0.8,
@@ -168,6 +176,7 @@ describe("Braintrust", () => {
},
name: "judge.boolean",
passed: false,
+ errored: false,
score: 0,
severity: "soft",
threshold: 0.8,
@@ -212,8 +221,8 @@ describe("Braintrust", () => {
reporter.onEvalComplete(
makeEvalResult({
assertions: [
- { name: "similarity", passed: true, score: 0.8, severity: "soft" },
- { name: "similarity", passed: true, score: 0.6, severity: "soft" },
+ { name: "similarity", passed: true, score: 0.8, severity: "soft", errored: false },
+ { name: "similarity", passed: true, score: 0.6, severity: "soft", errored: false },
],
}),
);
diff --git a/packages/eve/src/evals/runner/reporters/console.test.ts b/packages/eve/src/evals/runner/reporters/console.test.ts
index ae7dbaa7c..2b7cdf504 100644
--- a/packages/eve/src/evals/runner/reporters/console.test.ts
+++ b/packages/eve/src/evals/runner/reporters/console.test.ts
@@ -31,6 +31,7 @@ describe("Console", () => {
].join("\n"),
name: "judge.boolean",
passed: false,
+ errored: false,
score: 0,
severity: "soft",
threshold: 0.8,
@@ -66,6 +67,7 @@ describe("Console", () => {
metadata: { rationale: "Successful rationale details" },
name: "judge.boolean",
passed: true,
+ errored: false,
score: 1,
severity: "gate",
},
diff --git a/packages/eve/src/evals/runner/reporters/datadog.test.ts b/packages/eve/src/evals/runner/reporters/datadog.test.ts
index a97f82c6b..ee906cfc7 100644
--- a/packages/eve/src/evals/runner/reporters/datadog.test.ts
+++ b/packages/eve/src/evals/runner/reporters/datadog.test.ts
@@ -66,9 +66,16 @@ function makeEvalResult(overrides: Partial<EveEvalResult> = {}): EveEvalResult {
traceContexts: [],
},
assertions: [
- { name: "succeeded", score: 1, severity: "gate", passed: true },
- { name: "similarity", score: 0.9, severity: "soft", threshold: 0.6, passed: true },
- { name: "judge.boolean", score: 1, severity: "soft", passed: true },
+ { name: "succeeded", score: 1, severity: "gate", passed: true, errored: false },
+ {
+ name: "similarity",
+ score: 0.9,
+ severity: "soft",
+ threshold: 0.6,
+ passed: true,
+ errored: false,
+ },
+ { name: "judge.boolean", score: 1, severity: "soft", passed: true, errored: false },
],
verdict: "passed",
startedAt: "2026-01-01T00:00:00.000Z",
@@ -262,6 +269,7 @@ describe("Datadog", () => {
score: 0,
severity: "gate",
passed: false,
+ errored: false,
},
],
verdict: "failed",
@@ -298,6 +306,7 @@ describe("Datadog", () => {
score: 0,
severity: "gate",
passed: false,
+ errored: false,
},
],
verdict: "failed",
@@ -332,10 +341,10 @@ describe("Datadog", () => {
const reporter = Datadog(config);
const result = makeEvalResult({
assertions: [
- { name: "same label", score: 1, severity: "soft", passed: true },
- { name: "same@label", score: 1, severity: "soft", passed: true },
- { name: "same label", score: 1, severity: "soft", passed: true },
- { name: "eve_tool_call_count", score: 1, severity: "soft", passed: true },
+ { name: "same label", score: 1, severity: "soft", passed: true, errored: false },
+ { name: "same@label", score: 1, severity: "soft", passed: true, errored: false },
+ { name: "same label", score: 1, severity: "soft", passed: true, errored: false },
+ { name: "eve_tool_call_count", score: 1, severity: "soft", passed: true, errored: false },
],
});
diff --git a/packages/eve/src/evals/runner/reporters/junit.test.ts b/packages/eve/src/evals/runner/reporters/junit.test.ts
index d97112527..89a08b15a 100644
--- a/packages/eve/src/evals/runner/reporters/junit.test.ts
+++ b/packages/eve/src/evals/runner/reporters/junit.test.ts
@@ -74,6 +74,7 @@ function makeSummary(): EveEvalRunSummary {
score: 0,
severity: "gate",
passed: false,
+ errored: false,
message: "expected hello",
},
],
@@ -102,6 +103,7 @@ function makeEvalResult(overrides: Partial<EveEvalResult> = {}): EveEvalResult {
score: 1,
severity: "gate",
passed: true,
+ errored: false,
},
],
result: {
diff --git a/packages/eve/src/evals/runner/verdict.test.ts b/packages/eve/src/evals/runner/verdict.test.ts
index e49128b27..84f13b17f 100644
--- a/packages/eve/src/evals/runner/verdict.test.ts
+++ b/packages/eve/src/evals/runner/verdict.test.ts
@@ -9,6 +9,7 @@ function assertion(overrides: Partial<AssertionResult>): AssertionResult {
score: 1,
severity: "gate",
passed: true,
+ errored: false,
...overrides,
};
}
@@ -29,7 +30,7 @@ describe("computeEvalVerdict", () => {
it("fails when a gate assertion failed", () => {
expect(
computeEvalVerdict({
- assertions: [assertion({ severity: "gate", score: 0, passed: false })],
+ assertions: [assertion({ severity: "gate", score: 0, passed: false, errored: false })],
}),
).toBe("failed");
});
@@ -38,8 +39,14 @@ describe("computeEvalVerdict", () => {
expect(
computeEvalVerdict({
assertions: [
- assertion({ severity: "gate", passed: true }),
- assertion({ severity: "soft", threshold: 0.6, score: 0.3, passed: false }),
+ assertion({ severity: "gate", passed: true, errored: false }),
+ assertion({
+ severity: "soft",
+ threshold: 0.6,
+ score: 0.3,
+ passed: false,
+ errored: false,
+ }),
],
}),
).toBe("scored");
@@ -49,8 +56,14 @@ describe("computeEvalVerdict", () => {
expect(
computeEvalVerdict({
assertions: [
- assertion({ severity: "soft", threshold: 0.6, score: 0.3, passed: false }),
- assertion({ severity: "gate", score: 0, passed: false }),
+ assertion({
+ severity: "soft",
+ threshold: 0.6,
+ score: 0.3,
+ passed: false,
+ errored: false,
+ }),
+ assertion({ severity: "gate", score: 0, passed: false, errored: false }),
],
}),
).toBe("failed");
@@ -59,7 +72,7 @@ describe("computeEvalVerdict", () => {
it("passes when a tracked soft assertion has no threshold", () => {
expect(
computeEvalVerdict({
- assertions: [assertion({ severity: "soft", score: 0.1, passed: true })],
+ assertions: [assertion({ severity: "soft", score: 0.1, passed: true, errored: false })],
}),
).toBe("passed");
});
@@ -76,7 +89,7 @@ describe("computeEvalVerdict", () => {
it("never lets an explicit skip mask errors or failed gates", () => {
expect(
computeEvalVerdict({
- assertions: [assertion({ severity: "gate", score: 0, passed: false })],
+ assertions: [assertion({ severity: "gate", score: 0, passed: false, errored: false })],
error: "ignored",
skipReason: "unsupported target",
}),
diff --git a/packages/eve/src/evals/types.ts b/packages/eve/src/evals/types.ts
index 6dd858727..7513b53d6 100644
--- a/packages/eve/src/evals/types.ts
+++ b/packages/eve/src/evals/types.ts
@@ -213,6 +213,8 @@ export interface AssertionResult {
readonly severity: AssertionSeverity;
readonly threshold?: number;
readonly passed: boolean;
+ /** Whether the assertion failed because its scorer threw instead of producing a score. */
+ readonly errored: boolean;
/** Human-readable failure detail, shown in console output and artifacts. */
readonly message?: string;
readonly metadata?: Readonly<Record<string, unknown>>;