The Open Framework for Building Agents
feat(eve): expose assertion scorer errors (#3659)
Ben Pankowa3b795b8973a
diff --git a/.changeset/tidy-evals-error-status.md b/.changeset/tidy-evals-error-status.mdnew file mode 100644index 000000000..6db983c84--- /dev/null+++ b/.changeset/tidy-evals-error-status.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Export an `errored` flag on eval assertion results so scorer failures can be distinguished from legitimate zero scores.diff --git a/packages/eve/src/evals/assertions/collector.ts b/packages/eve/src/evals/assertions/collector.tsindex 74afb64fc..4e00a0053 100644--- a/packages/eve/src/evals/assertions/collector.ts+++ b/packages/eve/src/evals/assertions/collector.ts@@ -32,7 +32,7 @@ interface MutableEntry {message?: string;metadata?: Readonly<Record<string, unknown>>;/** A model/value assertion that threw — a hard failure regardless of severity. */- failed: boolean;+ errored: boolean;}/**@@ -66,7 +66,7 @@ export class AssertionCollector {spec,selectSubject,score: 0,- failed: false,+ errored: false,};this.#entries.push(entry);return makeHandle(entry);@@ -86,7 +86,7 @@ export class AssertionCollector {threshold: input.threshold,kind: "resolved",score: 0,- failed: false,+ errored: false,};this.#entries.push(entry);@@ -107,10 +107,10 @@ export class AssertionCollector {score: input.outcome.score,message: input.outcome.message,metadata: input.outcome.metadata,- failed: false,+ errored: false,};this.#entries.push(entry);- return computePassed(entry.severity, entry.threshold, entry.score, entry.failed);+ return computePassed(entry.severity, entry.threshold, entry.score, entry.errored);}/** Record and await a required value assertion, returning whether it passed. */@@ -126,11 +126,11 @@ export class AssertionCollector {threshold: input.threshold,kind: "resolved",score: 0,- failed: false,+ errored: false,};this.#entries.push(entry);await settleEntry(entry, input.score);- return computePassed(entry.severity, entry.threshold, entry.score, entry.failed);+ return computePassed(entry.severity, entry.threshold, entry.score, entry.errored);}/**@@ -153,7 +153,8 @@ export class AssertionCollector {score: entry.score,severity: entry.severity,threshold: entry.threshold,- passed: computePassed(entry.severity, entry.threshold, entry.score, entry.failed),+ passed: computePassed(entry.severity, entry.threshold, entry.score, entry.errored),+ errored: entry.errored,message: entry.message,metadata: entry.metadata,});@@ -171,9 +172,9 @@ function computePassed(severity: AssertionSeverity,threshold: number | undefined,score: number,- failed: boolean,+ errored: boolean,): boolean {- if (failed) return false;+ if (errored) return false;const min = threshold ?? (severity === "gate" ? 1 : undefined);return min === undefined || score >= min;}@@ -194,7 +195,7 @@ async function settleEntry(entry.severity = "gate";entry.threshold = undefined;entry.message = toErrorMessage(error);- entry.failed = true;+ entry.errored = true;}}diff --git a/packages/eve/src/evals/judge.integration.test.ts b/packages/eve/src/evals/judge.integration.test.tsindex ec922788b..0ee4cbac5 100644--- a/packages/eve/src/evals/judge.integration.test.ts+++ b/packages/eve/src/evals/judge.integration.test.ts@@ -50,7 +50,12 @@ describe("evaluation-backed judge runner", () => {t.judge("Answers the request").atLeast(0.9);});expect(factory).toHaveBeenCalledWith("typesafe-ai/jev");- expect(outcome.assertions[0]).toMatchObject({ score: 0.8, severity: "soft", passed: false });+ expect(outcome.assertions[0]).toMatchObject({+ score: 0.8,+ severity: "soft",+ passed: false,+ errored: false,+ });expect(computeEvalVerdict(outcome)).toBe("scored");});@@ -117,7 +122,12 @@ describe("evaluation-backed judge runner", () => {expect(evaluator.doEvaluate).not.toHaveBeenCalled();expect(outcome.assertions).toHaveLength(batch ? 2 : 1);for (const assertion of outcome.assertions) {- expect(assertion).toMatchObject({ score: 0, severity: "gate", passed: false });+ expect(assertion).toMatchObject({+ score: 0,+ severity: "gate",+ passed: false,+ errored: true,+ });expect(assertion.message).toContain("score criteria must contain at least two ordered levels",);@@ -159,7 +169,9 @@ describe("evaluation-backed judge runner", () => {);expect(outcome.skipReason).toBeUndefined();expect(outcome.assertions).toHaveLength(2);- expect(outcome.assertions.every((a) => !a.passed && a.severity === "gate")).toBe(true);+ expect(outcome.assertions.every((a) => !a.passed && a.severity === "gate" && a.errored)).toBe(+ true,+ );expect(computeEvalVerdict(outcome)).toBe("failed");},);diff --git a/packages/eve/src/evals/runner/artifacts.test.ts b/packages/eve/src/evals/runner/artifacts.test.tsindex a1976f925..587ad8ab5 100644--- a/packages/eve/src/evals/runner/artifacts.test.ts+++ b/packages/eve/src/evals/runner/artifacts.test.ts@@ -79,6 +79,7 @@ describe("eval artifacts", () => {},name: "judge.boolean",passed: false,+ errored: false,score: 0,severity: "soft",threshold: 0.8,@@ -138,6 +139,7 @@ function judgedSummary(): EveEvalRunSummary {severity: "soft",threshold: 0.8,passed: false,+ errored: false,message: 'state: {"input":"Name the source.","output":"No source."}',metadata: {question: { type: "boolean", instructions: "cites a source" },diff --git a/packages/eve/src/evals/runner/artifacts.ts b/packages/eve/src/evals/runner/artifacts.tsindex 73a536c72..8d6a85482 100644--- a/packages/eve/src/evals/runner/artifacts.ts+++ b/packages/eve/src/evals/runner/artifacts.ts@@ -70,6 +70,7 @@ function buildSummaryArtifact(summary: EveEvalRunSummary): Record<string, unknowseverity: a.severity,threshold: a.threshold,passed: a.passed,+ errored: a.errored,message: a.message,metadata: a.metadata,})),diff --git a/packages/eve/src/evals/runner/execute-eval.test.ts b/packages/eve/src/evals/runner/execute-eval.test.tsindex c800f49fb..63ab2bf63 100644--- a/packages/eve/src/evals/runner/execute-eval.test.ts+++ b/packages/eve/src/evals/runner/execute-eval.test.ts@@ -31,11 +31,18 @@ function taskOutcome(assertions: readonly AssertionResult[] = [], error?: string}function gate(passed: boolean): AssertionResult {- return { name: "gate", score: passed ? 1 : 0, severity: "gate", passed };+ return { name: "gate", score: passed ? 1 : 0, severity: "gate", passed, errored: false };}function softMiss(): AssertionResult {- return { name: "soft", score: 0.5, severity: "soft", threshold: 0.9, passed: false };+ return {+ name: "soft",+ score: 0.5,+ severity: "soft",+ threshold: 0.9,+ passed: false,+ errored: false,+ };}function createEval(overrides: Partial<EveEval> = {}): EveEval {diff --git a/packages/eve/src/evals/runner/reporters/assertion-diagnostics.test.ts b/packages/eve/src/evals/runner/reporters/assertion-diagnostics.test.tsindex 6d64d4daa..4b298431b 100644--- a/packages/eve/src/evals/runner/reporters/assertion-diagnostics.test.ts+++ b/packages/eve/src/evals/runner/reporters/assertion-diagnostics.test.ts@@ -12,6 +12,7 @@ describe("assertion failure diagnostics", () => {message: [`prompt: ${"x".repeat(300)}`, "one", "two", "three", "four", "five"].join("\n"),name: "judge.boolean [citation]",passed: false,+ errored: false,score: 0.42,severity: "soft",threshold: 0.8,diff --git a/packages/eve/src/evals/runner/reporters/braintrust.test.ts b/packages/eve/src/evals/runner/reporters/braintrust.test.tsindex 2ddcb1ea5..b3e8867f3 100644--- a/packages/eve/src/evals/runner/reporters/braintrust.test.ts+++ b/packages/eve/src/evals/runner/reporters/braintrust.test.ts@@ -66,8 +66,15 @@ function makeEvalResult(overrides: Partial<EveEvalResult> = {}): EveEvalResult {traceContexts: [],},assertions: [- { name: "succeeded", score: 1, severity: "gate", passed: true },- { name: "similarity", score: 1, severity: "soft", threshold: 0.6, passed: true },+ { name: "succeeded", score: 1, severity: "gate", passed: true, errored: false },+ {+ name: "similarity",+ score: 1,+ severity: "soft",+ threshold: 0.6,+ passed: true,+ errored: false,+ },],verdict: "passed",startedAt: "2026-01-01T00:00:00.000Z",@@ -146,6 +153,7 @@ describe("Braintrust", () => {},name: "judge.boolean",passed: false,+ errored: false,score: 0,severity: "soft",threshold: 0.8,@@ -168,6 +176,7 @@ describe("Braintrust", () => {},name: "judge.boolean",passed: false,+ errored: false,score: 0,severity: "soft",threshold: 0.8,@@ -212,8 +221,8 @@ describe("Braintrust", () => {reporter.onEvalComplete(makeEvalResult({assertions: [- { name: "similarity", passed: true, score: 0.8, severity: "soft" },- { name: "similarity", passed: true, score: 0.6, severity: "soft" },+ { name: "similarity", passed: true, score: 0.8, severity: "soft", errored: false },+ { name: "similarity", passed: true, score: 0.6, severity: "soft", errored: false },],}),);diff --git a/packages/eve/src/evals/runner/reporters/console.test.ts b/packages/eve/src/evals/runner/reporters/console.test.tsindex ae7dbaa7c..2b7cdf504 100644--- a/packages/eve/src/evals/runner/reporters/console.test.ts+++ b/packages/eve/src/evals/runner/reporters/console.test.ts@@ -31,6 +31,7 @@ describe("Console", () => {].join("\n"),name: "judge.boolean",passed: false,+ errored: false,score: 0,severity: "soft",threshold: 0.8,@@ -66,6 +67,7 @@ describe("Console", () => {metadata: { rationale: "Successful rationale details" },name: "judge.boolean",passed: true,+ errored: false,score: 1,severity: "gate",},diff --git a/packages/eve/src/evals/runner/reporters/datadog.test.ts b/packages/eve/src/evals/runner/reporters/datadog.test.tsindex a97f82c6b..ee906cfc7 100644--- a/packages/eve/src/evals/runner/reporters/datadog.test.ts+++ b/packages/eve/src/evals/runner/reporters/datadog.test.ts@@ -66,9 +66,16 @@ function makeEvalResult(overrides: Partial<EveEvalResult> = {}): EveEvalResult {traceContexts: [],},assertions: [- { name: "succeeded", score: 1, severity: "gate", passed: true },- { name: "similarity", score: 0.9, severity: "soft", threshold: 0.6, passed: true },- { name: "judge.boolean", score: 1, severity: "soft", passed: true },+ { name: "succeeded", score: 1, severity: "gate", passed: true, errored: false },+ {+ name: "similarity",+ score: 0.9,+ severity: "soft",+ threshold: 0.6,+ passed: true,+ errored: false,+ },+ { name: "judge.boolean", score: 1, severity: "soft", passed: true, errored: false },],verdict: "passed",startedAt: "2026-01-01T00:00:00.000Z",@@ -262,6 +269,7 @@ describe("Datadog", () => {score: 0,severity: "gate",passed: false,+ errored: false,},],verdict: "failed",@@ -298,6 +306,7 @@ describe("Datadog", () => {score: 0,severity: "gate",passed: false,+ errored: false,},],verdict: "failed",@@ -332,10 +341,10 @@ describe("Datadog", () => {const reporter = Datadog(config);const result = makeEvalResult({assertions: [- { name: "same label", score: 1, severity: "soft", passed: true },- { name: "same@label", score: 1, severity: "soft", passed: true },- { name: "same label", score: 1, severity: "soft", passed: true },- { name: "eve_tool_call_count", score: 1, severity: "soft", passed: true },+ { name: "same label", score: 1, severity: "soft", passed: true, errored: false },+ { name: "same@label", score: 1, severity: "soft", passed: true, errored: false },+ { name: "same label", score: 1, severity: "soft", passed: true, errored: false },+ { name: "eve_tool_call_count", score: 1, severity: "soft", passed: true, errored: false },],});diff --git a/packages/eve/src/evals/runner/reporters/junit.test.ts b/packages/eve/src/evals/runner/reporters/junit.test.tsindex d97112527..89a08b15a 100644--- a/packages/eve/src/evals/runner/reporters/junit.test.ts+++ b/packages/eve/src/evals/runner/reporters/junit.test.ts@@ -74,6 +74,7 @@ function makeSummary(): EveEvalRunSummary {score: 0,severity: "gate",passed: false,+ errored: false,message: "expected hello",},],@@ -102,6 +103,7 @@ function makeEvalResult(overrides: Partial<EveEvalResult> = {}): EveEvalResult {score: 1,severity: "gate",passed: true,+ errored: false,},],result: {diff --git a/packages/eve/src/evals/runner/verdict.test.ts b/packages/eve/src/evals/runner/verdict.test.tsindex e49128b27..84f13b17f 100644--- a/packages/eve/src/evals/runner/verdict.test.ts+++ b/packages/eve/src/evals/runner/verdict.test.ts@@ -9,6 +9,7 @@ function assertion(overrides: Partial<AssertionResult>): AssertionResult {score: 1,severity: "gate",passed: true,+ errored: false,...overrides,};}@@ -29,7 +30,7 @@ describe("computeEvalVerdict", () => {it("fails when a gate assertion failed", () => {expect(computeEvalVerdict({- assertions: [assertion({ severity: "gate", score: 0, passed: false })],+ assertions: [assertion({ severity: "gate", score: 0, passed: false, errored: false })],}),).toBe("failed");});@@ -38,8 +39,14 @@ describe("computeEvalVerdict", () => {expect(computeEvalVerdict({assertions: [- assertion({ severity: "gate", passed: true }),- assertion({ severity: "soft", threshold: 0.6, score: 0.3, passed: false }),+ assertion({ severity: "gate", passed: true, errored: false }),+ assertion({+ severity: "soft",+ threshold: 0.6,+ score: 0.3,+ passed: false,+ errored: false,+ }),],}),).toBe("scored");@@ -49,8 +56,14 @@ describe("computeEvalVerdict", () => {expect(computeEvalVerdict({assertions: [- assertion({ severity: "soft", threshold: 0.6, score: 0.3, passed: false }),- assertion({ severity: "gate", score: 0, passed: false }),+ assertion({+ severity: "soft",+ threshold: 0.6,+ score: 0.3,+ passed: false,+ errored: false,+ }),+ assertion({ severity: "gate", score: 0, passed: false, errored: false }),],}),).toBe("failed");@@ -59,7 +72,7 @@ describe("computeEvalVerdict", () => {it("passes when a tracked soft assertion has no threshold", () => {expect(computeEvalVerdict({- assertions: [assertion({ severity: "soft", score: 0.1, passed: true })],+ assertions: [assertion({ severity: "soft", score: 0.1, passed: true, errored: false })],}),).toBe("passed");});@@ -76,7 +89,7 @@ describe("computeEvalVerdict", () => {it("never lets an explicit skip mask errors or failed gates", () => {expect(computeEvalVerdict({- assertions: [assertion({ severity: "gate", score: 0, passed: false })],+ assertions: [assertion({ severity: "gate", score: 0, passed: false, errored: false })],error: "ignored",skipReason: "unsupported target",}),diff --git a/packages/eve/src/evals/types.ts b/packages/eve/src/evals/types.tsindex 6dd858727..7513b53d6 100644--- a/packages/eve/src/evals/types.ts+++ b/packages/eve/src/evals/types.ts@@ -213,6 +213,8 @@ export interface AssertionResult {readonly severity: AssertionSeverity;readonly threshold?: number;readonly passed: boolean;+ /** Whether the assertion failed because its scorer threw instead of producing a score. */+ readonly errored: boolean;/** Human-readable failure detail, shown in console output and artifacts. */readonly message?: string;readonly metadata?: Readonly<Record<string, unknown>>;