The Open Framework for Building Agents
fix(evals): export labeled assertions under their label (#4764)
Matan Kushnera4af30176dd2
diff --git a/.changeset/score-label-name.md b/.changeset/score-label-name.mdnew file mode 100644index 000000000..c484c47fd--- /dev/null+++ b/.changeset/score-label-name.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Labeled assertions now report to Braintrust and Datadog under their label, so `t.score(x).label("grounding")` logs as `grounding` instead of `score [grounding]` or `score_grounding`. Display names in the console and run artifacts are unchanged, and labeled metrics start a new series under the bare label.diff --git a/packages/eve/src/evals/runner/reporters/assertion-scores.ts b/packages/eve/src/evals/runner/reporters/assertion-scores.tsindex 449745d80..596193154 100644--- a/packages/eve/src/evals/runner/reporters/assertion-scores.ts+++ b/packages/eve/src/evals/runner/reporters/assertion-scores.ts@@ -1,13 +1,16 @@import type { AssertionResult } from "#evals/types.js";/**- * The name an assertion is exported under. Gates carry a `gate:` prefix so- * they do not share a column with a soft assertion of the same name, and so- * experiments diff gate regressions the same way they diff soft scores.- * Sinks with stricter label rules normalize this name themselves.+ * The name an assertion is exported under: its label when it has one, so a+ * labeled assertion keeps a stable column however its display name is built.+ * Gates carry a `gate:` prefix so they do not share a column with a soft+ * assertion of the same name, and so experiments diff gate regressions the+ * same way they diff soft scores. Sinks with stricter label rules normalize+ * this name themselves.*/export function exportedAssertionName(assertion: AssertionResult): string {- return assertion.severity === "gate" ? `gate:${assertion.name}` : assertion.name;+ const name = assertion.key ?? assertion.name;+ return assertion.severity === "gate" ? `gate:${name}` : name;}/** Assertions reported under one name, scored together. */diff --git a/packages/eve/src/evals/runner/reporters/braintrust.test.ts b/packages/eve/src/evals/runner/reporters/braintrust.test.tsindex 07a4498d2..de88b1d4a 100644--- a/packages/eve/src/evals/runner/reporters/braintrust.test.ts+++ b/packages/eve/src/evals/runner/reporters/braintrust.test.ts@@ -235,4 +235,34 @@ describe("Braintrust", () => {}),);});++ it("logs labeled assertions under their label", async () => {+ const reporter = Braintrust(makeConfig());+ await reporter.onRunStart([makeEval()], makeTarget());++ reporter.onEvalComplete(+ makeEvalResult({+ assertions: [+ {+ name: "score [grounding]",+ key: "grounding",+ score: 0.8,+ severity: "soft",+ errored: false,+ },+ {+ name: "judge.boolean [citation]",+ key: "citation",+ score: 1,+ severity: "gate",+ errored: false,+ },+ ],+ }),+ );++ expect(braintrustMocks.log).toHaveBeenCalledWith(+ expect.objectContaining({ scores: { grounding: 0.8, "gate:citation": 1 } }),+ );+ });});diff --git a/packages/eve/src/evals/runner/reporters/datadog.test.ts b/packages/eve/src/evals/runner/reporters/datadog.test.tsindex b7f5afce1..dfc0f9405 100644--- a/packages/eve/src/evals/runner/reporters/datadog.test.ts+++ b/packages/eve/src/evals/runner/reporters/datadog.test.ts@@ -484,6 +484,41 @@ describe("Datadog", () => {]);});+ it("names labeled assertion metrics by their label", async () => {+ const { config, experiment } = makeConfig({ recordInputs: false });+ const reporter = Datadog(config);+ const result = makeEvalResult({+ assertions: [+ {+ name: "score [grounding]",+ key: "grounding",+ score: 0.8,+ severity: "soft",+ errored: false,+ },+ {+ name: "judge.boolean [citation]",+ key: "citation",+ score: 1,+ severity: "gate",+ threshold: 1,+ passed: true,+ errored: false,+ },+ ],+ });++ await reporter.onRunStart([makeEval()], makeTarget());+ await reporter.onEvalComplete(result);++ expect(experiment.submitEvaluationMetrics.mock.calls[0]?.[1]).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ label: "grounding", value: 0.8 }),+ expect.objectContaining({ label: "gate_citation", value: 1 }),+ ]),+ );+ });+it("omits assertion_passed when no member of a group has a verdict", async () => {const { config, experiment } = makeConfig({ recordInputs: false });const reporter = Datadog(config);