Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(evals): export labeled assertions under their label (#4764)

Matan Kushnera4af30176dd2
diff --git a/.changeset/score-label-name.md b/.changeset/score-label-name.md
new file mode 100644
index 000000000..c484c47fd
--- /dev/null
+++ b/.changeset/score-label-name.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Labeled assertions now report to Braintrust and Datadog under their label, so `t.score(x).label("grounding")` logs as `grounding` instead of `score [grounding]` or `score_grounding`. Display names in the console and run artifacts are unchanged, and labeled metrics start a new series under the bare label.
diff --git a/packages/eve/src/evals/runner/reporters/assertion-scores.ts b/packages/eve/src/evals/runner/reporters/assertion-scores.ts
index 449745d80..596193154 100644
--- a/packages/eve/src/evals/runner/reporters/assertion-scores.ts
+++ b/packages/eve/src/evals/runner/reporters/assertion-scores.ts
@@ -1,13 +1,16 @@
import type { AssertionResult } from "#evals/types.js";
/**
- * The name an assertion is exported under. Gates carry a `gate:` prefix so
- * they do not share a column with a soft assertion of the same name, and so
- * experiments diff gate regressions the same way they diff soft scores.
- * Sinks with stricter label rules normalize this name themselves.
+ * The name an assertion is exported under: its label when it has one, so a
+ * labeled assertion keeps a stable column however its display name is built.
+ * Gates carry a `gate:` prefix so they do not share a column with a soft
+ * assertion of the same name, and so experiments diff gate regressions the
+ * same way they diff soft scores. Sinks with stricter label rules normalize
+ * this name themselves.
*/
export function exportedAssertionName(assertion: AssertionResult): string {
- return assertion.severity === "gate" ? `gate:${assertion.name}` : assertion.name;
+ const name = assertion.key ?? assertion.name;
+ return assertion.severity === "gate" ? `gate:${name}` : name;
}
/** Assertions reported under one name, scored together. */
diff --git a/packages/eve/src/evals/runner/reporters/braintrust.test.ts b/packages/eve/src/evals/runner/reporters/braintrust.test.ts
index 07a4498d2..de88b1d4a 100644
--- a/packages/eve/src/evals/runner/reporters/braintrust.test.ts
+++ b/packages/eve/src/evals/runner/reporters/braintrust.test.ts
@@ -235,4 +235,34 @@ describe("Braintrust", () => {
}),
);
});
+
+ it("logs labeled assertions under their label", async () => {
+ const reporter = Braintrust(makeConfig());
+ await reporter.onRunStart([makeEval()], makeTarget());
+
+ reporter.onEvalComplete(
+ makeEvalResult({
+ assertions: [
+ {
+ name: "score [grounding]",
+ key: "grounding",
+ score: 0.8,
+ severity: "soft",
+ errored: false,
+ },
+ {
+ name: "judge.boolean [citation]",
+ key: "citation",
+ score: 1,
+ severity: "gate",
+ errored: false,
+ },
+ ],
+ }),
+ );
+
+ expect(braintrustMocks.log).toHaveBeenCalledWith(
+ expect.objectContaining({ scores: { grounding: 0.8, "gate:citation": 1 } }),
+ );
+ });
});
diff --git a/packages/eve/src/evals/runner/reporters/datadog.test.ts b/packages/eve/src/evals/runner/reporters/datadog.test.ts
index b7f5afce1..dfc0f9405 100644
--- a/packages/eve/src/evals/runner/reporters/datadog.test.ts
+++ b/packages/eve/src/evals/runner/reporters/datadog.test.ts
@@ -484,6 +484,41 @@ describe("Datadog", () => {
]);
});
+ it("names labeled assertion metrics by their label", async () => {
+ const { config, experiment } = makeConfig({ recordInputs: false });
+ const reporter = Datadog(config);
+ const result = makeEvalResult({
+ assertions: [
+ {
+ name: "score [grounding]",
+ key: "grounding",
+ score: 0.8,
+ severity: "soft",
+ errored: false,
+ },
+ {
+ name: "judge.boolean [citation]",
+ key: "citation",
+ score: 1,
+ severity: "gate",
+ threshold: 1,
+ passed: true,
+ errored: false,
+ },
+ ],
+ });
+
+ await reporter.onRunStart([makeEval()], makeTarget());
+ await reporter.onEvalComplete(result);
+
+ expect(experiment.submitEvaluationMetrics.mock.calls[0]?.[1]).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ label: "grounding", value: 0.8 }),
+ expect.objectContaining({ label: "gate_citation", value: 1 }),
+ ]),
+ );
+ });
+
it("omits assertion_passed when no member of a group has a verdict", async () => {
const { config, experiment } = makeConfig({ recordInputs: false });
const reporter = Datadog(config);