Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): log empty Braintrust output for evals without agent output (#4787)

Ben Pankow620022bf6f0e
diff --git a/.changeset/braintrust-null-output.md b/.changeset/braintrust-null-output.md
new file mode 100644
index 000000000..0d67e4966
--- /dev/null
+++ b/.changeset/braintrust-null-output.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+The Braintrust eval reporter no longer aborts the run when an eval produces no agent output. Those evals are now logged with an empty output.
diff --git a/packages/eve/src/evals/runner/reporters/braintrust.test.ts b/packages/eve/src/evals/runner/reporters/braintrust.test.ts
index de88b1d4a..108bef178 100644
--- a/packages/eve/src/evals/runner/reporters/braintrust.test.ts
+++ b/packages/eve/src/evals/runner/reporters/braintrust.test.ts
@@ -208,6 +208,20 @@ describe("Braintrust", () => {
);
});
+ it("logs an empty output for an eval that produced no agent output", async () => {
+ const reporter = Braintrust(makeConfig());
+ await reporter.onRunStart([makeEval()], makeTarget());
+ const result = makeEvalResult();
+
+ reporter.onEvalComplete({
+ ...result,
+ result: { ...result.result, finalMessage: null, output: null },
+ });
+
+ // Braintrust rejects a null or undefined output with "output must be specified".
+ expect(braintrustMocks.log).toHaveBeenCalledWith(expect.objectContaining({ output: "" }));
+ });
+
it("scores repeated assertions by their lowest member and keeps each score in metadata", async () => {
const reporter = Braintrust(makeConfig());
await reporter.onRunStart([makeEval()], makeTarget());
diff --git a/packages/eve/src/evals/runner/reporters/braintrust.ts b/packages/eve/src/evals/runner/reporters/braintrust.ts
index 0263e62ab..543f9f335 100644
--- a/packages/eve/src/evals/runner/reporters/braintrust.ts
+++ b/packages/eve/src/evals/runner/reporters/braintrust.ts
@@ -182,7 +182,7 @@ class BraintrustReporter implements EvalReporter {
this.#experiment.log({
id: result.id,
input: evaluation?.description ?? "",
- output: result.result.output,
+ output: result.result.output ?? "",
error: result.error ?? undefined,
scores,
metadata,