The Open Framework for Building Agents
fix(eve): log empty Braintrust output for evals without agent output (#4787)
Ben Pankow620022bf6f0e
diff --git a/.changeset/braintrust-null-output.md b/.changeset/braintrust-null-output.mdnew file mode 100644index 000000000..0d67e4966--- /dev/null+++ b/.changeset/braintrust-null-output.md@@ -0,0 +1,5 @@+---+"eve": patch+---++The Braintrust eval reporter no longer aborts the run when an eval produces no agent output. Those evals are now logged with an empty output.diff --git a/packages/eve/src/evals/runner/reporters/braintrust.test.ts b/packages/eve/src/evals/runner/reporters/braintrust.test.tsindex de88b1d4a..108bef178 100644--- a/packages/eve/src/evals/runner/reporters/braintrust.test.ts+++ b/packages/eve/src/evals/runner/reporters/braintrust.test.ts@@ -208,6 +208,20 @@ describe("Braintrust", () => {);});+ it("logs an empty output for an eval that produced no agent output", async () => {+ const reporter = Braintrust(makeConfig());+ await reporter.onRunStart([makeEval()], makeTarget());+ const result = makeEvalResult();++ reporter.onEvalComplete({+ ...result,+ result: { ...result.result, finalMessage: null, output: null },+ });++ // Braintrust rejects a null or undefined output with "output must be specified".+ expect(braintrustMocks.log).toHaveBeenCalledWith(expect.objectContaining({ output: "" }));+ });+it("scores repeated assertions by their lowest member and keeps each score in metadata", async () => {const reporter = Braintrust(makeConfig());await reporter.onRunStart([makeEval()], makeTarget());diff --git a/packages/eve/src/evals/runner/reporters/braintrust.ts b/packages/eve/src/evals/runner/reporters/braintrust.tsindex 0263e62ab..543f9f335 100644--- a/packages/eve/src/evals/runner/reporters/braintrust.ts+++ b/packages/eve/src/evals/runner/reporters/braintrust.ts@@ -182,7 +182,7 @@ class BraintrustReporter implements EvalReporter {this.#experiment.log({id: result.id,input: evaluation?.description ?? "",- output: result.result.output,+ output: result.result.output ?? "",error: result.error ?? undefined,scores,metadata,