The Open Framework for Building Agents
fix(eve): evals - preserve autoevals judges with Braintrust (#1183)
Andrew Barbaa45e4cbc302c
diff --git a/.changeset/tall-frogs-judge.md b/.changeset/tall-frogs-judge.mdnew file mode 100644index 000000000..6e56c5c51--- /dev/null+++ b/.changeset/tall-frogs-judge.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Keep `t.judge.autoevals.*` assertions working when Braintrust reporting is active by preserving the judge client's OpenAI-compatible interface during autoevals wrapping checks.diff --git a/packages/eve/src/evals/autoevals-client.test.ts b/packages/eve/src/evals/autoevals-client.test.tsindex 01477c5e5..45b6cdac4 100644--- a/packages/eve/src/evals/autoevals-client.test.ts+++ b/packages/eve/src/evals/autoevals-client.test.ts@@ -1,4 +1,5 @@-import { describe, expect, it, vi } from "vitest";+import { afterEach, describe, expect, it, vi } from "vitest";+import { Factuality } from "autoevals";const mocks = vi.hoisted(() => ({ generateText: vi.fn() }));@@ -7,11 +8,25 @@ vi.mock("ai", () => ({jsonSchema: vi.fn(),}));-vi.mock("autoevals", () => ({ Factuality: vi.fn() }));-import { createAutoevalsClient } from "#evals/autoevals-client.js";+type BraintrustGlobal = typeof globalThis & {+ __inherited_braintrust_wrap_openai?: (client: unknown) => unknown;+};++const braintrustGlobal = globalThis as BraintrustGlobal;+const originalBraintrustWrapper = braintrustGlobal.__inherited_braintrust_wrap_openai;+describe("createAutoevalsClient", () => {+ afterEach(() => {+ mocks.generateText.mockReset();+ if (originalBraintrustWrapper === undefined) {+ delete braintrustGlobal.__inherited_braintrust_wrap_openai;+ } else {+ braintrustGlobal.__inherited_braintrust_wrap_openai = originalBraintrustWrapper;+ }+ });+it("identifies eve on Gateway-routed judge calls", async () => {mocks.generateText.mockResolvedValueOnce({ text: "ok", toolCalls: [] });const client = createAutoevalsClient({ languageModel: "openai/gpt-5.5" });@@ -24,4 +39,45 @@ describe("createAutoevalsClient", () => {}),);});++ it("retains a distinct create function when autoevals reconstructs it", () => {+ const client = createAutoevalsClient({ languageModel: "openai/gpt-5.5" });+ const Constructor = Object.getPrototypeOf(client).constructor as new (config: {+ apiKey: string;+ }) => typeof client;+ const reconstructed = new Constructor({ apiKey: "dummy" });++ expect(typeof reconstructed.chat.completions.create).toBe("function");+ expect(String(reconstructed.chat.completions.create)).not.toBe(+ String(client.chat.completions.create),+ );+ });++ it("runs a bundled autoevals judge while the Braintrust global wrapper is active", async () => {+ const { wrapOpenAI } = await import("braintrust");+ expect(braintrustGlobal.__inherited_braintrust_wrap_openai).toBe(wrapOpenAI);++ mocks.generateText.mockResolvedValueOnce({+ text: "",+ toolCalls: [+ {+ input: { choice: "C" },+ toolCallId: "choice-1",+ toolName: "select_choice",+ },+ ],+ });++ const result = await Factuality({+ client: createAutoevalsClient({ languageModel: "openai/gpt-5.5" }),+ expected: "Paris",+ input: "What is the capital of France?",+ model: "openai/gpt-5.5",+ output: "Paris",+ useCoT: false,+ });++ expect(result.score).toBe(1);+ expect(mocks.generateText).toHaveBeenCalledTimes(1);+ });});diff --git a/packages/eve/src/evals/autoevals-client.ts b/packages/eve/src/evals/autoevals-client.tsindex 19bbe2107..2ed47b724 100644--- a/packages/eve/src/evals/autoevals-client.ts+++ b/packages/eve/src/evals/autoevals-client.ts@@ -53,17 +53,47 @@ type ChatToolChoice =| "required"| { readonly type: "function"; readonly function: { readonly name: string } };-export function createAutoevalsClient(config: AutoevalsClientConfig): AutoevalsClient {- const adapter: unknown = {- chat: {+/**+ * Constructible OpenAI-shaped adapter for bundled autoevals graders.+ *+ * Braintrust installs a global wrapper that makes autoevals reconstruct the+ * supplied client through its prototype constructor. The reconstructed client+ * must retain `chat.completions.create`, and its probe function must differ+ * from the live function so autoevals does not wrap the AI SDK bridge.+ */+class AutoevalsOpenAIClientAdapter {+ readonly chat: {+ readonly completions: {+ readonly create: (params: ChatParams) => Promise<{ readonly choices: readonly unknown[] }>;+ };+ };++ constructor(config: AutoevalsClientConfig | { readonly apiKey?: string }) {+ this.chat = {completions: {- create: (params: ChatParams) => createChatCompletion(params, config),+ create: isAutoevalsClientConfig(config)+ ? (params) => createChatCompletion(params, config)+ : createProbeChatCompletion,},- },- };+ };+ }+}++export function createAutoevalsClient(config: AutoevalsClientConfig): AutoevalsClient {+ const adapter: unknown = new AutoevalsOpenAIClientAdapter(config);return adapter as AutoevalsClient;}+function isAutoevalsClientConfig(+ config: AutoevalsClientConfig | { readonly apiKey?: string },+): config is AutoevalsClientConfig {+ return "languageModel" in config;+}++async function createProbeChatCompletion(): Promise<{ readonly choices: readonly unknown[] }> {+ return { choices: [] };+}+async function createChatCompletion(params: ChatParams,config: AutoevalsClientConfig,