Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): evals - preserve autoevals judges with Braintrust (#1183)

Andrew Barbaa45e4cbc302c
diff --git a/.changeset/tall-frogs-judge.md b/.changeset/tall-frogs-judge.md
new file mode 100644
index 000000000..6e56c5c51
--- /dev/null
+++ b/.changeset/tall-frogs-judge.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Keep `t.judge.autoevals.*` assertions working when Braintrust reporting is active by preserving the judge client's OpenAI-compatible interface during autoevals wrapping checks.
diff --git a/packages/eve/src/evals/autoevals-client.test.ts b/packages/eve/src/evals/autoevals-client.test.ts
index 01477c5e5..45b6cdac4 100644
--- a/packages/eve/src/evals/autoevals-client.test.ts
+++ b/packages/eve/src/evals/autoevals-client.test.ts
@@ -1,4 +1,5 @@
-import { describe, expect, it, vi } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { Factuality } from "autoevals";
const mocks = vi.hoisted(() => ({ generateText: vi.fn() }));
@@ -7,11 +8,25 @@ vi.mock("ai", () => ({
jsonSchema: vi.fn(),
}));
-vi.mock("autoevals", () => ({ Factuality: vi.fn() }));
-
import { createAutoevalsClient } from "#evals/autoevals-client.js";
+type BraintrustGlobal = typeof globalThis & {
+ __inherited_braintrust_wrap_openai?: (client: unknown) => unknown;
+};
+
+const braintrustGlobal = globalThis as BraintrustGlobal;
+const originalBraintrustWrapper = braintrustGlobal.__inherited_braintrust_wrap_openai;
+
describe("createAutoevalsClient", () => {
+ afterEach(() => {
+ mocks.generateText.mockReset();
+ if (originalBraintrustWrapper === undefined) {
+ delete braintrustGlobal.__inherited_braintrust_wrap_openai;
+ } else {
+ braintrustGlobal.__inherited_braintrust_wrap_openai = originalBraintrustWrapper;
+ }
+ });
+
it("identifies eve on Gateway-routed judge calls", async () => {
mocks.generateText.mockResolvedValueOnce({ text: "ok", toolCalls: [] });
const client = createAutoevalsClient({ languageModel: "openai/gpt-5.5" });
@@ -24,4 +39,45 @@ describe("createAutoevalsClient", () => {
}),
);
});
+
+ it("retains a distinct create function when autoevals reconstructs it", () => {
+ const client = createAutoevalsClient({ languageModel: "openai/gpt-5.5" });
+ const Constructor = Object.getPrototypeOf(client).constructor as new (config: {
+ apiKey: string;
+ }) => typeof client;
+ const reconstructed = new Constructor({ apiKey: "dummy" });
+
+ expect(typeof reconstructed.chat.completions.create).toBe("function");
+ expect(String(reconstructed.chat.completions.create)).not.toBe(
+ String(client.chat.completions.create),
+ );
+ });
+
+ it("runs a bundled autoevals judge while the Braintrust global wrapper is active", async () => {
+ const { wrapOpenAI } = await import("braintrust");
+ expect(braintrustGlobal.__inherited_braintrust_wrap_openai).toBe(wrapOpenAI);
+
+ mocks.generateText.mockResolvedValueOnce({
+ text: "",
+ toolCalls: [
+ {
+ input: { choice: "C" },
+ toolCallId: "choice-1",
+ toolName: "select_choice",
+ },
+ ],
+ });
+
+ const result = await Factuality({
+ client: createAutoevalsClient({ languageModel: "openai/gpt-5.5" }),
+ expected: "Paris",
+ input: "What is the capital of France?",
+ model: "openai/gpt-5.5",
+ output: "Paris",
+ useCoT: false,
+ });
+
+ expect(result.score).toBe(1);
+ expect(mocks.generateText).toHaveBeenCalledTimes(1);
+ });
});
diff --git a/packages/eve/src/evals/autoevals-client.ts b/packages/eve/src/evals/autoevals-client.ts
index 19bbe2107..2ed47b724 100644
--- a/packages/eve/src/evals/autoevals-client.ts
+++ b/packages/eve/src/evals/autoevals-client.ts
@@ -53,17 +53,47 @@ type ChatToolChoice =
| "required"
| { readonly type: "function"; readonly function: { readonly name: string } };
-export function createAutoevalsClient(config: AutoevalsClientConfig): AutoevalsClient {
- const adapter: unknown = {
- chat: {
+/**
+ * Constructible OpenAI-shaped adapter for bundled autoevals graders.
+ *
+ * Braintrust installs a global wrapper that makes autoevals reconstruct the
+ * supplied client through its prototype constructor. The reconstructed client
+ * must retain `chat.completions.create`, and its probe function must differ
+ * from the live function so autoevals does not wrap the AI SDK bridge.
+ */
+class AutoevalsOpenAIClientAdapter {
+ readonly chat: {
+ readonly completions: {
+ readonly create: (params: ChatParams) => Promise<{ readonly choices: readonly unknown[] }>;
+ };
+ };
+
+ constructor(config: AutoevalsClientConfig | { readonly apiKey?: string }) {
+ this.chat = {
completions: {
- create: (params: ChatParams) => createChatCompletion(params, config),
+ create: isAutoevalsClientConfig(config)
+ ? (params) => createChatCompletion(params, config)
+ : createProbeChatCompletion,
},
- },
- };
+ };
+ }
+}
+
+export function createAutoevalsClient(config: AutoevalsClientConfig): AutoevalsClient {
+ const adapter: unknown = new AutoevalsOpenAIClientAdapter(config);
return adapter as AutoevalsClient;
}
+function isAutoevalsClientConfig(
+ config: AutoevalsClientConfig | { readonly apiKey?: string },
+): config is AutoevalsClientConfig {
+ return "languageModel" in config;
+}
+
+async function createProbeChatCompletion(): Promise<{ readonly choices: readonly unknown[] }> {
+ return { choices: [] };
+}
+
async function createChatCompletion(
params: ChatParams,
config: AutoevalsClientConfig,