Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(workbench): attribute judge evidence correctly

超超超超超级喜欢你的达妮娅a11cfcadf544
diff --git a/workbench/README.md b/workbench/README.md
index 0aaca20ee..f72a23225 100644
--- a/workbench/README.md
+++ b/workbench/README.md
@@ -79,7 +79,7 @@ sélectionner le client Responses natif ; un désaccord échoue avant tout trans
Un run Responses borné nomme toujours ses deux plafonds :
```bash
-node workbench/.build/cli.cjs run --scenario target-right-clip --reps 1 \
+node --env-file=.env.workbench workbench/.build/cli.cjs run --scenario target-right-clip --reps 1 \
--label <label> --record --timeout 180000 --max-requests 6 --max-output-tokens 2048
```
diff --git a/workbench/cli.ts b/workbench/cli.ts
index b862690ed..f4224ed62 100644
--- a/workbench/cli.ts
+++ b/workbench/cli.ts
@@ -510,6 +510,9 @@ async function commandJudge(options: Options): Promise<number> {
? transportIdentity({
wireApi: env.wireApi,
maxOutputTokens: options.maxOutputTokens ?? 2048,
+ // `askJudge` issues one raw fetch per verdict — no SDK, no retries.
+ // Stamp the policy that actually ran, never the agent's SDK default.
+ retryPolicy: { sdkMaxRetries: 0, repetitionRetries: 0 },
publicHeadersSha256: env.publicHeaders.sha256,
limits: {
maxRequests: options.maxRequests ?? 100,
@@ -559,9 +562,15 @@ async function commandJudge(options: Options): Promise<number> {
let replayHandle: ReplayHandle | null = null;
let endpoint: ModelServerHandle;
let judgeWireApi: "chat-completions" | "responses" = "chat-completions";
+ // The judge request hash covers the model: offline replay must send the
+ // model the cassette was RECORDED with, or every round looks stale and
+ // assertFresh() fails the replay against itself.
+ let replayModel: string | undefined;
try {
if (env === null) {
- judgeWireApi = readCassette(cassetteFile).wireApi ?? "chat-completions";
+ const replayCassette = readCassette(cassetteFile);
+ judgeWireApi = replayCassette.wireApi ?? "chat-completions";
+ replayModel = replayCassette.model;
replayHandle = await startReplay({ file: cassetteFile });
endpoint = replayHandle;
} else {
@@ -615,7 +624,7 @@ async function commandJudge(options: Options): Promise<number> {
const reading = await askJudge({
endpoint: {
baseUrl: endpoint.url,
- model: env?.model ?? "cassette",
+ model: env?.model ?? replayModel ?? "cassette",
...(env ? { apiKey: env.apiKey } : {}),
wireApi: judgeWireApi,
},
diff --git a/workbench/l0/persist.wb.ts b/workbench/l0/persist.wb.ts
index 699b17022..6c117b845 100644
--- a/workbench/l0/persist.wb.ts
+++ b/workbench/l0/persist.wb.ts
@@ -139,6 +139,20 @@ function readTurn(file: string): PersistedTurn {
}
describe("persistRepetition", () => {
+ it("retains the full answer for the judge pass", () => {
+ const long = `${"x".repeat(MAX_FIELD_CHARS)} tail-after-cut`;
+ const turn = buildPersistedTurn({
+ label: "probe",
+ result: repetition({ answer: long }),
+ prompt: scenario.prompt,
+ allowAgentEdits: true,
+ });
+ // The judge scores turn.answer and replay regenerates the full text:
+ // a truncated answer would make the recorded verdict evaluate different
+ // text than the measured turn, and the strict replay hash diverge.
+ expect(turn.answer).toBe(long);
+ expect(turn.truncated.join(" ")).not.toContain("answer");
+ });
it("writes one self-contained file per repetition", () => {
const root = scratch();
const written = persistRepetition({
diff --git a/workbench/l0/transport-policy.wb.ts b/workbench/l0/transport-policy.wb.ts
index 18c9c6949..b78d11927 100644
--- a/workbench/l0/transport-policy.wb.ts
+++ b/workbench/l0/transport-policy.wb.ts
@@ -57,6 +57,20 @@ function cloneIdentity(identity: TransportIdentity): TransportIdentity {
}
describe("workbench environment contract", () => {
+ it("lets the caller stamp the retry policy that actually ran, keeping the agent default intact", () => {
+ const agentDefault = transportIdentity({ wireApi: "chat-completions" });
+ expect(agentDefault.retryPolicy).toEqual({ sdkMaxRetries: "default", repetitionRetries: 2 });
+
+ // The judge issues one raw fetch per verdict — its identity must say so.
+ const judge = transportIdentity({
+ wireApi: "chat-completions",
+ retryPolicy: { sdkMaxRetries: 0, repetitionRetries: 0 },
+ });
+ expect(judge.retryPolicy).toEqual({ sdkMaxRetries: 0, repetitionRetries: 0 });
+ // Different policy -> different fingerprint: evidence cannot share identity
+ // with a transport that would have behaved differently.
+ expect(transportSha256(judge)).not.toBe(transportSha256(agentDefault));
+ });
it("uses the declared env fields and defaults to legacy Chat mode", () => {
expect(Object.keys(ENV_KEYS)).toEqual(envFields);
diff --git a/workbench/l1/cassette.wb.ts b/workbench/l1/cassette.wb.ts
index 0e1371d13..16ff890a9 100644
--- a/workbench/l1/cassette.wb.ts
+++ b/workbench/l1/cassette.wb.ts
@@ -21,6 +21,7 @@ import {
startRecorder,
startReplay,
usageFromSse,
+ validateCassette,
writeCassette,
} from "../lib/cassette";
import { ENV_KEYS } from "../lib/env";
@@ -28,6 +29,7 @@ import { singleClip } from "../lib/fixtures";
import { normalizeIds, runScenario } from "../lib/harness";
import { startScriptedModel } from "../lib/model-server";
import { adoptCassetteForRetainedAttempt } from "../lib/runner";
+import { transportIdentity } from "../lib/transport";
const DIRECTORY = mkdtempSync(join(tmpdir(), "wb-cassette-"));
const FILE = join(DIRECTORY, "wizard.json");
@@ -97,6 +99,80 @@ async function record() {
}
describe("record then replay", () => {
+ it("accepts a Chat cassette stamped with its true zero-retry judge policy", () => {
+ // A judge cassette records the policy that actually ran (one raw fetch per
+ // verdict). Reconstructing the identity from the recorded transport must use
+ // that policy, not the agent SDK default, or every judge recording fails.
+ const transport = transportIdentity({
+ wireApi: "chat-completions",
+ retryPolicy: { sdkMaxRetries: 0, repetitionRetries: 0 },
+ });
+ expect(() =>
+ validateCassette({
+ scenario: "judge-probe",
+ provider: "openai-compatible",
+ model: "workbench-loopback-fixture",
+ resolvedModel: "workbench",
+ recordedAt: "2026-09-19T00:00:00.000Z",
+ rounds: [
+ {
+ round: 0,
+ requestHash: "0".repeat(16),
+ digest: { systemChars: 1, toolCount: 0, roles: [], lastUserText: "" },
+ sse: 'data: {"model":"workbench"}\n\n',
+ },
+ ],
+ attempts: [],
+ wireApi: "chat-completions",
+ transport,
+ } as never),
+ ).not.toThrow();
+ });
+
+ it("replays a judge cassette only with the model it was recorded with", async () => {
+ // The judge request hash covers the model field. Offline replay that sends
+ // a placeholder model marks every round stale and fails against itself —
+ // the recorded model is the one the replay must send.
+ const DIRECTORY = mkdtempSync(join(tmpdir(), "wb-jmodel-"));
+ const file = join(DIRECTORY, "judge.json");
+ const upstream = await startScriptedModel([{ kind: "text", text: "ok" }]);
+ const recorder = await startRecorder({
+ upstream: upstream.url,
+ file,
+ scenario: "judge-model-probe",
+ provider: "loopback",
+ model: "recorded-model-x",
+ // Transport-bound: only then does the request hash cover the model —
+ // which is exactly how the CLI's live judge transport records.
+ wireApi: "chat-completions",
+ transport: transportIdentity({ wireApi: "chat-completions" }),
+ });
+ const ask = (url: string, model: string) =>
+ fetch(`${url}/chat/completions`, {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ model, messages: [{ role: "user", content: "v" }] }),
+ }).then((response) => response.text());
+ try {
+ await ask(recorder.url, "recorded-model-x");
+ } finally {
+ recorder.close();
+ upstream.close();
+ }
+ for (const [model, expectStale] of [
+ ["recorded-model-x", false],
+ ["cassette", true],
+ ] as const) {
+ const replay = await startReplay({ file });
+ try {
+ await ask(replay.url, model);
+ } finally {
+ replay.close();
+ }
+ expect(replay.staleRounds.length > 0).toBe(expectStale);
+ }
+ rmSync(DIRECTORY, { recursive: true, force: true });
+ });
it("replaying reproduces the recorded document exactly", async () => {
const recorded = await record();
expect(recorded.ok).toBe(true);
diff --git a/workbench/lib/cassette.ts b/workbench/lib/cassette.ts
index 3cfd01150..cd1d8fd83 100644
--- a/workbench/lib/cassette.ts
+++ b/workbench/lib/cassette.ts
@@ -362,6 +362,12 @@ export function validateCassette(cassette: Cassette): void {
: undefined,
limits: cassette.transport.limits,
publicHeadersSha256: cassette.transport.publicHeadersSha256,
+ // Reconstruct the policy that was RECORDED, not the factory default — a
+ // judge cassette stamped with its true zero-retry policy must survive
+ // replay instead of failing the identity check here.
+ ...(declaredWireApi === "chat-completions"
+ ? { retryPolicy: cassette.transport.retryPolicy }
+ : {}),
});
if (JSON.stringify(expected) !== JSON.stringify(cassette.transport)) {
throw new Error(`${declaredWireApi} cassette transport identity is invalid`);
diff --git a/workbench/lib/persist.ts b/workbench/lib/persist.ts
index ae67d6523..8a6239aab 100644
--- a/workbench/lib/persist.ts
+++ b/workbench/lib/persist.ts
@@ -190,7 +190,10 @@ export function buildPersistedTurn(options: BuildPersistedTurnOptions): Persiste
ms: result.run.ms,
failureClass: result.scored.failureClass,
},
- answer: cut(result.run.answer, "answer", truncated),
+ // The judge scores this field and replay regenerates the FULL answer:
+ // truncating it here would make the recorded verdict evaluate different
+ // text than the measured turn, and the strict replay hash diverge.
+ answer: result.run.answer,
mutated: result.context.mutated,
wire: {
rounds: wire.rounds,
diff --git a/workbench/lib/transport.ts b/workbench/lib/transport.ts
index b2aa747d1..819f5a098 100644
--- a/workbench/lib/transport.ts
+++ b/workbench/lib/transport.ts
@@ -75,6 +75,10 @@ export function transportIdentity(options: {
maxOutputTokens?: number;
limits?: Partial<TransportLimits>;
publicHeadersSha256?: string;
+ /** The retry policy ACTUALLY executed by the caller, not the SDK default. The
+ * chat branch's fallback describes the agent's SDK path; a judge that issues
+ * one raw fetch per verdict must stamp zero retries instead. */
+ retryPolicy?: TransportIdentity["retryPolicy"];
}): TransportIdentity {
const limits = { ...DEFAULT_TRANSPORT_LIMITS, ...options.limits };
if (options.publicHeadersSha256 && !/^[a-f0-9]{64}$/.test(options.publicHeadersSha256)) {
@@ -115,7 +119,7 @@ export function transportIdentity(options: {
maxOutputTokens: null,
reasoning: "unchanged",
},
- retryPolicy: { sdkMaxRetries: "default", repetitionRetries: 2 },
+ retryPolicy: options.retryPolicy ?? { sdkMaxRetries: "default", repetitionRetries: 2 },
limits,
publicHeadersSha256: options.publicHeadersSha256 ?? sha256Bytes("[]"),
};