Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(workbench): attribute judge evidence correctly
超超超超超级喜欢你的达妮娅a11cfcadf544
diff --git a/workbench/README.md b/workbench/README.mdindex 0aaca20ee..f72a23225 100644--- a/workbench/README.md+++ b/workbench/README.md@@ -79,7 +79,7 @@ sélectionner le client Responses natif ; un désaccord échoue avant tout transUn run Responses borné nomme toujours ses deux plafonds :```bash-node workbench/.build/cli.cjs run --scenario target-right-clip --reps 1 \+node --env-file=.env.workbench workbench/.build/cli.cjs run --scenario target-right-clip --reps 1 \--label <label> --record --timeout 180000 --max-requests 6 --max-output-tokens 2048```diff --git a/workbench/cli.ts b/workbench/cli.tsindex b862690ed..f4224ed62 100644--- a/workbench/cli.ts+++ b/workbench/cli.ts@@ -510,6 +510,9 @@ async function commandJudge(options: Options): Promise<number> {? transportIdentity({wireApi: env.wireApi,maxOutputTokens: options.maxOutputTokens ?? 2048,+ // `askJudge` issues one raw fetch per verdict — no SDK, no retries.+ // Stamp the policy that actually ran, never the agent's SDK default.+ retryPolicy: { sdkMaxRetries: 0, repetitionRetries: 0 },publicHeadersSha256: env.publicHeaders.sha256,limits: {maxRequests: options.maxRequests ?? 100,@@ -559,9 +562,15 @@ async function commandJudge(options: Options): Promise<number> {let replayHandle: ReplayHandle | null = null;let endpoint: ModelServerHandle;let judgeWireApi: "chat-completions" | "responses" = "chat-completions";+ // The judge request hash covers the model: offline replay must send the+ // model the cassette was RECORDED with, or every round looks stale and+ // assertFresh() fails the replay against itself.+ let replayModel: string | undefined;try {if (env === null) {- judgeWireApi = readCassette(cassetteFile).wireApi ?? "chat-completions";+ const replayCassette = readCassette(cassetteFile);+ judgeWireApi = replayCassette.wireApi ?? "chat-completions";+ replayModel = replayCassette.model;replayHandle = await startReplay({ file: cassetteFile });endpoint = replayHandle;} else {@@ -615,7 +624,7 @@ async function commandJudge(options: Options): Promise<number> {const reading = await askJudge({endpoint: {baseUrl: endpoint.url,- model: env?.model ?? "cassette",+ model: env?.model ?? replayModel ?? "cassette",...(env ? { apiKey: env.apiKey } : {}),wireApi: judgeWireApi,},diff --git a/workbench/l0/persist.wb.ts b/workbench/l0/persist.wb.tsindex 699b17022..6c117b845 100644--- a/workbench/l0/persist.wb.ts+++ b/workbench/l0/persist.wb.ts@@ -139,6 +139,20 @@ function readTurn(file: string): PersistedTurn {}describe("persistRepetition", () => {+ it("retains the full answer for the judge pass", () => {+ const long = `${"x".repeat(MAX_FIELD_CHARS)} tail-after-cut`;+ const turn = buildPersistedTurn({+ label: "probe",+ result: repetition({ answer: long }),+ prompt: scenario.prompt,+ allowAgentEdits: true,+ });+ // The judge scores turn.answer and replay regenerates the full text:+ // a truncated answer would make the recorded verdict evaluate different+ // text than the measured turn, and the strict replay hash diverge.+ expect(turn.answer).toBe(long);+ expect(turn.truncated.join(" ")).not.toContain("answer");+ });it("writes one self-contained file per repetition", () => {const root = scratch();const written = persistRepetition({diff --git a/workbench/l0/transport-policy.wb.ts b/workbench/l0/transport-policy.wb.tsindex 18c9c6949..b78d11927 100644--- a/workbench/l0/transport-policy.wb.ts+++ b/workbench/l0/transport-policy.wb.ts@@ -57,6 +57,20 @@ function cloneIdentity(identity: TransportIdentity): TransportIdentity {}describe("workbench environment contract", () => {+ it("lets the caller stamp the retry policy that actually ran, keeping the agent default intact", () => {+ const agentDefault = transportIdentity({ wireApi: "chat-completions" });+ expect(agentDefault.retryPolicy).toEqual({ sdkMaxRetries: "default", repetitionRetries: 2 });++ // The judge issues one raw fetch per verdict — its identity must say so.+ const judge = transportIdentity({+ wireApi: "chat-completions",+ retryPolicy: { sdkMaxRetries: 0, repetitionRetries: 0 },+ });+ expect(judge.retryPolicy).toEqual({ sdkMaxRetries: 0, repetitionRetries: 0 });+ // Different policy -> different fingerprint: evidence cannot share identity+ // with a transport that would have behaved differently.+ expect(transportSha256(judge)).not.toBe(transportSha256(agentDefault));+ });it("uses the declared env fields and defaults to legacy Chat mode", () => {expect(Object.keys(ENV_KEYS)).toEqual(envFields);diff --git a/workbench/l1/cassette.wb.ts b/workbench/l1/cassette.wb.tsindex 0e1371d13..16ff890a9 100644--- a/workbench/l1/cassette.wb.ts+++ b/workbench/l1/cassette.wb.ts@@ -21,6 +21,7 @@ import {startRecorder,startReplay,usageFromSse,+ validateCassette,writeCassette,} from "../lib/cassette";import { ENV_KEYS } from "../lib/env";@@ -28,6 +29,7 @@ import { singleClip } from "../lib/fixtures";import { normalizeIds, runScenario } from "../lib/harness";import { startScriptedModel } from "../lib/model-server";import { adoptCassetteForRetainedAttempt } from "../lib/runner";+import { transportIdentity } from "../lib/transport";const DIRECTORY = mkdtempSync(join(tmpdir(), "wb-cassette-"));const FILE = join(DIRECTORY, "wizard.json");@@ -97,6 +99,80 @@ async function record() {}describe("record then replay", () => {+ it("accepts a Chat cassette stamped with its true zero-retry judge policy", () => {+ // A judge cassette records the policy that actually ran (one raw fetch per+ // verdict). Reconstructing the identity from the recorded transport must use+ // that policy, not the agent SDK default, or every judge recording fails.+ const transport = transportIdentity({+ wireApi: "chat-completions",+ retryPolicy: { sdkMaxRetries: 0, repetitionRetries: 0 },+ });+ expect(() =>+ validateCassette({+ scenario: "judge-probe",+ provider: "openai-compatible",+ model: "workbench-loopback-fixture",+ resolvedModel: "workbench",+ recordedAt: "2026-09-19T00:00:00.000Z",+ rounds: [+ {+ round: 0,+ requestHash: "0".repeat(16),+ digest: { systemChars: 1, toolCount: 0, roles: [], lastUserText: "" },+ sse: 'data: {"model":"workbench"}\n\n',+ },+ ],+ attempts: [],+ wireApi: "chat-completions",+ transport,+ } as never),+ ).not.toThrow();+ });++ it("replays a judge cassette only with the model it was recorded with", async () => {+ // The judge request hash covers the model field. Offline replay that sends+ // a placeholder model marks every round stale and fails against itself —+ // the recorded model is the one the replay must send.+ const DIRECTORY = mkdtempSync(join(tmpdir(), "wb-jmodel-"));+ const file = join(DIRECTORY, "judge.json");+ const upstream = await startScriptedModel([{ kind: "text", text: "ok" }]);+ const recorder = await startRecorder({+ upstream: upstream.url,+ file,+ scenario: "judge-model-probe",+ provider: "loopback",+ model: "recorded-model-x",+ // Transport-bound: only then does the request hash cover the model —+ // which is exactly how the CLI's live judge transport records.+ wireApi: "chat-completions",+ transport: transportIdentity({ wireApi: "chat-completions" }),+ });+ const ask = (url: string, model: string) =>+ fetch(`${url}/chat/completions`, {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({ model, messages: [{ role: "user", content: "v" }] }),+ }).then((response) => response.text());+ try {+ await ask(recorder.url, "recorded-model-x");+ } finally {+ recorder.close();+ upstream.close();+ }+ for (const [model, expectStale] of [+ ["recorded-model-x", false],+ ["cassette", true],+ ] as const) {+ const replay = await startReplay({ file });+ try {+ await ask(replay.url, model);+ } finally {+ replay.close();+ }+ expect(replay.staleRounds.length > 0).toBe(expectStale);+ }+ rmSync(DIRECTORY, { recursive: true, force: true });+ });it("replaying reproduces the recorded document exactly", async () => {const recorded = await record();expect(recorded.ok).toBe(true);diff --git a/workbench/lib/cassette.ts b/workbench/lib/cassette.tsindex 3cfd01150..cd1d8fd83 100644--- a/workbench/lib/cassette.ts+++ b/workbench/lib/cassette.ts@@ -362,6 +362,12 @@ export function validateCassette(cassette: Cassette): void {: undefined,limits: cassette.transport.limits,publicHeadersSha256: cassette.transport.publicHeadersSha256,+ // Reconstruct the policy that was RECORDED, not the factory default — a+ // judge cassette stamped with its true zero-retry policy must survive+ // replay instead of failing the identity check here.+ ...(declaredWireApi === "chat-completions"+ ? { retryPolicy: cassette.transport.retryPolicy }+ : {}),});if (JSON.stringify(expected) !== JSON.stringify(cassette.transport)) {throw new Error(`${declaredWireApi} cassette transport identity is invalid`);diff --git a/workbench/lib/persist.ts b/workbench/lib/persist.tsindex ae67d6523..8a6239aab 100644--- a/workbench/lib/persist.ts+++ b/workbench/lib/persist.ts@@ -190,7 +190,10 @@ export function buildPersistedTurn(options: BuildPersistedTurnOptions): Persistems: result.run.ms,failureClass: result.scored.failureClass,},- answer: cut(result.run.answer, "answer", truncated),+ // The judge scores this field and replay regenerates the FULL answer:+ // truncating it here would make the recorded verdict evaluate different+ // text than the measured turn, and the strict replay hash diverge.+ answer: result.run.answer,mutated: result.context.mutated,wire: {rounds: wire.rounds,diff --git a/workbench/lib/transport.ts b/workbench/lib/transport.tsindex b2aa747d1..819f5a098 100644--- a/workbench/lib/transport.ts+++ b/workbench/lib/transport.ts@@ -75,6 +75,10 @@ export function transportIdentity(options: {maxOutputTokens?: number;limits?: Partial<TransportLimits>;publicHeadersSha256?: string;+ /** The retry policy ACTUALLY executed by the caller, not the SDK default. The+ * chat branch's fallback describes the agent's SDK path; a judge that issues+ * one raw fetch per verdict must stamp zero retries instead. */+ retryPolicy?: TransportIdentity["retryPolicy"];}): TransportIdentity {const limits = { ...DEFAULT_TRANSPORT_LIMITS, ...options.limits };if (options.publicHeadersSha256 && !/^[a-f0-9]{64}$/.test(options.publicHeadersSha256)) {@@ -115,7 +119,7 @@ export function transportIdentity(options: {maxOutputTokens: null,reasoning: "unchanged",},- retryPolicy: { sdkMaxRetries: "default", repetitionRetries: 2 },+ retryPolicy: options.retryPolicy ?? { sdkMaxRetries: "default", repetitionRetries: 2 },limits,publicHeadersSha256: options.publicHeadersSha256 ?? sha256Bytes("[]"),};