Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(workbench): share the judge budget and hash transport once

超超超超超级喜欢你的达妮娅7a47da9890c9
diff --git a/workbench/cli.ts b/workbench/cli.ts
index a5a9d6b8c..04fa926c9 100644
--- a/workbench/cli.ts
+++ b/workbench/cli.ts
@@ -506,6 +506,21 @@ async function commandJudge(options: Options): Promise<number> {
);
} else log("juge : replay de cassettes, aucun appel sortant");
+ const liveJudgeTransport = env
+ ? transportIdentity({
+ wireApi: env.wireApi,
+ maxOutputTokens: options.maxOutputTokens ?? 2048,
+ publicHeadersSha256: env.publicHeaders.sha256,
+ limits: {
+ maxRequests: options.maxRequests ?? 100,
+ invocationTimeoutMs: options.timeoutMs,
+ },
+ })
+ : undefined;
+ const liveJudgeBudget = liveJudgeTransport
+ ? createInvocationBudget(liveJudgeTransport.limits)
+ : undefined;
+
const summaries: ScenarioReport[] = [];
const notices: string[] = [];
let firstWire: ReturnType<typeof readPersistedTurn>["wire"] | undefined;
@@ -551,15 +566,10 @@ async function commandJudge(options: Options): Promise<number> {
endpoint = replayHandle;
} else {
judgeWireApi = env.wireApi;
- const transport = transportIdentity({
- wireApi: env.wireApi,
- maxOutputTokens: options.maxOutputTokens ?? 2048,
- publicHeadersSha256: env.publicHeaders.sha256,
- limits: {
- maxRequests: options.maxRequests ?? 100,
- invocationTimeoutMs: options.timeoutMs,
- },
- });
+ if (!liveJudgeTransport || !liveJudgeBudget) {
+ throw new Error("live judge transport was not prepared");
+ }
+ liveJudgeBudget.refreshDeadline();
endpoint = await startRecorder({
// Le proxy, pas le provider en direct — même règle que `runner.ts` :
// c'est le seul endroit d'où une cassette peut sortir, et le seul
@@ -570,9 +580,9 @@ async function commandJudge(options: Options): Promise<number> {
provider: "openai-compatible",
model: env.model,
wireApi: env.wireApi,
- transport,
+ transport: liveJudgeTransport,
publicHeaders: env.publicHeaders,
- budget: createInvocationBudget(transport.limits),
+ budget: liveJudgeBudget,
});
}
} catch (error) {
diff --git a/workbench/l0/transport-policy.wb.ts b/workbench/l0/transport-policy.wb.ts
index 0b05519bc..18c9c6949 100644
--- a/workbench/l0/transport-policy.wb.ts
+++ b/workbench/l0/transport-policy.wb.ts
@@ -277,7 +277,7 @@ describe("transport identity and request policy", () => {
expect(transportSha256(identity)).toHaveLength(64);
});
- it("binds agent order and the optional judge profile into the measurement fingerprint", () => {
+ it("binds a single agent contract and the optional judge profile into the measurement fingerprint", () => {
const headers = publicHeaderProfile({}).sha256;
const agent = responseIdentity(headers);
const secondAgent = responseIdentity(headers, { maxRequests: 7 });
@@ -285,9 +285,12 @@ describe("transport identity and request policy", () => {
const noJudge = measurementTransportSha256([agent], "no-judge");
expect(noJudge).toBe(measurementTransportSha256([cloneIdentity(agent)], "no-judge"));
+ expect(noJudge).toBe(
+ measurementTransportSha256([agent, cloneIdentity(agent), agent], "no-judge"),
+ );
expect(measurementTransportSha256([agent], judge)).not.toBe(noJudge);
- expect(measurementTransportSha256([agent, secondAgent], "no-judge")).not.toBe(
- measurementTransportSha256([secondAgent, agent], "no-judge"),
+ expect(() => measurementTransportSha256([agent, secondAgent], "no-judge")).toThrow(
+ /incompatible agent transport/,
);
});
});
diff --git a/workbench/lib/measurement.ts b/workbench/lib/measurement.ts
index 033ef70d5..6e912e4da 100644
--- a/workbench/lib/measurement.ts
+++ b/workbench/lib/measurement.ts
@@ -637,6 +637,20 @@ function wireIdentity(results: RepetitionResult[]): MeasurementIdentity["fingerp
};
}
+function transportFingerprint(
+ agent: TransportIdentity[],
+ judge: TransportIdentity | "no-judge",
+): string {
+ try {
+ return measurementTransportSha256(agent, judge);
+ } catch (error) {
+ fail(
+ "INCOMPATIBLE_IDENTITIES",
+ error instanceof Error ? error.message : "incompatible agent transport identities",
+ );
+ }
+}
+
function cassetteTransportProfiles(files: string[]): TransportIdentity[] | null {
const profiles = files.map((file) => readCassette(file).transport);
if (profiles.every((profile) => profile === undefined)) return null;
@@ -728,7 +742,7 @@ export function prepareMeasurementCandidate(options: PrepareMeasurementCandidate
fingerprints.rubricSha256 = scenarioRubricSha256(options.scenario);
const agentTransports = cassetteTransportProfiles(options.cassetteFiles);
if (agentTransports) {
- fingerprints.transportSha256 = measurementTransportSha256(agentTransports, "no-judge");
+ fingerprints.transportSha256 = transportFingerprint(agentTransports, "no-judge");
}
const manifest: MeasurementManifest = {
schema: 1,
@@ -829,7 +843,7 @@ export function finalizeJudgedMeasurementCandidate(options: {
fingerprints: {
...candidate.fingerprints,
...(agentTransports && judgeTransport
- ? { transportSha256: measurementTransportSha256(agentTransports, judgeTransport) }
+ ? { transportSha256: transportFingerprint(agentTransports, judgeTransport) }
: {}),
},
models: {
@@ -1097,7 +1111,7 @@ function assertTransportFingerprint(manifest: MeasurementManifest, root: string)
if (agent === null || (judgeRef && judge === undefined)) {
fail("INCOMPATIBLE_IDENTITIES", "measurement mixes legacy and transport-bound cassettes");
}
- const expected = measurementTransportSha256(agent, judge ?? "no-judge");
+ const expected = transportFingerprint(agent, judge ?? "no-judge");
if (manifest.fingerprints.transportSha256 !== expected) {
fail("INCOMPATIBLE_IDENTITIES", "measurement transport fingerprint does not match cassettes");
}
diff --git a/workbench/lib/transport.ts b/workbench/lib/transport.ts
index bb1fb51e9..b2aa747d1 100644
--- a/workbench/lib/transport.ts
+++ b/workbench/lib/transport.ts
@@ -129,7 +129,13 @@ export function measurementTransportSha256(
agent: TransportIdentity[],
judge: TransportIdentity | "no-judge",
): string {
- return sha256Bytes(JSON.stringify({ schema: 1, agent, judge }));
+ const contract = agent[0];
+ for (const profile of agent) {
+ if (JSON.stringify(profile) !== JSON.stringify(contract)) {
+ throw new Error("measurement mixes incompatible agent transport identities");
+ }
+ }
+ return sha256Bytes(JSON.stringify({ schema: 1, agent: contract ?? null, judge }));
}
export function applyRequestPolicy(