Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(workbench): reject transport failures and keep compare weights honest

超超超超超级喜欢你的达妮娅7cf887787850
diff --git a/workbench/l0/measurements.wb.ts b/workbench/l0/measurements.wb.ts
index 3acd03064..788c287a2 100644
--- a/workbench/l0/measurements.wb.ts
+++ b/workbench/l0/measurements.wb.ts
@@ -23,6 +23,7 @@ import {
verifyBoundBaseline,
verifyMeasurementDirectory,
verifyMeasurementId,
+ weightedAxisTrials,
} from "../lib/measurement";
import {
MANDATORY_SOURCE_ANCHORS,
@@ -300,6 +301,8 @@ describe("versioned measurement export and verification", () => {
});
expect(counts.axisScores.behaviour).toBe(0.625);
expect(counts.axisScores.dsl).toBe(1);
+ expect(weightedAxisTrials(counts, "behaviour")).toEqual({ k: 1, n: 2 });
+ expect(weightedAxisTrials(counts, "dsl")).toEqual({ k: 2, n: 2 });
});
it("accepts the exact finite CLI argument shapes without reading environment configuration", async () => {
@@ -420,6 +423,30 @@ describe("versioned measurement export and verification", () => {
readFileSync(join(counts.measurementsDir, "valid-measurement", "measurement.json")),
).toThrow();
+ const foreignChecks = createCandidate(root());
+ const checksFile = join(foreignChecks.runDir, "recorded-checks.json");
+ const recorded = JSON.parse(readFileSync(checksFile, "utf8")) as RecordedChecks;
+ recorded.scenarioId = "another-scenario";
+ writeJson(checksFile, recorded);
+ rewriteCandidate(foreignChecks.runDir, (manifest) => {
+ const artifact = manifest.artifacts.find((entry) => entry.path === "recorded-checks.json");
+ if (artifact) artifact.sha256 = sha256Bytes(readFileSync(checksFile));
+ });
+ expectCode(() => exportValid(foreignChecks), "CHECK_COUNT_MISMATCH");
+
+ const transport = createCandidate(root());
+ const reportFile = join(transport.runDir, "measurement-report.json");
+ const report = JSON.parse(readFileSync(reportFile, "utf8")) as {
+ scenarios: Array<{ failureClasses: Record<string, number> }>;
+ };
+ report.scenarios[0].failureClasses = { TRANSPORT: 1 };
+ writeJson(reportFile, report);
+ rewriteCandidate(transport.runDir, (manifest) => {
+ const artifact = manifest.artifacts.find((entry) => entry.path === "measurement-report.json");
+ if (artifact) artifact.sha256 = sha256Bytes(readFileSync(reportFile));
+ });
+ expectCode(() => exportValid(transport), "CANDIDATE_INCOMPLETE");
+
const source = createCandidate(root());
rewriteCandidate(source.runDir, (manifest) => {
manifest.source.effectiveManifest[0].sha256 = HASH("0");
diff --git a/workbench/l0/responses-wire.wb.ts b/workbench/l0/responses-wire.wb.ts
index 8026f7bd9..ccb1ce5d7 100644
--- a/workbench/l0/responses-wire.wb.ts
+++ b/workbench/l0/responses-wire.wb.ts
@@ -4,7 +4,7 @@
// call_id-linked outputs. They do not exercise a live provider.
import { describe, expect, it } from "vitest";
-import { hashRequest, hashResponsesRequest } from "../lib/cassette";
+import { hashChatRequest, hashRequest, hashResponsesRequest } from "../lib/cassette";
import type { CapturedRequest } from "../lib/model-server";
import { applyRequestPolicy, transportIdentity } from "../lib/transport";
import { systemTextOf, wireFromRequests } from "../lib/wire";
@@ -313,4 +313,25 @@ describe("Responses request fingerprints", () => {
).toBe(expected);
expect(hashResponsesRequest({ ...legacy, model: "gpt-5-test" })).not.toBe(expected);
});
+
+ it("hashes the full Chat body for transport-bound cassettes", () => {
+ const body = {
+ model: "gpt-5-test",
+ temperature: 0,
+ messages: [{ role: "user", content: "hello" }],
+ tools: [
+ { function: { name: "addZoom", description: "one", parameters: { type: "object" } } },
+ ],
+ };
+ expect(hashChatRequest(body)).not.toBe(hashRequest(body));
+ expect(hashChatRequest({ ...body, temperature: 1 })).not.toBe(hashChatRequest(body));
+ expect(
+ hashChatRequest({
+ ...body,
+ tools: [
+ { function: { name: "addZoom", description: "two", parameters: { type: "object" } } },
+ ],
+ }),
+ ).not.toBe(hashChatRequest(body));
+ });
});
diff --git a/workbench/l0/transport-policy.wb.ts b/workbench/l0/transport-policy.wb.ts
index ff252ad42..0b05519bc 100644
--- a/workbench/l0/transport-policy.wb.ts
+++ b/workbench/l0/transport-policy.wb.ts
@@ -333,6 +333,10 @@ describe("transport validation and invocation budget", () => {
expect(() => budget.reserveRequest(5)).toThrow(/exceeds 4/);
expect(budget.requestCount).toBe(1);
expect(budget.reserveRequest(1)).toBe(2);
+ const beforeRefresh = budget.startedAt;
+ budget.refreshDeadline();
+ expect(budget.startedAt).toBeGreaterThanOrEqual(beforeRefresh);
+ expect(budget.requestCount).toBe(2);
expect(() => budget.reserveRequest(1)).toThrow(/budget exhausted at 2/);
expect(budget.requestCount).toBe(2);
diff --git a/workbench/lib/cassette.ts b/workbench/lib/cassette.ts
index 417e7471c..3b574f8e2 100644
--- a/workbench/lib/cassette.ts
+++ b/workbench/lib/cassette.ts
@@ -50,6 +50,8 @@ export interface CassetteRound {
requestSha256?: string;
response?: ResponseByteEvidence;
terminalStatus?: "completed" | "incomplete" | "failed" | "missing";
+ /** Chat Completions HTTP status. Absent on historical text-only cassettes. */
+ httpStatus?: number;
}
export interface Cassette {
@@ -184,10 +186,24 @@ export function hashRequest(body: Record<string, unknown>): string {
return sha256(canonical).slice(0, 16);
}
+/** Transport-bound Chat hash. Includes tool schemas, temperature, and model options. */
+export function hashChatRequest(body: Record<string, unknown>): string {
+ return sha256(JSON.stringify(normalizeIds(body)));
+}
+
export function hashResponsesRequest(body: Record<string, unknown>): string {
return sha256(JSON.stringify(normalizeIds(body)));
}
+function hashRoundRequest(
+ body: Record<string, unknown>,
+ wireApi: WireApi,
+ transportBound: boolean,
+): string {
+ if (wireApi === "responses") return hashResponsesRequest(body);
+ return transportBound ? hashChatRequest(body) : hashRequest(body);
+}
+
function responseInput(body: Record<string, unknown>): Array<Record<string, unknown>> {
return Array.isArray(body.input) ? (body.input as Array<Record<string, unknown>>) : [];
}
@@ -690,8 +706,11 @@ export async function startRecorder(options: {
};
const recordedRound: CassetteRound = {
round: myRound,
- requestHash:
- wireApi === "responses" ? hashResponsesRequest(effectiveBody) : hashRequest(effectiveBody),
+ requestHash: hashRoundRequest(
+ effectiveBody,
+ wireApi,
+ Boolean(options.transport || options.wireApi),
+ ),
digest: digestOf(effectiveBody, wireApi),
sse: decoded,
...(usage ? { usage } : {}),
@@ -704,7 +723,7 @@ export async function startRecorder(options: {
response: responseEvidence,
terminalStatus: terminal?.status ?? "missing",
}
- : {}),
+ : { httpStatus: responseEvidence.status }),
};
rounds.push(recordedRound);
retainAttempt({ attempt, phase: "complete", status: "recorded" });
@@ -806,8 +825,11 @@ export async function startReplay(options: {
res.end('{"error":"workbench replay: cassette exhausted"}');
return;
}
- const requestHash =
- wireApi === "responses" ? hashResponsesRequest(effectiveBody) : hashRequest(effectiveBody);
+ const requestHash = hashRoundRequest(
+ effectiveBody,
+ wireApi,
+ Boolean(cassette.transport) && wireApi !== "responses",
+ );
if (
requestHash !== stored.requestHash ||
(stored.inboundPath && stored.inboundPath !== inboundPath)
diff --git a/workbench/lib/measurement.ts b/workbench/lib/measurement.ts
index f3886707a..25271899a 100644
--- a/workbench/lib/measurement.ts
+++ b/workbench/lib/measurement.ts
@@ -559,6 +559,15 @@ export function recomputeMeasurementCounts(recorded: RecordedChecks): Measuremen
};
}
+/** One independent trial per repetition, weighted the same way as `axisScores`. */
+export function weightedAxisTrials(
+ results: Pick<MeasurementCounts, "repetitions" | "axisScores">,
+ axis: "behaviour" | "dsl",
+): { k: number; n: number } {
+ const n = results.repetitions;
+ return { k: Math.round(results.axisScores[axis] * n), n };
+}
+
function scenarioRubricSha256(scenario: Scenario): string {
return sha256Canonical({
behaviour: scenario.behaviour.map((check) => ({
@@ -947,6 +956,14 @@ function assertRequiredRoles(manifest: MeasurementManifest): void {
function assertCountsAgainstReport(manifest: MeasurementManifest, report: WorkbenchReport): void {
const scenario = report.scenarios.find((entry) => entry.scenarioId === manifest.scenario.id);
if (!scenario) fail("CHECK_COUNT_MISMATCH", "report does not contain the measured scenario");
+ const infrastructureFailures =
+ (scenario.failureClasses.TRANSPORT ?? 0) + (scenario.failureClasses.TIMEOUT ?? 0);
+ if (infrastructureFailures > 0) {
+ fail(
+ "CANDIDATE_INCOMPLETE",
+ "measurement contains infrastructure failure classes and cannot be adopted",
+ );
+ }
if (
report.scenarios.length !== 1 ||
scenario.reps !== manifest.results.repetitions ||
@@ -1014,16 +1031,25 @@ function assertCassettes(manifest: MeasurementManifest, root: string): void {
fail("MANIFEST_INVALID", `${ref.path} observed model is inconsistent with the manifest`);
}
if (
- cassette.wireApi === "responses" &&
- cassette.rounds.some(
- (round) =>
- round.terminalStatus !== "completed" ||
- !round.response ||
- round.response.status < 200 ||
- round.response.status >= 300,
- )
+ cassette.rounds.some((round) => {
+ if (cassette.wireApi === "responses") {
+ return (
+ round.terminalStatus !== "completed" ||
+ !round.response ||
+ round.response.status < 200 ||
+ round.response.status >= 300
+ );
+ }
+ return (
+ typeof round.httpStatus === "number" &&
+ (round.httpStatus < 200 || round.httpStatus >= 300)
+ );
+ })
) {
- fail("CANDIDATE_INCOMPLETE", `${ref.path} contains a failed Responses round`);
+ fail(
+ "CANDIDATE_INCOMPLETE",
+ `${ref.path} contains a failed ${cassette.wireApi === "responses" ? "Responses" : "Chat"} round`,
+ );
}
if (
cassette.wireApi === "responses" &&
@@ -1164,6 +1190,12 @@ export function verifyMeasurementDirectory(
fail("CANDIDATE_INCOMPLETE", "exactly one recorded-checks artifact is required");
const recorded = readJson(resolveArtifact(root, checksRefs[0].path)) as RecordedChecks;
if (!recorded.complete) fail("CANDIDATE_INCOMPLETE", "recorded checks are not complete");
+ if (recorded.scenarioId !== manifest.scenario.id) {
+ fail(
+ "CHECK_COUNT_MISMATCH",
+ "recorded-checks scenario does not match the measurement scenario",
+ );
+ }
const recomputed = recomputeMeasurementCounts(recorded);
if (canonicalJson(recomputed) !== canonicalJson(manifest.results)) {
fail("CHECK_COUNT_MISMATCH", "manifest k/n does not match recorded check outcomes");
diff --git a/workbench/lib/runner.ts b/workbench/lib/runner.ts
index 12ac8c9b2..c2d2e963e 100644
--- a/workbench/lib/runner.ts
+++ b/workbench/lib/runner.ts
@@ -198,6 +198,7 @@ export async function runScenarioReps(
options.live?.budget ?? (transport ? createInvocationBudget(transport.limits) : undefined);
for (let rep = 0; rep < options.reps; rep += 1) {
+ budget?.refreshDeadline();
let attempt = 0;
for (;;) {
const endpoint = options.live
diff --git a/workbench/lib/transport.ts b/workbench/lib/transport.ts
index deaf47a4b..bb1fb51e9 100644
--- a/workbench/lib/transport.ts
+++ b/workbench/lib/transport.ts
@@ -153,17 +153,20 @@ export interface InvocationBudget {
readonly requestCount: number;
reserveRequest: (bodyBytes: number) => number;
remainingMs: () => number;
+ refreshDeadline: () => void;
track: (controller: AbortController) => () => void;
abort: () => void;
}
export function createInvocationBudget(limits: TransportLimits): InvocationBudget {
- const startedAt = Date.now();
+ let startedAt = Date.now();
let requestCount = 0;
const controllers = new Set<AbortController>();
return {
limits,
- startedAt,
+ get startedAt() {
+ return startedAt;
+ },
get requestCount() {
return requestCount;
},
@@ -181,6 +184,9 @@ export function createInvocationBudget(limits: TransportLimits): InvocationBudge
return requestCount;
},
remainingMs: () => Math.max(0, limits.invocationTimeoutMs - (Date.now() - startedAt)),
+ refreshDeadline() {
+ startedAt = Date.now();
+ },
track(controller) {
controllers.add(controller);
return () => controllers.delete(controller);
diff --git a/workbench/measurement-cli.ts b/workbench/measurement-cli.ts
index 43e701384..010e3e883 100644
--- a/workbench/measurement-cli.ts
+++ b/workbench/measurement-cli.ts
@@ -5,6 +5,7 @@ import {
MeasurementError,
replayMeasurement,
verifyMeasurementId,
+ weightedAxisTrials,
writeBoundBaseline,
} from "./lib/measurement";
import { newcombeDelta } from "./lib/stats";
@@ -146,24 +147,12 @@ export async function runMeasurementCli(argv: string[]): Promise<number> {
leftScores: left.manifest.results.axisScores,
rightScores: right.manifest.results.axisScores,
behaviour: newcombeDelta(
- {
- k: left.manifest.results.axes.behaviour.passed,
- n: left.manifest.results.axes.behaviour.decided,
- },
- {
- k: right.manifest.results.axes.behaviour.passed,
- n: right.manifest.results.axes.behaviour.decided,
- },
+ weightedAxisTrials(left.manifest.results, "behaviour"),
+ weightedAxisTrials(right.manifest.results, "behaviour"),
),
dsl: newcombeDelta(
- {
- k: left.manifest.results.axes.dsl.passed,
- n: left.manifest.results.axes.dsl.decided,
- },
- {
- k: right.manifest.results.axes.dsl.passed,
- n: right.manifest.results.axes.dsl.decided,
- },
+ weightedAxisTrials(left.manifest.results, "dsl"),
+ weightedAxisTrials(right.manifest.results, "dsl"),
),
});
return 0;