Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(workbench): reject transport failures and keep compare weights honest
超超超超超级喜欢你的达妮娅7cf887787850
diff --git a/workbench/l0/measurements.wb.ts b/workbench/l0/measurements.wb.tsindex 3acd03064..788c287a2 100644--- a/workbench/l0/measurements.wb.ts+++ b/workbench/l0/measurements.wb.ts@@ -23,6 +23,7 @@ import {verifyBoundBaseline,verifyMeasurementDirectory,verifyMeasurementId,+ weightedAxisTrials,} from "../lib/measurement";import {MANDATORY_SOURCE_ANCHORS,@@ -300,6 +301,8 @@ describe("versioned measurement export and verification", () => {});expect(counts.axisScores.behaviour).toBe(0.625);expect(counts.axisScores.dsl).toBe(1);+ expect(weightedAxisTrials(counts, "behaviour")).toEqual({ k: 1, n: 2 });+ expect(weightedAxisTrials(counts, "dsl")).toEqual({ k: 2, n: 2 });});it("accepts the exact finite CLI argument shapes without reading environment configuration", async () => {@@ -420,6 +423,30 @@ describe("versioned measurement export and verification", () => {readFileSync(join(counts.measurementsDir, "valid-measurement", "measurement.json")),).toThrow();+ const foreignChecks = createCandidate(root());+ const checksFile = join(foreignChecks.runDir, "recorded-checks.json");+ const recorded = JSON.parse(readFileSync(checksFile, "utf8")) as RecordedChecks;+ recorded.scenarioId = "another-scenario";+ writeJson(checksFile, recorded);+ rewriteCandidate(foreignChecks.runDir, (manifest) => {+ const artifact = manifest.artifacts.find((entry) => entry.path === "recorded-checks.json");+ if (artifact) artifact.sha256 = sha256Bytes(readFileSync(checksFile));+ });+ expectCode(() => exportValid(foreignChecks), "CHECK_COUNT_MISMATCH");++ const transport = createCandidate(root());+ const reportFile = join(transport.runDir, "measurement-report.json");+ const report = JSON.parse(readFileSync(reportFile, "utf8")) as {+ scenarios: Array<{ failureClasses: Record<string, number> }>;+ };+ report.scenarios[0].failureClasses = { TRANSPORT: 1 };+ writeJson(reportFile, report);+ rewriteCandidate(transport.runDir, (manifest) => {+ const artifact = manifest.artifacts.find((entry) => entry.path === "measurement-report.json");+ if (artifact) artifact.sha256 = sha256Bytes(readFileSync(reportFile));+ });+ expectCode(() => exportValid(transport), "CANDIDATE_INCOMPLETE");+const source = createCandidate(root());rewriteCandidate(source.runDir, (manifest) => {manifest.source.effectiveManifest[0].sha256 = HASH("0");diff --git a/workbench/l0/responses-wire.wb.ts b/workbench/l0/responses-wire.wb.tsindex 8026f7bd9..ccb1ce5d7 100644--- a/workbench/l0/responses-wire.wb.ts+++ b/workbench/l0/responses-wire.wb.ts@@ -4,7 +4,7 @@// call_id-linked outputs. They do not exercise a live provider.import { describe, expect, it } from "vitest";-import { hashRequest, hashResponsesRequest } from "../lib/cassette";+import { hashChatRequest, hashRequest, hashResponsesRequest } from "../lib/cassette";import type { CapturedRequest } from "../lib/model-server";import { applyRequestPolicy, transportIdentity } from "../lib/transport";import { systemTextOf, wireFromRequests } from "../lib/wire";@@ -313,4 +313,25 @@ describe("Responses request fingerprints", () => {).toBe(expected);expect(hashResponsesRequest({ ...legacy, model: "gpt-5-test" })).not.toBe(expected);});++ it("hashes the full Chat body for transport-bound cassettes", () => {+ const body = {+ model: "gpt-5-test",+ temperature: 0,+ messages: [{ role: "user", content: "hello" }],+ tools: [+ { function: { name: "addZoom", description: "one", parameters: { type: "object" } } },+ ],+ };+ expect(hashChatRequest(body)).not.toBe(hashRequest(body));+ expect(hashChatRequest({ ...body, temperature: 1 })).not.toBe(hashChatRequest(body));+ expect(+ hashChatRequest({+ ...body,+ tools: [+ { function: { name: "addZoom", description: "two", parameters: { type: "object" } } },+ ],+ }),+ ).not.toBe(hashChatRequest(body));+ });});diff --git a/workbench/l0/transport-policy.wb.ts b/workbench/l0/transport-policy.wb.tsindex ff252ad42..0b05519bc 100644--- a/workbench/l0/transport-policy.wb.ts+++ b/workbench/l0/transport-policy.wb.ts@@ -333,6 +333,10 @@ describe("transport validation and invocation budget", () => {expect(() => budget.reserveRequest(5)).toThrow(/exceeds 4/);expect(budget.requestCount).toBe(1);expect(budget.reserveRequest(1)).toBe(2);+ const beforeRefresh = budget.startedAt;+ budget.refreshDeadline();+ expect(budget.startedAt).toBeGreaterThanOrEqual(beforeRefresh);+ expect(budget.requestCount).toBe(2);expect(() => budget.reserveRequest(1)).toThrow(/budget exhausted at 2/);expect(budget.requestCount).toBe(2);diff --git a/workbench/lib/cassette.ts b/workbench/lib/cassette.tsindex 417e7471c..3b574f8e2 100644--- a/workbench/lib/cassette.ts+++ b/workbench/lib/cassette.ts@@ -50,6 +50,8 @@ export interface CassetteRound {requestSha256?: string;response?: ResponseByteEvidence;terminalStatus?: "completed" | "incomplete" | "failed" | "missing";+ /** Chat Completions HTTP status. Absent on historical text-only cassettes. */+ httpStatus?: number;}export interface Cassette {@@ -184,10 +186,24 @@ export function hashRequest(body: Record<string, unknown>): string {return sha256(canonical).slice(0, 16);}+/** Transport-bound Chat hash. Includes tool schemas, temperature, and model options. */+export function hashChatRequest(body: Record<string, unknown>): string {+ return sha256(JSON.stringify(normalizeIds(body)));+}+export function hashResponsesRequest(body: Record<string, unknown>): string {return sha256(JSON.stringify(normalizeIds(body)));}+function hashRoundRequest(+ body: Record<string, unknown>,+ wireApi: WireApi,+ transportBound: boolean,+): string {+ if (wireApi === "responses") return hashResponsesRequest(body);+ return transportBound ? hashChatRequest(body) : hashRequest(body);+}+function responseInput(body: Record<string, unknown>): Array<Record<string, unknown>> {return Array.isArray(body.input) ? (body.input as Array<Record<string, unknown>>) : [];}@@ -690,8 +706,11 @@ export async function startRecorder(options: {};const recordedRound: CassetteRound = {round: myRound,- requestHash:- wireApi === "responses" ? hashResponsesRequest(effectiveBody) : hashRequest(effectiveBody),+ requestHash: hashRoundRequest(+ effectiveBody,+ wireApi,+ Boolean(options.transport || options.wireApi),+ ),digest: digestOf(effectiveBody, wireApi),sse: decoded,...(usage ? { usage } : {}),@@ -704,7 +723,7 @@ export async function startRecorder(options: {response: responseEvidence,terminalStatus: terminal?.status ?? "missing",}- : {}),+ : { httpStatus: responseEvidence.status }),};rounds.push(recordedRound);retainAttempt({ attempt, phase: "complete", status: "recorded" });@@ -806,8 +825,11 @@ export async function startReplay(options: {res.end('{"error":"workbench replay: cassette exhausted"}');return;}- const requestHash =- wireApi === "responses" ? hashResponsesRequest(effectiveBody) : hashRequest(effectiveBody);+ const requestHash = hashRoundRequest(+ effectiveBody,+ wireApi,+ Boolean(cassette.transport) && wireApi !== "responses",+ );if (requestHash !== stored.requestHash ||(stored.inboundPath && stored.inboundPath !== inboundPath)diff --git a/workbench/lib/measurement.ts b/workbench/lib/measurement.tsindex f3886707a..25271899a 100644--- a/workbench/lib/measurement.ts+++ b/workbench/lib/measurement.ts@@ -559,6 +559,15 @@ export function recomputeMeasurementCounts(recorded: RecordedChecks): Measuremen};}+/** One independent trial per repetition, weighted the same way as `axisScores`. */+export function weightedAxisTrials(+ results: Pick<MeasurementCounts, "repetitions" | "axisScores">,+ axis: "behaviour" | "dsl",+): { k: number; n: number } {+ const n = results.repetitions;+ return { k: Math.round(results.axisScores[axis] * n), n };+}+function scenarioRubricSha256(scenario: Scenario): string {return sha256Canonical({behaviour: scenario.behaviour.map((check) => ({@@ -947,6 +956,14 @@ function assertRequiredRoles(manifest: MeasurementManifest): void {function assertCountsAgainstReport(manifest: MeasurementManifest, report: WorkbenchReport): void {const scenario = report.scenarios.find((entry) => entry.scenarioId === manifest.scenario.id);if (!scenario) fail("CHECK_COUNT_MISMATCH", "report does not contain the measured scenario");+ const infrastructureFailures =+ (scenario.failureClasses.TRANSPORT ?? 0) + (scenario.failureClasses.TIMEOUT ?? 0);+ if (infrastructureFailures > 0) {+ fail(+ "CANDIDATE_INCOMPLETE",+ "measurement contains infrastructure failure classes and cannot be adopted",+ );+ }if (report.scenarios.length !== 1 ||scenario.reps !== manifest.results.repetitions ||@@ -1014,16 +1031,25 @@ function assertCassettes(manifest: MeasurementManifest, root: string): void {fail("MANIFEST_INVALID", `${ref.path} observed model is inconsistent with the manifest`);}if (- cassette.wireApi === "responses" &&- cassette.rounds.some(- (round) =>- round.terminalStatus !== "completed" ||- !round.response ||- round.response.status < 200 ||- round.response.status >= 300,- )+ cassette.rounds.some((round) => {+ if (cassette.wireApi === "responses") {+ return (+ round.terminalStatus !== "completed" ||+ !round.response ||+ round.response.status < 200 ||+ round.response.status >= 300+ );+ }+ return (+ typeof round.httpStatus === "number" &&+ (round.httpStatus < 200 || round.httpStatus >= 300)+ );+ })) {- fail("CANDIDATE_INCOMPLETE", `${ref.path} contains a failed Responses round`);+ fail(+ "CANDIDATE_INCOMPLETE",+ `${ref.path} contains a failed ${cassette.wireApi === "responses" ? "Responses" : "Chat"} round`,+ );}if (cassette.wireApi === "responses" &&@@ -1164,6 +1190,12 @@ export function verifyMeasurementDirectory(fail("CANDIDATE_INCOMPLETE", "exactly one recorded-checks artifact is required");const recorded = readJson(resolveArtifact(root, checksRefs[0].path)) as RecordedChecks;if (!recorded.complete) fail("CANDIDATE_INCOMPLETE", "recorded checks are not complete");+ if (recorded.scenarioId !== manifest.scenario.id) {+ fail(+ "CHECK_COUNT_MISMATCH",+ "recorded-checks scenario does not match the measurement scenario",+ );+ }const recomputed = recomputeMeasurementCounts(recorded);if (canonicalJson(recomputed) !== canonicalJson(manifest.results)) {fail("CHECK_COUNT_MISMATCH", "manifest k/n does not match recorded check outcomes");diff --git a/workbench/lib/runner.ts b/workbench/lib/runner.tsindex 12ac8c9b2..c2d2e963e 100644--- a/workbench/lib/runner.ts+++ b/workbench/lib/runner.ts@@ -198,6 +198,7 @@ export async function runScenarioReps(options.live?.budget ?? (transport ? createInvocationBudget(transport.limits) : undefined);for (let rep = 0; rep < options.reps; rep += 1) {+ budget?.refreshDeadline();let attempt = 0;for (;;) {const endpoint = options.livediff --git a/workbench/lib/transport.ts b/workbench/lib/transport.tsindex deaf47a4b..bb1fb51e9 100644--- a/workbench/lib/transport.ts+++ b/workbench/lib/transport.ts@@ -153,17 +153,20 @@ export interface InvocationBudget {readonly requestCount: number;reserveRequest: (bodyBytes: number) => number;remainingMs: () => number;+ refreshDeadline: () => void;track: (controller: AbortController) => () => void;abort: () => void;}export function createInvocationBudget(limits: TransportLimits): InvocationBudget {- const startedAt = Date.now();+ let startedAt = Date.now();let requestCount = 0;const controllers = new Set<AbortController>();return {limits,- startedAt,+ get startedAt() {+ return startedAt;+ },get requestCount() {return requestCount;},@@ -181,6 +184,9 @@ export function createInvocationBudget(limits: TransportLimits): InvocationBudgereturn requestCount;},remainingMs: () => Math.max(0, limits.invocationTimeoutMs - (Date.now() - startedAt)),+ refreshDeadline() {+ startedAt = Date.now();+ },track(controller) {controllers.add(controller);return () => controllers.delete(controller);diff --git a/workbench/measurement-cli.ts b/workbench/measurement-cli.tsindex 43e701384..010e3e883 100644--- a/workbench/measurement-cli.ts+++ b/workbench/measurement-cli.ts@@ -5,6 +5,7 @@ import {MeasurementError,replayMeasurement,verifyMeasurementId,+ weightedAxisTrials,writeBoundBaseline,} from "./lib/measurement";import { newcombeDelta } from "./lib/stats";@@ -146,24 +147,12 @@ export async function runMeasurementCli(argv: string[]): Promise<number> {leftScores: left.manifest.results.axisScores,rightScores: right.manifest.results.axisScores,behaviour: newcombeDelta(- {- k: left.manifest.results.axes.behaviour.passed,- n: left.manifest.results.axes.behaviour.decided,- },- {- k: right.manifest.results.axes.behaviour.passed,- n: right.manifest.results.axes.behaviour.decided,- },+ weightedAxisTrials(left.manifest.results, "behaviour"),+ weightedAxisTrials(right.manifest.results, "behaviour"),),dsl: newcombeDelta(- {- k: left.manifest.results.axes.dsl.passed,- n: left.manifest.results.axes.dsl.decided,- },- {- k: right.manifest.results.axes.dsl.passed,- n: right.manifest.results.axes.dsl.decided,- },+ weightedAxisTrials(left.manifest.results, "dsl"),+ weightedAxisTrials(right.manifest.results, "dsl"),),});return 0;