Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(workbench): inspect response evidence before adoption
超超超超超级喜欢你的达妮娅669d4b420aa8
diff --git a/workbench/l1/measurements.wb.ts b/workbench/l1/measurements.wb.tsindex b80795f1c..7c2dca87c 100644--- a/workbench/l1/measurements.wb.ts+++ b/workbench/l1/measurements.wb.ts@@ -7,6 +7,7 @@ import { offlineStore } from "../lib/harness";import { askJudge } from "../lib/judge";import {assertComparable,+ assertPublicArtifact,boundBaselineFromMeasurement,exportMeasurement,finalizeJudgedMeasurementCandidate,@@ -150,20 +151,26 @@ describe("recorded measurement lifecycle", () => {).toThrow(expect.objectContaining({ code: "CANDIDATE_INCOMPLETE" }));});- it("binds, adopts, replays, and baselines a native Responses measurement", async () => {+ it.each([+ "clean",+ "transcript",+ ])("adopts clean Responses evidence, rejects private text: %s", async (name) => {+ const text = name === "transcript" ? 'Done. {"transcript": "synthetic private note"}' : null;const scenario = getScenario("target-right-clip");const currentSource = captureSourceIdentity();const source = alternateProvenance(currentSource);const script = (scenario.demoScript ?? []).map((turn, index) =>turn.kind === "tools" && index === 0? { ...turn, opaqueReasoning: "synthetic_measurement_opaque" }- : turn,+ : turn.kind === "text" && text !== null+ ? { ...turn, text }+ : turn,);const upstream = await startScriptedModel(script, {wireApi: "responses",model: "gpt-5-measurement",});- const runDir = join(DIRECTORY, "responses-run");+ const runDir = join(DIRECTORY, `responses-${name}-run`);const cassetteFile = join(runDir, "main-cassette-rep-0.json");const headers = publicHeaderProfile({});const transport = transportIdentity({@@ -198,6 +205,7 @@ describe("recorded measurement lifecycle", () => {recorder.close();upstream.close();}+ expect(result.run.ok, result.run.error).toBe(true);const summary = summarizeScenario({scenarioId: scenario.id,title: scenario.title,@@ -250,7 +258,21 @@ describe("recorded measurement lifecycle", () => {})}\n`,"utf8",);- const measurementsDir = join(DIRECTORY, "responses-measurements");+ const measurementsDir = join(DIRECTORY, `responses-${name}-measurements`);+ if (text !== null) {+ expect(result.run.answer).toBe(text);+ const response = readCassette(cassetteFile).rounds.at(-1)?.response;+ expect(response?.bodyBase64).toBeTruthy();+ expect(Buffer.from(response!.bodyBase64, "base64").toString("utf8")).toContain(+ JSON.stringify(text).slice(1, -1),+ );+ // The real recorder's serialized cassette hides the quoted field from the file scan.+ expect(() => assertPublicArtifact(cassetteFile)).not.toThrow();+ expect(() => exportMeasurement({ runDir, id, reviewFile: review, measurementsDir })).toThrow(+ expect.objectContaining({ code: "PRIVACY_TRANSCRIPT" }),+ );+ return;+ }const adopted = exportMeasurement({ runDir, id, reviewFile: review, measurementsDir });const verified = verifyMeasurementDirectory(adopted, { requireApprovedReview: true });expect(await replayMeasurement(id, measurementsDir)).toEqual(candidate.manifest.results);diff --git a/workbench/lib/measurement.ts b/workbench/lib/measurement.tsindex 14d483047..7d9dc2194 100644--- a/workbench/lib/measurement.ts+++ b/workbench/lib/measurement.ts@@ -1074,21 +1074,16 @@ function assertCountsAgainstReport(manifest: MeasurementManifest, report: Workbe}}-/** The file-level privacy scan sees base64 request bodies as opaque text. A- * Responses cassette stores the full request that way — the user prompt, tool- * inputs, file paths — so decode every round's request and apply the complete- * existing rule set (caller secrets, credentials, raw config, private paths,- * transcripts) to what the request actually contains. Keys and string values are- * inspected unescaped: the serialized form cannot be scanned as text, because- * its own escape sequences read as drive paths to the private-path rule. */+/** Base64 bodies are opaque to the file-level scan. Inspect decoded request+ * documents and Responses SSE payloads with the existing privacy rules.+ * Scan keys and string values unescaped, not JSON serialization (whose escape+ * sequences can also look like drive paths). */export function assertCassettePrivacy(cassette: ReturnType<typeof readCassette>,knownSecrets: string[],label: string,): void {- for (const round of cassette.rounds) {- if (!round.requestBodyBase64) continue;- const decoded = Buffer.from(round.requestBodyBase64, "base64").toString("utf8");+ const scan = (decoded: string): void => {let scannable: string;try {const fragments: string[] = [];@@ -1115,6 +1110,17 @@ export function assertCassettePrivacy(}const issue = privacyFailure(scannable, knownSecrets);if (issue) throw new MeasurementError(issue.code, `${label}: ${issue.message}`);+ };+ for (const round of cassette.rounds) {+ if (round.requestBodyBase64) {+ scan(Buffer.from(round.requestBodyBase64, "base64").toString("utf8"));+ }+ if (cassette.wireApi === "responses" && round.response) {+ const decoded = Buffer.from(round.response.bodyBase64, "base64").toString("utf8");+ for (const line of decoded.split("\n")) {+ if (line.startsWith("data:")) scan(line.slice(5).trim());+ }+ }}}