Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(workbench): inspect response evidence before adoption

超超超超超级喜欢你的达妮娅669d4b420aa8
diff --git a/workbench/l1/measurements.wb.ts b/workbench/l1/measurements.wb.ts
index b80795f1c..7c2dca87c 100644
--- a/workbench/l1/measurements.wb.ts
+++ b/workbench/l1/measurements.wb.ts
@@ -7,6 +7,7 @@ import { offlineStore } from "../lib/harness";
import { askJudge } from "../lib/judge";
import {
assertComparable,
+ assertPublicArtifact,
boundBaselineFromMeasurement,
exportMeasurement,
finalizeJudgedMeasurementCandidate,
@@ -150,20 +151,26 @@ describe("recorded measurement lifecycle", () => {
).toThrow(expect.objectContaining({ code: "CANDIDATE_INCOMPLETE" }));
});
- it("binds, adopts, replays, and baselines a native Responses measurement", async () => {
+ it.each([
+ "clean",
+ "transcript",
+ ])("adopts clean Responses evidence, rejects private text: %s", async (name) => {
+ const text = name === "transcript" ? 'Done. {"transcript": "synthetic private note"}' : null;
const scenario = getScenario("target-right-clip");
const currentSource = captureSourceIdentity();
const source = alternateProvenance(currentSource);
const script = (scenario.demoScript ?? []).map((turn, index) =>
turn.kind === "tools" && index === 0
? { ...turn, opaqueReasoning: "synthetic_measurement_opaque" }
- : turn,
+ : turn.kind === "text" && text !== null
+ ? { ...turn, text }
+ : turn,
);
const upstream = await startScriptedModel(script, {
wireApi: "responses",
model: "gpt-5-measurement",
});
- const runDir = join(DIRECTORY, "responses-run");
+ const runDir = join(DIRECTORY, `responses-${name}-run`);
const cassetteFile = join(runDir, "main-cassette-rep-0.json");
const headers = publicHeaderProfile({});
const transport = transportIdentity({
@@ -198,6 +205,7 @@ describe("recorded measurement lifecycle", () => {
recorder.close();
upstream.close();
}
+ expect(result.run.ok, result.run.error).toBe(true);
const summary = summarizeScenario({
scenarioId: scenario.id,
title: scenario.title,
@@ -250,7 +258,21 @@ describe("recorded measurement lifecycle", () => {
})}\n`,
"utf8",
);
- const measurementsDir = join(DIRECTORY, "responses-measurements");
+ const measurementsDir = join(DIRECTORY, `responses-${name}-measurements`);
+ if (text !== null) {
+ expect(result.run.answer).toBe(text);
+ const response = readCassette(cassetteFile).rounds.at(-1)?.response;
+ expect(response?.bodyBase64).toBeTruthy();
+ expect(Buffer.from(response!.bodyBase64, "base64").toString("utf8")).toContain(
+ JSON.stringify(text).slice(1, -1),
+ );
+ // The real recorder's serialized cassette hides the quoted field from the file scan.
+ expect(() => assertPublicArtifact(cassetteFile)).not.toThrow();
+ expect(() => exportMeasurement({ runDir, id, reviewFile: review, measurementsDir })).toThrow(
+ expect.objectContaining({ code: "PRIVACY_TRANSCRIPT" }),
+ );
+ return;
+ }
const adopted = exportMeasurement({ runDir, id, reviewFile: review, measurementsDir });
const verified = verifyMeasurementDirectory(adopted, { requireApprovedReview: true });
expect(await replayMeasurement(id, measurementsDir)).toEqual(candidate.manifest.results);
diff --git a/workbench/lib/measurement.ts b/workbench/lib/measurement.ts
index 14d483047..7d9dc2194 100644
--- a/workbench/lib/measurement.ts
+++ b/workbench/lib/measurement.ts
@@ -1074,21 +1074,16 @@ function assertCountsAgainstReport(manifest: MeasurementManifest, report: Workbe
}
}
-/** The file-level privacy scan sees base64 request bodies as opaque text. A
- * Responses cassette stores the full request that way — the user prompt, tool
- * inputs, file paths — so decode every round's request and apply the complete
- * existing rule set (caller secrets, credentials, raw config, private paths,
- * transcripts) to what the request actually contains. Keys and string values are
- * inspected unescaped: the serialized form cannot be scanned as text, because
- * its own escape sequences read as drive paths to the private-path rule. */
+/** Base64 bodies are opaque to the file-level scan. Inspect decoded request
+ * documents and Responses SSE payloads with the existing privacy rules.
+ * Scan keys and string values unescaped, not JSON serialization (whose escape
+ * sequences can also look like drive paths). */
export function assertCassettePrivacy(
cassette: ReturnType<typeof readCassette>,
knownSecrets: string[],
label: string,
): void {
- for (const round of cassette.rounds) {
- if (!round.requestBodyBase64) continue;
- const decoded = Buffer.from(round.requestBodyBase64, "base64").toString("utf8");
+ const scan = (decoded: string): void => {
let scannable: string;
try {
const fragments: string[] = [];
@@ -1115,6 +1110,17 @@ export function assertCassettePrivacy(
}
const issue = privacyFailure(scannable, knownSecrets);
if (issue) throw new MeasurementError(issue.code, `${label}: ${issue.message}`);
+ };
+ for (const round of cassette.rounds) {
+ if (round.requestBodyBase64) {
+ scan(Buffer.from(round.requestBodyBase64, "base64").toString("utf8"));
+ }
+ if (cassette.wireApi === "responses" && round.response) {
+ const decoded = Buffer.from(round.response.bodyBase64, "base64").toString("utf8");
+ for (const line of decoded.split("\n")) {
+ if (line.startsWith("data:")) scan(line.slice(5).trim());
+ }
+ }
}
}