Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(workbench): scan decoded Responses requests before adoption

超超超超超级喜欢你的达妮娅f7112b91c65d
diff --git a/workbench/l0/measurements.wb.ts b/workbench/l0/measurements.wb.ts
index ddbca7d55..f19cae982 100644
--- a/workbench/l0/measurements.wb.ts
+++ b/workbench/l0/measurements.wb.ts
@@ -11,8 +11,10 @@ import {
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
+import { readCassette } from "../lib/cassette";
import {
type ArtifactReference,
+ assertCassettePrivacy,
assertComparable,
boundBaselineFromMeasurement,
boundMeasurementFingerprints,
@@ -256,6 +258,40 @@ afterEach(() => {
});
describe("versioned measurement export and verification", () => {
+ it("scans decoded Responses request bodies for private material", () => {
+ const base64 = (text: string) => Buffer.from(text, "utf8").toString("base64");
+ const cassette = {
+ scenario: "privacy-probe",
+ provider: "openai-compatible",
+ model: "workbench-loopback-fixture",
+ resolvedModel: "workbench",
+ recordedAt: "2026-09-20T00:00:00.000Z",
+ rounds: [
+ {
+ round: 0,
+ requestHash: "0".repeat(64),
+ digest: { systemChars: 1, toolCount: 0, roles: [], lastUserText: "" },
+ sse: "",
+ requestBodyBase64: base64(JSON.stringify({ prompt: "look at /Users/alice/private.txt" })),
+ },
+ ],
+ attempts: [],
+ wireApi: "responses",
+ } as unknown as ReturnType<typeof readCassette>;
+ expect(() => assertCassettePrivacy(cassette, [], "probe.json")).toThrow(
+ expect.objectContaining({ code: "PRIVACY_PRIVATE_PATH" }),
+ );
+ const clean = {
+ ...cassette,
+ rounds: [
+ {
+ ...cassette.rounds[0],
+ requestBodyBase64: base64(JSON.stringify({ prompt: "clean request" })),
+ },
+ ],
+ } as unknown as ReturnType<typeof readCassette>;
+ expect(() => assertCassettePrivacy(clean, [], "probe.json")).not.toThrow();
+ });
it("keeps fractional weighted scores exact through the comparison trials", () => {
const checks = (heavyPasses: boolean) => [
{ id: "beh.light", axis: "behaviour" as const, weight: 1, ok: true, indeterminate: false },
diff --git a/workbench/lib/measurement.ts b/workbench/lib/measurement.ts
index 867d2ca92..14d483047 100644
--- a/workbench/lib/measurement.ts
+++ b/workbench/lib/measurement.ts
@@ -1074,7 +1074,55 @@ function assertCountsAgainstReport(manifest: MeasurementManifest, report: Workbe
}
}
-function assertCassettes(manifest: MeasurementManifest, root: string): void {
+/** The file-level privacy scan sees base64 request bodies as opaque text. A
+ * Responses cassette stores the full request that way — the user prompt, tool
+ * inputs, file paths — so decode every round's request and apply the complete
+ * existing rule set (caller secrets, credentials, raw config, private paths,
+ * transcripts) to what the request actually contains. Keys and string values are
+ * inspected unescaped: the serialized form cannot be scanned as text, because
+ * its own escape sequences read as drive paths to the private-path rule. */
+export function assertCassettePrivacy(
+ cassette: ReturnType<typeof readCassette>,
+ knownSecrets: string[],
+ label: string,
+): void {
+ for (const round of cassette.rounds) {
+ if (!round.requestBodyBase64) continue;
+ const decoded = Buffer.from(round.requestBodyBase64, "base64").toString("utf8");
+ let scannable: string;
+ try {
+ const fragments: string[] = [];
+ const collect = (node: unknown): void => {
+ if (typeof node === "string") {
+ fragments.push(node);
+ return;
+ }
+ if (Array.isArray(node)) {
+ for (const item of node) collect(item);
+ return;
+ }
+ if (node && typeof node === "object") {
+ for (const [key, value] of Object.entries(node)) {
+ fragments.push(`"${key}":`);
+ collect(value);
+ }
+ }
+ };
+ collect(JSON.parse(decoded));
+ scannable = fragments.join("\n");
+ } catch {
+ scannable = decoded;
+ }
+ const issue = privacyFailure(scannable, knownSecrets);
+ if (issue) throw new MeasurementError(issue.code, `${label}: ${issue.message}`);
+ }
+}
+
+function assertCassettes(
+ manifest: MeasurementManifest,
+ root: string,
+ knownSecrets: string[] = [],
+): void {
const assertOne = (ref: ArtifactReference, judged: boolean): void => {
let cassette: ReturnType<typeof readCassette>;
try {
@@ -1131,6 +1179,7 @@ function assertCassettes(manifest: MeasurementManifest, root: string): void {
`${ref.path} contains an unmatched rejected or failed Responses attempt`,
);
}
+ assertCassettePrivacy(cassette, knownSecrets, ref.path);
};
for (const ref of roleRefs(manifest, "main-cassette")) assertOne(ref, false);
for (const ref of roleRefs(manifest, "judge-cassette")) assertOne(ref, true);
@@ -1248,7 +1297,7 @@ export function verifyMeasurementDirectory(
assertPublicArtifact(file, options.knownSecrets);
}
assertFixture(manifest, root);
- assertCassettes(manifest, root);
+ assertCassettes(manifest, root, options.knownSecrets ?? []);
assertTransportFingerprint(manifest, root);
assertObservedUsage(manifest, root);
if (roleRefs(manifest, "main-cassette").length !== manifest.results.repetitions) {