Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(workbench): validate fingerprints and judge evidence

超超超超超级喜欢你的达妮娅2bbbeee60265
diff --git a/workbench/l0/measurements.wb.ts b/workbench/l0/measurements.wb.ts
index f19cae982..1b189182d 100644
--- a/workbench/l0/measurements.wb.ts
+++ b/workbench/l0/measurements.wb.ts
@@ -781,6 +781,75 @@ describe("measurement identity and bound baseline", () => {
expect(() => assertComparable(baseline.identity, verified.manifest)).not.toThrow();
});
+ it("requires every fingerprint except a historical missing transport hash", () => {
+ const adopted = exportValid(createCandidate(root()));
+ const adoptedManifest = JSON.parse(
+ readFileSync(join(adopted, "measurement.json"), "utf8"),
+ ) as MeasurementManifest;
+ expect(adoptedManifest.fingerprints.transportSha256).toBeUndefined();
+ expect(() => verifyMeasurementDirectory(adopted)).not.toThrow();
+
+ const required = [
+ "promptSha256",
+ "systemSha256",
+ "toolsSha256",
+ "wireSha256",
+ "rubricSha256",
+ ] as const;
+ for (const key of required) {
+ const base = createCandidate(root());
+ const scenario = getScenario("target-right-clip");
+ const reportFile = join(base.runDir, "measurement-report.json");
+ const report = JSON.parse(readFileSync(reportFile, "utf8")) as {
+ fingerprint: {
+ systemSha256?: string;
+ toolsSha256?: string;
+ toolNames?: string[];
+ };
+ };
+ if (key === "systemSha256" || key === "toolsSha256") {
+ delete report.fingerprint[key];
+ writeJson(reportFile, report);
+ }
+ rewriteCandidate(base.runDir, (manifest) => {
+ if (key === "systemSha256" || key === "toolsSha256") {
+ manifest.fingerprints.wireSha256 = boundMeasurementFingerprints(scenario, {
+ systemSha256: report.fingerprint.systemSha256,
+ toolsSha256: report.fingerprint.toolsSha256,
+ toolNames: report.fingerprint.toolNames ?? [],
+ }).wireSha256;
+ const reportRef = manifest.artifacts.find(
+ (entry) => entry.path === "measurement-report.json",
+ );
+ if (!reportRef) throw new Error("missing report artifact");
+ reportRef.sha256 = sha256Bytes(readFileSync(reportFile));
+ }
+ delete (manifest.fingerprints as Partial<MeasurementManifest["fingerprints"]>)[key];
+ });
+ try {
+ exportValid(base);
+ throw new Error(`expected ${key} to be required`);
+ } catch (error) {
+ expect(error).toBeInstanceOf(MeasurementError);
+ expect((error as MeasurementError).code).toBe("MANIFEST_INVALID");
+ expect((error as MeasurementError).message).toContain(`fingerprints.${key}`);
+ }
+ }
+
+ const transport = createCandidate(root());
+ rewriteCandidate(transport.runDir, (manifest) => {
+ manifest.fingerprints.transportSha256 = "not-a-sha";
+ });
+ try {
+ exportValid(transport);
+ throw new Error("expected a present transport fingerprint to be checked");
+ } catch (error) {
+ expect(error).toBeInstanceOf(MeasurementError);
+ expect((error as MeasurementError).code).toBe("MANIFEST_INVALID");
+ expect((error as MeasurementError).message).toContain("fingerprints.transportSha256");
+ }
+ });
+
it("rejects a rewritten prompt, wire, or rubric fingerprint", () => {
const fingerprints = createCandidate(root());
rewriteCandidate(fingerprints.runDir, (manifest) => {
diff --git a/workbench/l1/measurements.wb.ts b/workbench/l1/measurements.wb.ts
index 7c2dca87c..a9721b701 100644
--- a/workbench/l1/measurements.wb.ts
+++ b/workbench/l1/measurements.wb.ts
@@ -1,4 +1,4 @@
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
@@ -11,6 +11,7 @@ import {
boundBaselineFromMeasurement,
exportMeasurement,
finalizeJudgedMeasurementCandidate,
+ type MeasurementManifest,
prepareMeasurementCandidate,
replayMeasurement,
verifyBoundBaseline,
@@ -20,6 +21,7 @@ import { startScriptedModel } from "../lib/model-server";
import {
captureSourceIdentity,
type SourceIdentity,
+ sha256Bytes,
sourceIdentityFromManifests,
} from "../lib/provenance";
import { buildReport, fingerprintOf, summarizeScenario } from "../lib/report";
@@ -371,10 +373,10 @@ describe("recorded measurement lifecycle", () => {
expect(() => assertComparable(verified.manifest, verified.manifest)).not.toThrow();
});
- it("keeps a judged candidate incomplete until a recorded judge pass is attached", async () => {
- const scenario = getScenario("consent");
+ it("keeps a judged candidate incomplete until complete judge evidence is attached", async () => {
+ const scenario = getScenario("no-invented-bounds");
const judged = scenario.judged ?? [];
- expect(judged).toHaveLength(1);
+ expect(judged).toHaveLength(2);
const currentSource = captureSourceIdentity();
const source = alternateProvenance(currentSource);
const upstream = await startScriptedModel([
@@ -383,7 +385,7 @@ describe("recorded measurement lifecycle", () => {
kind: "text",
text: JSON.stringify({
verdict: "conforme",
- raison: "La réponse demande clairement l'accord avant toute modification.",
+ raison: "Lecture déterministe pour ce test du cycle de mesure.",
}),
},
]);
@@ -497,7 +499,7 @@ describe("recorded measurement lifecycle", () => {
"judge-cassette",
);
- const id = "loopback-consent-judged";
+ const id = "loopback-bounds-judged";
const review = join(DIRECTORY, "judged-review.json");
writeFileSync(
review,
@@ -514,5 +516,74 @@ describe("recorded measurement lifecycle", () => {
const measurementsDir = join(DIRECTORY, "judged-measurements");
exportMeasurement({ runDir, id, reviewFile: review, measurementsDir });
expect(await replayMeasurement(id, measurementsDir)).toEqual(finalized.manifest.results);
+
+ const truncated = join(DIRECTORY, "truncated-judge");
+ cpSync(join(measurementsDir, id), truncated, { recursive: true });
+ const manifestPath = join(truncated, "measurement.json");
+ const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
+ results: { repetitions: number };
+ artifacts: Array<{ role: string; path: string; sha256: string }>;
+ };
+ const ref = manifest.artifacts.find((artifact) => artifact.role === "judge-cassette");
+ if (!ref) throw new Error("missing judge cassette");
+ const cassettePath = join(truncated, ref.path);
+ const cassette = JSON.parse(readFileSync(cassettePath, "utf8")) as {
+ rounds: unknown[];
+ attempts?: unknown[];
+ };
+ expect(cassette.rounds).toHaveLength(manifest.results.repetitions * judged.length);
+ cassette.rounds.push(structuredClone(cassette.rounds[0]));
+ if (cassette.attempts) cassette.attempts.push(structuredClone(cassette.attempts[0]));
+ writeFileSync(cassettePath, `${JSON.stringify(cassette, null, "\t")}\n`, "utf8");
+ ref.sha256 = sha256Bytes(readFileSync(cassettePath));
+ writeFileSync(manifestPath, `${JSON.stringify(manifest, null, "\t")}\n`, "utf8");
+ expect(() => verifyMeasurementDirectory(truncated)).toThrow(
+ expect.objectContaining({
+ code: "CANDIDATE_INCOMPLETE",
+ message: expect.stringContaining(
+ `${manifest.results.repetitions} repetitions × ${judged.length} judged checks`,
+ ),
+ }),
+ );
+
+ const undercount = join(DIRECTORY, "undercount-judge");
+ cpSync(join(measurementsDir, id), undercount, { recursive: true });
+ const underManifestPath = join(undercount, "measurement.json");
+ const underManifest = JSON.parse(
+ readFileSync(underManifestPath, "utf8"),
+ ) as MeasurementManifest;
+ const underRef = underManifest.artifacts.find((artifact) => artifact.role === "judge-cassette");
+ if (!underRef) throw new Error("missing judge cassette");
+ const underCassettePath = join(undercount, underRef.path);
+ const underCassette = readCassette(underCassettePath);
+ const expectedRounds = underManifest.results.repetitions * judged.length;
+ expect(expectedRounds).toBeGreaterThan(1);
+ expect(underCassette.rounds).toHaveLength(expectedRounds);
+ underCassette.rounds.pop();
+ underCassette.attempts?.pop();
+ expect(underCassette.rounds).toHaveLength(expectedRounds - 1);
+ expect(underCassette.rounds.length).toBeGreaterThan(0);
+ // This scripted Chat fixture has no usage; retain an accurate summary after deletion.
+ expect(underCassette.rounds.every((round) => !round.usage)).toBe(true);
+ if (!underManifest.usage) throw new Error("missing usage summary");
+ underManifest.usage.judge = {
+ status: "incomplete",
+ attempts: underCassette.attempts?.length ?? underCassette.rounds.length,
+ rounds: underCassette.rounds.length,
+ withUsage: 0,
+ missingUsageRounds: underCassette.rounds.map((_, index) => index),
+ totals: {},
+ };
+ writeFileSync(underCassettePath, `${JSON.stringify(underCassette, null, "\t")}\n`, "utf8");
+ underRef.sha256 = sha256Bytes(readFileSync(underCassettePath));
+ writeFileSync(underManifestPath, `${JSON.stringify(underManifest, null, "\t")}\n`, "utf8");
+ expect(() => verifyMeasurementDirectory(undercount)).toThrow(
+ expect.objectContaining({
+ code: "CANDIDATE_INCOMPLETE",
+ message: expect.stringContaining(
+ `${underManifest.results.repetitions} repetitions × ${judged.length} judged checks`,
+ ),
+ }),
+ );
});
});
diff --git a/workbench/lib/measurement.ts b/workbench/lib/measurement.ts
index 7d9dc2194..97c21772b 100644
--- a/workbench/lib/measurement.ts
+++ b/workbench/lib/measurement.ts
@@ -970,9 +970,24 @@ function assertManifestShape(value: unknown): MeasurementManifest {
);
}
for (const [key, value] of Object.entries(manifest.input)) assertSha(value, `input.${key}`);
- for (const [key, value] of Object.entries(manifest.fingerprints)) {
+ // Historical manifests omit transportSha256. The other five are required even when
+ // a later equality check would compare two missing values as equal.
+ for (const key of [
+ "promptSha256",
+ "systemSha256",
+ "toolsSha256",
+ "wireSha256",
+ "rubricSha256",
+ ] as const) {
+ const value = manifest.fingerprints[key];
if (value !== "unknown") assertSha(value, `fingerprints.${key}`);
}
+ if (
+ manifest.fingerprints.transportSha256 !== undefined &&
+ manifest.fingerprints.transportSha256 !== "unknown"
+ ) {
+ assertSha(manifest.fingerprints.transportSha256, "fingerprints.transportSha256");
+ }
assertString(manifest.models.agent.requested, "models.agent.requested");
assertString(manifest.models.agent.observed, "models.agent.observed");
assertString(manifest.models.judge.requested, "models.judge.requested");
@@ -1185,6 +1200,16 @@ function assertCassettes(
`${ref.path} contains an unmatched rejected or failed Responses attempt`,
);
}
+ if (judged) {
+ const judgedChecks = getScenario(manifest.scenario.id).judged ?? [];
+ const expectedRounds = manifest.results.repetitions * judgedChecks.length;
+ if (cassette.rounds.length !== expectedRounds) {
+ fail(
+ "CANDIDATE_INCOMPLETE",
+ `${ref.path} must retain ${expectedRounds} judge rounds (${manifest.results.repetitions} repetitions × ${judgedChecks.length} judged checks)`,
+ );
+ }
+ }
assertCassettePrivacy(cassette, knownSecrets, ref.path);
};
for (const ref of roleRefs(manifest, "main-cassette")) assertOne(ref, false);