Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(workbench): validate fingerprints and judge evidence
超超超超超级喜欢你的达妮娅2bbbeee60265
diff --git a/workbench/l0/measurements.wb.ts b/workbench/l0/measurements.wb.tsindex f19cae982..1b189182d 100644--- a/workbench/l0/measurements.wb.ts+++ b/workbench/l0/measurements.wb.ts@@ -781,6 +781,75 @@ describe("measurement identity and bound baseline", () => {expect(() => assertComparable(baseline.identity, verified.manifest)).not.toThrow();});+ it("requires every fingerprint except a historical missing transport hash", () => {+ const adopted = exportValid(createCandidate(root()));+ const adoptedManifest = JSON.parse(+ readFileSync(join(adopted, "measurement.json"), "utf8"),+ ) as MeasurementManifest;+ expect(adoptedManifest.fingerprints.transportSha256).toBeUndefined();+ expect(() => verifyMeasurementDirectory(adopted)).not.toThrow();++ const required = [+ "promptSha256",+ "systemSha256",+ "toolsSha256",+ "wireSha256",+ "rubricSha256",+ ] as const;+ for (const key of required) {+ const base = createCandidate(root());+ const scenario = getScenario("target-right-clip");+ const reportFile = join(base.runDir, "measurement-report.json");+ const report = JSON.parse(readFileSync(reportFile, "utf8")) as {+ fingerprint: {+ systemSha256?: string;+ toolsSha256?: string;+ toolNames?: string[];+ };+ };+ if (key === "systemSha256" || key === "toolsSha256") {+ delete report.fingerprint[key];+ writeJson(reportFile, report);+ }+ rewriteCandidate(base.runDir, (manifest) => {+ if (key === "systemSha256" || key === "toolsSha256") {+ manifest.fingerprints.wireSha256 = boundMeasurementFingerprints(scenario, {+ systemSha256: report.fingerprint.systemSha256,+ toolsSha256: report.fingerprint.toolsSha256,+ toolNames: report.fingerprint.toolNames ?? [],+ }).wireSha256;+ const reportRef = manifest.artifacts.find(+ (entry) => entry.path === "measurement-report.json",+ );+ if (!reportRef) throw new Error("missing report artifact");+ reportRef.sha256 = sha256Bytes(readFileSync(reportFile));+ }+ delete (manifest.fingerprints as Partial<MeasurementManifest["fingerprints"]>)[key];+ });+ try {+ exportValid(base);+ throw new Error(`expected ${key} to be required`);+ } catch (error) {+ expect(error).toBeInstanceOf(MeasurementError);+ expect((error as MeasurementError).code).toBe("MANIFEST_INVALID");+ expect((error as MeasurementError).message).toContain(`fingerprints.${key}`);+ }+ }++ const transport = createCandidate(root());+ rewriteCandidate(transport.runDir, (manifest) => {+ manifest.fingerprints.transportSha256 = "not-a-sha";+ });+ try {+ exportValid(transport);+ throw new Error("expected a present transport fingerprint to be checked");+ } catch (error) {+ expect(error).toBeInstanceOf(MeasurementError);+ expect((error as MeasurementError).code).toBe("MANIFEST_INVALID");+ expect((error as MeasurementError).message).toContain("fingerprints.transportSha256");+ }+ });+it("rejects a rewritten prompt, wire, or rubric fingerprint", () => {const fingerprints = createCandidate(root());rewriteCandidate(fingerprints.runDir, (manifest) => {diff --git a/workbench/l1/measurements.wb.ts b/workbench/l1/measurements.wb.tsindex 7c2dca87c..a9721b701 100644--- a/workbench/l1/measurements.wb.ts+++ b/workbench/l1/measurements.wb.ts@@ -1,4 +1,4 @@-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { afterAll, describe, expect, it } from "vitest";@@ -11,6 +11,7 @@ import {boundBaselineFromMeasurement,exportMeasurement,finalizeJudgedMeasurementCandidate,+ type MeasurementManifest,prepareMeasurementCandidate,replayMeasurement,verifyBoundBaseline,@@ -20,6 +21,7 @@ import { startScriptedModel } from "../lib/model-server";import {captureSourceIdentity,type SourceIdentity,+ sha256Bytes,sourceIdentityFromManifests,} from "../lib/provenance";import { buildReport, fingerprintOf, summarizeScenario } from "../lib/report";@@ -371,10 +373,10 @@ describe("recorded measurement lifecycle", () => {expect(() => assertComparable(verified.manifest, verified.manifest)).not.toThrow();});- it("keeps a judged candidate incomplete until a recorded judge pass is attached", async () => {- const scenario = getScenario("consent");+ it("keeps a judged candidate incomplete until complete judge evidence is attached", async () => {+ const scenario = getScenario("no-invented-bounds");const judged = scenario.judged ?? [];- expect(judged).toHaveLength(1);+ expect(judged).toHaveLength(2);const currentSource = captureSourceIdentity();const source = alternateProvenance(currentSource);const upstream = await startScriptedModel([@@ -383,7 +385,7 @@ describe("recorded measurement lifecycle", () => {kind: "text",text: JSON.stringify({verdict: "conforme",- raison: "La réponse demande clairement l'accord avant toute modification.",+ raison: "Lecture déterministe pour ce test du cycle de mesure.",}),},]);@@ -497,7 +499,7 @@ describe("recorded measurement lifecycle", () => {"judge-cassette",);- const id = "loopback-consent-judged";+ const id = "loopback-bounds-judged";const review = join(DIRECTORY, "judged-review.json");writeFileSync(review,@@ -514,5 +516,74 @@ describe("recorded measurement lifecycle", () => {const measurementsDir = join(DIRECTORY, "judged-measurements");exportMeasurement({ runDir, id, reviewFile: review, measurementsDir });expect(await replayMeasurement(id, measurementsDir)).toEqual(finalized.manifest.results);++ const truncated = join(DIRECTORY, "truncated-judge");+ cpSync(join(measurementsDir, id), truncated, { recursive: true });+ const manifestPath = join(truncated, "measurement.json");+ const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {+ results: { repetitions: number };+ artifacts: Array<{ role: string; path: string; sha256: string }>;+ };+ const ref = manifest.artifacts.find((artifact) => artifact.role === "judge-cassette");+ if (!ref) throw new Error("missing judge cassette");+ const cassettePath = join(truncated, ref.path);+ const cassette = JSON.parse(readFileSync(cassettePath, "utf8")) as {+ rounds: unknown[];+ attempts?: unknown[];+ };+ expect(cassette.rounds).toHaveLength(manifest.results.repetitions * judged.length);+ cassette.rounds.push(structuredClone(cassette.rounds[0]));+ if (cassette.attempts) cassette.attempts.push(structuredClone(cassette.attempts[0]));+ writeFileSync(cassettePath, `${JSON.stringify(cassette, null, "\t")}\n`, "utf8");+ ref.sha256 = sha256Bytes(readFileSync(cassettePath));+ writeFileSync(manifestPath, `${JSON.stringify(manifest, null, "\t")}\n`, "utf8");+ expect(() => verifyMeasurementDirectory(truncated)).toThrow(+ expect.objectContaining({+ code: "CANDIDATE_INCOMPLETE",+ message: expect.stringContaining(+ `${manifest.results.repetitions} repetitions × ${judged.length} judged checks`,+ ),+ }),+ );++ const undercount = join(DIRECTORY, "undercount-judge");+ cpSync(join(measurementsDir, id), undercount, { recursive: true });+ const underManifestPath = join(undercount, "measurement.json");+ const underManifest = JSON.parse(+ readFileSync(underManifestPath, "utf8"),+ ) as MeasurementManifest;+ const underRef = underManifest.artifacts.find((artifact) => artifact.role === "judge-cassette");+ if (!underRef) throw new Error("missing judge cassette");+ const underCassettePath = join(undercount, underRef.path);+ const underCassette = readCassette(underCassettePath);+ const expectedRounds = underManifest.results.repetitions * judged.length;+ expect(expectedRounds).toBeGreaterThan(1);+ expect(underCassette.rounds).toHaveLength(expectedRounds);+ underCassette.rounds.pop();+ underCassette.attempts?.pop();+ expect(underCassette.rounds).toHaveLength(expectedRounds - 1);+ expect(underCassette.rounds.length).toBeGreaterThan(0);+ // This scripted Chat fixture has no usage; retain an accurate summary after deletion.+ expect(underCassette.rounds.every((round) => !round.usage)).toBe(true);+ if (!underManifest.usage) throw new Error("missing usage summary");+ underManifest.usage.judge = {+ status: "incomplete",+ attempts: underCassette.attempts?.length ?? underCassette.rounds.length,+ rounds: underCassette.rounds.length,+ withUsage: 0,+ missingUsageRounds: underCassette.rounds.map((_, index) => index),+ totals: {},+ };+ writeFileSync(underCassettePath, `${JSON.stringify(underCassette, null, "\t")}\n`, "utf8");+ underRef.sha256 = sha256Bytes(readFileSync(underCassettePath));+ writeFileSync(underManifestPath, `${JSON.stringify(underManifest, null, "\t")}\n`, "utf8");+ expect(() => verifyMeasurementDirectory(undercount)).toThrow(+ expect.objectContaining({+ code: "CANDIDATE_INCOMPLETE",+ message: expect.stringContaining(+ `${underManifest.results.repetitions} repetitions × ${judged.length} judged checks`,+ ),+ }),+ );});});diff --git a/workbench/lib/measurement.ts b/workbench/lib/measurement.tsindex 7d9dc2194..97c21772b 100644--- a/workbench/lib/measurement.ts+++ b/workbench/lib/measurement.ts@@ -970,9 +970,24 @@ function assertManifestShape(value: unknown): MeasurementManifest {);}for (const [key, value] of Object.entries(manifest.input)) assertSha(value, `input.${key}`);- for (const [key, value] of Object.entries(manifest.fingerprints)) {+ // Historical manifests omit transportSha256. The other five are required even when+ // a later equality check would compare two missing values as equal.+ for (const key of [+ "promptSha256",+ "systemSha256",+ "toolsSha256",+ "wireSha256",+ "rubricSha256",+ ] as const) {+ const value = manifest.fingerprints[key];if (value !== "unknown") assertSha(value, `fingerprints.${key}`);}+ if (+ manifest.fingerprints.transportSha256 !== undefined &&+ manifest.fingerprints.transportSha256 !== "unknown"+ ) {+ assertSha(manifest.fingerprints.transportSha256, "fingerprints.transportSha256");+ }assertString(manifest.models.agent.requested, "models.agent.requested");assertString(manifest.models.agent.observed, "models.agent.observed");assertString(manifest.models.judge.requested, "models.judge.requested");@@ -1185,6 +1200,16 @@ function assertCassettes(`${ref.path} contains an unmatched rejected or failed Responses attempt`,);}+ if (judged) {+ const judgedChecks = getScenario(manifest.scenario.id).judged ?? [];+ const expectedRounds = manifest.results.repetitions * judgedChecks.length;+ if (cassette.rounds.length !== expectedRounds) {+ fail(+ "CANDIDATE_INCOMPLETE",+ `${ref.path} must retain ${expectedRounds} judge rounds (${manifest.results.repetitions} repetitions × ${judgedChecks.length} judged checks)`,+ );+ }+ }assertCassettePrivacy(cassette, knownSecrets, ref.path);};for (const ref of roleRefs(manifest, "main-cassette")) assertOne(ref, false);