Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

19.3 KB
Raw Blame
import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { readCassette, startRecorder } from "../lib/cassette";
import { offlineStore } from "../lib/harness";
import { askJudge } from "../lib/judge";
import {
	assertComparable,
	assertPublicArtifact,
	boundBaselineFromMeasurement,
	exportMeasurement,
	finalizeJudgedMeasurementCandidate,
	type MeasurementManifest,
	prepareMeasurementCandidate,
	replayMeasurement,
	verifyBoundBaseline,
	verifyMeasurementDirectory,
} from "../lib/measurement";
import { startScriptedModel } from "../lib/model-server";
import {
	captureSourceIdentity,
	type SourceIdentity,
	sha256Bytes,
	sourceIdentityFromManifests,
} from "../lib/provenance";
import { buildReport, fingerprintOf, summarizeScenario } from "../lib/report";
import { runRepetition } from "../lib/runner";
import { scoreRun } from "../lib/score";
import { publicHeaderProfile, transportIdentity } from "../lib/transport";
import { getScenario } from "../scenarios/registry";

const DIRECTORY = mkdtempSync(join(tmpdir(), "wb-measurement-l1-"));

function alternateProvenance(source: SourceIdentity): SourceIdentity {
	return sourceIdentityFromManifests({
		commit: source.commit === "b".repeat(40) ? "c".repeat(40) : "b".repeat(40),
		dirtyManifest: [source.effectiveManifest[0]],
		effectiveManifest: source.effectiveManifest,
	});
}

afterAll(() => rmSync(DIRECTORY, { recursive: true, force: true }));

describe("recorded measurement lifecycle", () => {
	it("keeps success-then-rejection usage incomplete and refuses adoption", async () => {
		const scenario = getScenario("target-right-clip");
		const currentSource = captureSourceIdentity();
		const source = alternateProvenance(currentSource);
		const upstream = await startScriptedModel(scenario.demoScript ?? [], {
			wireApi: "responses",
			model: "gpt-5-incomplete",
		});
		const runDir = join(DIRECTORY, "responses-incomplete-run");
		const cassetteFile = join(runDir, "main-cassette-rep-0.json");
		const headers = publicHeaderProfile({});
		const transport = transportIdentity({
			wireApi: "responses",
			maxOutputTokens: 2048,
			publicHeadersSha256: headers.sha256,
			limits: { maxRequests: 1 },
		});
		const recorder = await startRecorder({
			upstream: upstream.url,
			file: cassetteFile,
			scenario: scenario.id,
			provider: "loopback",
			model: "gpt-5-incomplete",
			wireApi: "responses",
			transport,
			publicHeaders: headers,
		});
		let result: Awaited<ReturnType<typeof runRepetition>>;
		try {
			result = await runRepetition({
				scenario,
				rep: 0,
				endpoint: recorder,
				store: offlineStore({
					baseUrl: recorder.url,
					allowAgentEdits: true,
					model: "gpt-5-incomplete",
				}),
				maxRetries: 0,
			});
		} finally {
			recorder.close();
			upstream.close();
		}
		expect(result.run.ok).toBe(false);
		const cassette = readCassette(cassetteFile);
		expect(cassette.rounds).toHaveLength(1);
		expect(cassette.attempts).toHaveLength(2);

		const summary = summarizeScenario({
			scenarioId: scenario.id,
			title: scenario.title,
			tags: scenario.tags,
			gate: scenario.gate,
			results: [result],
		});
		const report = buildReport({
			label: "responses-incomplete-test-only",
			fingerprint: fingerprintOf({
				wire: result.run.wire,
				model: "gpt-5-incomplete",
				reps: 1,
				effectiveSourceSha256: source.effectiveSha256,
			}),
			scenarios: [summary],
			notices: [],
		});
		const candidate = prepareMeasurementCandidate({
			runDir,
			scenario,
			results: [result],
			cassetteFiles: [cassetteFile],
			report,
			requestedModel: "gpt-5-incomplete",
			observedModel: cassette.resolvedModel,
			provider: "loopback",
			endpoint: upstream.url,
			source,
		});
		expect(candidate.manifest.usage?.agent).toMatchObject({
			status: "incomplete",
			attempts: 2,
			rounds: 1,
			withUsage: 1,
		});

		const id = "loopback-responses-incomplete";
		const review = join(DIRECTORY, "responses-incomplete-review.json");
		writeFileSync(
			review,
			`${JSON.stringify({
				schema: 1,
				measurementId: id,
				reviewer: { kind: "independent", label: "test fixture reviewer" },
				disposition: "approved",
				reviewedAt: "2026-09-12T00:35:00.000Z",
				summary: "Test-only incomplete Responses receipt.",
			})}\n`,
			"utf8",
		);
		expect(() =>
			exportMeasurement({
				runDir,
				id,
				reviewFile: review,
				measurementsDir: join(DIRECTORY, "incomplete-measurements"),
			}),
		).toThrow(expect.objectContaining({ code: "CANDIDATE_INCOMPLETE" }));
	});

	it.each([
		"clean",
		"transcript",
	])("adopts clean Responses evidence, rejects private text: %s", async (name) => {
		const text = name === "transcript" ? 'Done. {"transcript": "synthetic private note"}' : null;
		const scenario = getScenario("target-right-clip");
		const currentSource = captureSourceIdentity();
		const source = alternateProvenance(currentSource);
		const script = (scenario.demoScript ?? []).map((turn, index) =>
			turn.kind === "tools" && index === 0
				? { ...turn, opaqueReasoning: "synthetic_measurement_opaque" }
				: turn.kind === "text" && text !== null
					? { ...turn, text }
					: turn,
		);
		const upstream = await startScriptedModel(script, {
			wireApi: "responses",
			model: "gpt-5-measurement",
		});
		const runDir = join(DIRECTORY, `responses-${name}-run`);
		const cassetteFile = join(runDir, "main-cassette-rep-0.json");
		const headers = publicHeaderProfile({});
		const transport = transportIdentity({
			wireApi: "responses",
			maxOutputTokens: 2048,
			publicHeadersSha256: headers.sha256,
		});
		const recorder = await startRecorder({
			upstream: upstream.url,
			file: cassetteFile,
			scenario: scenario.id,
			provider: "loopback",
			model: "gpt-5-measurement",
			wireApi: "responses",
			transport,
			publicHeaders: headers,
		});
		let result: Awaited<ReturnType<typeof runRepetition>>;
		try {
			result = await runRepetition({
				scenario,
				rep: 0,
				endpoint: recorder,
				store: offlineStore({
					baseUrl: recorder.url,
					allowAgentEdits: true,
					model: "gpt-5-measurement",
				}),
				maxRetries: 0,
			});
		} finally {
			recorder.close();
			upstream.close();
		}
		expect(result.run.ok, result.run.error).toBe(true);
		const summary = summarizeScenario({
			scenarioId: scenario.id,
			title: scenario.title,
			tags: scenario.tags,
			gate: scenario.gate,
			results: [result],
		});
		const report = buildReport({
			label: "responses-loopback-test-only",
			fingerprint: fingerprintOf({
				wire: result.run.wire,
				model: "gpt-5-measurement",
				reps: 1,
				effectiveSourceSha256: source.effectiveSha256,
			}),
			scenarios: [summary],
			notices: [],
		});
		const candidate = prepareMeasurementCandidate({
			runDir,
			scenario,
			results: [result],
			cassetteFiles: [cassetteFile],
			report,
			requestedModel: "gpt-5-measurement",
			observedModel: "gpt-5-measurement",
			provider: "loopback",
			endpoint: upstream.url,
			source,
		});
		expect(candidate.manifest.fingerprints.transportSha256).toMatch(/^[a-f0-9]{64}$/);
		expect(candidate.manifest.usage?.agent).toMatchObject({
			status: "complete",
			rounds: 3,
			withUsage: 3,
			totals: { input_tokens: 303, output_tokens: 33, total_tokens: 336 },
		});

		const id = "loopback-responses-target-right-clip";
		const review = join(DIRECTORY, "responses-review.json");
		writeFileSync(
			review,
			`${JSON.stringify({
				schema: 1,
				measurementId: id,
				reviewer: { kind: "independent", label: "test fixture reviewer" },
				disposition: "approved",
				reviewedAt: "2026-09-12T00:30:00.000Z",
				summary: "Test-only Responses lifecycle receipt; no model-quality claim.",
			})}\n`,
			"utf8",
		);
		const measurementsDir = join(DIRECTORY, `responses-${name}-measurements`);
		if (text !== null) {
			expect(result.run.answer).toBe(text);
			const response = readCassette(cassetteFile).rounds.at(-1)?.response;
			expect(response?.bodyBase64).toBeTruthy();
			expect(Buffer.from(response!.bodyBase64, "base64").toString("utf8")).toContain(
				JSON.stringify(text).slice(1, -1),
			);
			// The real recorder's serialized cassette hides the quoted field from the file scan.
			expect(() => assertPublicArtifact(cassetteFile)).not.toThrow();
			expect(() => exportMeasurement({ runDir, id, reviewFile: review, measurementsDir })).toThrow(
				expect.objectContaining({ code: "PRIVACY_TRANSCRIPT" }),
			);
			return;
		}
		const adopted = exportMeasurement({ runDir, id, reviewFile: review, measurementsDir });
		const verified = verifyMeasurementDirectory(adopted, { requireApprovedReview: true });
		expect(await replayMeasurement(id, measurementsDir)).toEqual(candidate.manifest.results);
		const baseline = boundBaselineFromMeasurement(verified);
		expect(() => verifyBoundBaseline(baseline, verified)).not.toThrow();
	});

	it("records, adopts, verifies, replays, and binds one public synthetic measurement", async () => {
		const scenario = getScenario("target-right-clip");
		const currentSource = captureSourceIdentity();
		const source = alternateProvenance(currentSource);
		const upstream = await startScriptedModel(scenario.demoScript ?? []);
		const runDir = join(DIRECTORY, "run");
		const cassetteFile = join(runDir, "main-cassette-rep-0.json");
		const recorder = await startRecorder({
			upstream: upstream.url,
			file: cassetteFile,
			scenario: scenario.id,
			provider: "loopback",
			model: "workbench",
		});
		let result: Awaited<ReturnType<typeof runRepetition>>;
		try {
			result = await runRepetition({ scenario, rep: 0, endpoint: recorder });
		} finally {
			recorder.close();
			upstream.close();
		}
		const summary = summarizeScenario({
			scenarioId: scenario.id,
			title: scenario.title,
			tags: scenario.tags,
			gate: scenario.gate,
			results: [result],
		});
		const report = buildReport({
			label: "loopback-test-only",
			fingerprint: fingerprintOf({
				wire: result.run.wire,
				model: "workbench",
				reps: 1,
				effectiveSourceSha256: source.effectiveSha256,
			}),
			scenarios: [summary],
			notices: [],
		});
		const candidate = prepareMeasurementCandidate({
			runDir,
			scenario,
			results: [result],
			cassetteFiles: [cassetteFile],
			report,
			requestedModel: "workbench",
			observedModel: readCassette(cassetteFile).resolvedModel,
			provider: "loopback",
			endpoint: upstream.url,
			source,
		});
		expect(candidate.manifest.complete).toBe(true);
		expect(result.run.wire.systemChars).toBeGreaterThan(0);
		expect(candidate.manifest.fingerprints.systemSha256).toBe(result.run.wire.systemSha256);
		expect(candidate.manifest.source.commit).not.toBe(currentSource.commit);
		expect(candidate.manifest.source.effectiveSha256).toBe(currentSource.effectiveSha256);
		expect(candidate.manifest.source.effectiveManifest.length).toBeGreaterThanOrEqual(14);

		const id = "loopback-target-right-clip";
		const review = join(DIRECTORY, "review.json");
		writeFileSync(
			review,
			`${JSON.stringify(
				{
					schema: 1,
					measurementId: id,
					reviewer: { kind: "independent", label: "test fixture reviewer" },
					disposition: "approved",
					reviewedAt: "2026-09-12T00:10:00.000Z",
					summary: "Test-only review fixture; no product or model-quality claim.",
				},
				null,
				"\t",
			)}\n`,
			"utf8",
		);
		const measurementsDir = join(DIRECTORY, "measurements");
		const adopted = exportMeasurement({
			runDir,
			id,
			reviewFile: review,
			measurementsDir,
		});
		const verified = verifyMeasurementDirectory(adopted, { requireApprovedReview: true });
		expect(verified.manifest.results).toEqual(candidate.manifest.results);
		expect(await replayMeasurement(id, measurementsDir)).toEqual(candidate.manifest.results);
		const baseline = boundBaselineFromMeasurement(verified);
		expect(() => verifyBoundBaseline(baseline, verified)).not.toThrow();
		expect(() => assertComparable(verified.manifest, verified.manifest)).not.toThrow();
	});

	it("keeps a judged candidate incomplete until complete judge evidence is attached", async () => {
		const scenario = getScenario("no-invented-bounds");
		const judged = scenario.judged ?? [];
		expect(judged).toHaveLength(2);
		const currentSource = captureSourceIdentity();
		const source = alternateProvenance(currentSource);
		const upstream = await startScriptedModel([
			...(scenario.demoScript ?? []),
			{
				kind: "text",
				text: JSON.stringify({
					verdict: "conforme",
					raison: "Lecture déterministe pour ce test du cycle de mesure.",
				}),
			},
		]);
		const runDir = join(DIRECTORY, "judged-run");
		const cassetteFile = join(runDir, "main-cassette-rep-0.json");
		const mainRecorder = await startRecorder({
			upstream: upstream.url,
			file: cassetteFile,
			scenario: scenario.id,
			provider: "loopback",
			model: "workbench",
		});
		let result: Awaited<ReturnType<typeof runRepetition>>;
		try {
			result = await runRepetition({ scenario, rep: 0, endpoint: mainRecorder });
		} finally {
			mainRecorder.close();
		}
		const initialSummary = summarizeScenario({
			scenarioId: scenario.id,
			title: scenario.title,
			tags: scenario.tags,
			gate: scenario.gate,
			results: [result],
		});
		const initialReport = buildReport({
			label: "judged-loopback-test-only",
			fingerprint: fingerprintOf({
				wire: result.run.wire,
				model: "workbench",
				reps: 1,
				effectiveSourceSha256: source.effectiveSha256,
			}),
			scenarios: [initialSummary],
			notices: [],
		});
		const candidate = prepareMeasurementCandidate({
			runDir,
			scenario,
			results: [result],
			cassetteFiles: [cassetteFile],
			report: initialReport,
			requestedModel: "workbench",
			observedModel: readCassette(cassetteFile).resolvedModel,
			provider: "loopback",
			endpoint: upstream.url,
			source,
		});
		expect(candidate.manifest.complete).toBe(false);

		const judgeCassette = join(runDir, "judge-cassette.json");
		const judgeRecorder = await startRecorder({
			upstream: upstream.url,
			file: judgeCassette,
			scenario: `judge-${scenario.id}`,
			provider: "loopback",
			model: "workbench",
		});
		const readings = new Map();
		try {
			for (const check of judged) {
				readings.set(
					check.id,
					await askJudge({
						endpoint: { baseUrl: judgeRecorder.url, model: "workbench" },
						rubric: check.rubric,
						input: {
							prompt: scenario.prompt,
							answer: result.run.answer,
							facts: check.facts(result.context),
						},
					}),
				);
			}
		} finally {
			judgeRecorder.close();
			upstream.close();
		}
		const finalScored = scoreRun(scenario, result.context, readings);
		const finalSummary = summarizeScenario({
			scenarioId: scenario.id,
			title: scenario.title,
			tags: scenario.tags,
			gate: scenario.gate,
			results: [{ scored: finalScored }],
		});
		const finalReport = buildReport({
			label: "judged-loopback-test-only/final",
			fingerprint: fingerprintOf({
				wire: result.run.wire,
				model: "workbench",
				reps: 1,
				effectiveSourceSha256: source.effectiveSha256,
			}),
			scenarios: [finalSummary],
			notices: [],
		});
		const finalized = finalizeJudgedMeasurementCandidate({
			runDir,
			scenario,
			scored: [{ rep: 0, scored: finalScored }],
			judgeCassetteFile: judgeCassette,
			report: finalReport,
			requestedJudgeModel: "workbench",
			observedJudgeModel: readCassette(judgeCassette).resolvedModel,
			provider: "loopback",
			endpoint: upstream.url,
		});
		expect(finalized.manifest.complete).toBe(true);
		expect(finalized.manifest.artifacts.map((artifact) => artifact.role)).toContain(
			"judge-cassette",
		);

		const id = "loopback-bounds-judged";
		const review = join(DIRECTORY, "judged-review.json");
		writeFileSync(
			review,
			`${JSON.stringify({
				schema: 1,
				measurementId: id,
				reviewer: { kind: "independent", label: "test fixture reviewer" },
				disposition: "approved",
				reviewedAt: "2026-09-12T00:20:00.000Z",
				summary: "Test-only judged lifecycle receipt; no model-quality claim.",
			})}\n`,
			"utf8",
		);
		const measurementsDir = join(DIRECTORY, "judged-measurements");
		exportMeasurement({ runDir, id, reviewFile: review, measurementsDir });
		expect(await replayMeasurement(id, measurementsDir)).toEqual(finalized.manifest.results);

		const truncated = join(DIRECTORY, "truncated-judge");
		cpSync(join(measurementsDir, id), truncated, { recursive: true });
		const manifestPath = join(truncated, "measurement.json");
		const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
			results: { repetitions: number };
			artifacts: Array<{ role: string; path: string; sha256: string }>;
		};
		const ref = manifest.artifacts.find((artifact) => artifact.role === "judge-cassette");
		if (!ref) throw new Error("missing judge cassette");
		const cassettePath = join(truncated, ref.path);
		const cassette = JSON.parse(readFileSync(cassettePath, "utf8")) as {
			rounds: unknown[];
			attempts?: unknown[];
		};
		expect(cassette.rounds).toHaveLength(manifest.results.repetitions * judged.length);
		cassette.rounds.push(structuredClone(cassette.rounds[0]));
		if (cassette.attempts) cassette.attempts.push(structuredClone(cassette.attempts[0]));
		writeFileSync(cassettePath, `${JSON.stringify(cassette, null, "\t")}\n`, "utf8");
		ref.sha256 = sha256Bytes(readFileSync(cassettePath));
		writeFileSync(manifestPath, `${JSON.stringify(manifest, null, "\t")}\n`, "utf8");
		expect(() => verifyMeasurementDirectory(truncated)).toThrow(
			expect.objectContaining({
				code: "CANDIDATE_INCOMPLETE",
				message: expect.stringContaining(
					`${manifest.results.repetitions} repetitions × ${judged.length} judged checks`,
				),
			}),
		);

		const undercount = join(DIRECTORY, "undercount-judge");
		cpSync(join(measurementsDir, id), undercount, { recursive: true });
		const underManifestPath = join(undercount, "measurement.json");
		const underManifest = JSON.parse(
			readFileSync(underManifestPath, "utf8"),
		) as MeasurementManifest;
		const underRef = underManifest.artifacts.find((artifact) => artifact.role === "judge-cassette");
		if (!underRef) throw new Error("missing judge cassette");
		const underCassettePath = join(undercount, underRef.path);
		const underCassette = readCassette(underCassettePath);
		const expectedRounds = underManifest.results.repetitions * judged.length;
		expect(expectedRounds).toBeGreaterThan(1);
		expect(underCassette.rounds).toHaveLength(expectedRounds);
		underCassette.rounds.pop();
		underCassette.attempts?.pop();
		expect(underCassette.rounds).toHaveLength(expectedRounds - 1);
		expect(underCassette.rounds.length).toBeGreaterThan(0);
		// This scripted Chat fixture has no usage; retain an accurate summary after deletion.
		expect(underCassette.rounds.every((round) => !round.usage)).toBe(true);
		if (!underManifest.usage) throw new Error("missing usage summary");
		underManifest.usage.judge = {
			status: "incomplete",
			attempts: underCassette.attempts?.length ?? underCassette.rounds.length,
			rounds: underCassette.rounds.length,
			withUsage: 0,
			missingUsageRounds: underCassette.rounds.map((_, index) => index),
			totals: {},
		};
		writeFileSync(underCassettePath, `${JSON.stringify(underCassette, null, "\t")}\n`, "utf8");
		underRef.sha256 = sha256Bytes(readFileSync(underCassettePath));
		writeFileSync(underManifestPath, `${JSON.stringify(underManifest, null, "\t")}\n`, "utf8");
		expect(() => verifyMeasurementDirectory(undercount)).toThrow(
			expect.objectContaining({
				code: "CANDIDATE_INCOMPLETE",
				message: expect.stringContaining(
					`${underManifest.results.repetitions} repetitions × ${judged.length} judged checks`,
				),
			}),
		);
	});
});