Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { readCassette, startRecorder } from "../lib/cassette";
import { offlineStore } from "../lib/harness";
import { askJudge } from "../lib/judge";
import {
assertComparable,
assertPublicArtifact,
boundBaselineFromMeasurement,
exportMeasurement,
finalizeJudgedMeasurementCandidate,
type MeasurementManifest,
prepareMeasurementCandidate,
replayMeasurement,
verifyBoundBaseline,
verifyMeasurementDirectory,
} from "../lib/measurement";
import { startScriptedModel } from "../lib/model-server";
import {
captureSourceIdentity,
type SourceIdentity,
sha256Bytes,
sourceIdentityFromManifests,
} from "../lib/provenance";
import { buildReport, fingerprintOf, summarizeScenario } from "../lib/report";
import { runRepetition } from "../lib/runner";
import { scoreRun } from "../lib/score";
import { publicHeaderProfile, transportIdentity } from "../lib/transport";
import { getScenario } from "../scenarios/registry";
const DIRECTORY = mkdtempSync(join(tmpdir(), "wb-measurement-l1-"));
function alternateProvenance(source: SourceIdentity): SourceIdentity {
return sourceIdentityFromManifests({
commit: source.commit === "b".repeat(40) ? "c".repeat(40) : "b".repeat(40),
dirtyManifest: [source.effectiveManifest[0]],
effectiveManifest: source.effectiveManifest,
});
}
afterAll(() => rmSync(DIRECTORY, { recursive: true, force: true }));
describe("recorded measurement lifecycle", () => {
it("keeps success-then-rejection usage incomplete and refuses adoption", async () => {
const scenario = getScenario("target-right-clip");
const currentSource = captureSourceIdentity();
const source = alternateProvenance(currentSource);
const upstream = await startScriptedModel(scenario.demoScript ?? [], {
wireApi: "responses",
model: "gpt-5-incomplete",
});
const runDir = join(DIRECTORY, "responses-incomplete-run");
const cassetteFile = join(runDir, "main-cassette-rep-0.json");
const headers = publicHeaderProfile({});
const transport = transportIdentity({
wireApi: "responses",
maxOutputTokens: 2048,
publicHeadersSha256: headers.sha256,
limits: { maxRequests: 1 },
});
const recorder = await startRecorder({
upstream: upstream.url,
file: cassetteFile,
scenario: scenario.id,
provider: "loopback",
model: "gpt-5-incomplete",
wireApi: "responses",
transport,
publicHeaders: headers,
});
let result: Awaited<ReturnType<typeof runRepetition>>;
try {
result = await runRepetition({
scenario,
rep: 0,
endpoint: recorder,
store: offlineStore({
baseUrl: recorder.url,
allowAgentEdits: true,
model: "gpt-5-incomplete",
}),
maxRetries: 0,
});
} finally {
recorder.close();
upstream.close();
}
expect(result.run.ok).toBe(false);
const cassette = readCassette(cassetteFile);
expect(cassette.rounds).toHaveLength(1);
expect(cassette.attempts).toHaveLength(2);
const summary = summarizeScenario({
scenarioId: scenario.id,
title: scenario.title,
tags: scenario.tags,
gate: scenario.gate,
results: [result],
});
const report = buildReport({
label: "responses-incomplete-test-only",
fingerprint: fingerprintOf({
wire: result.run.wire,
model: "gpt-5-incomplete",
reps: 1,
effectiveSourceSha256: source.effectiveSha256,
}),
scenarios: [summary],
notices: [],
});
const candidate = prepareMeasurementCandidate({
runDir,
scenario,
results: [result],
cassetteFiles: [cassetteFile],
report,
requestedModel: "gpt-5-incomplete",
observedModel: cassette.resolvedModel,
provider: "loopback",
endpoint: upstream.url,
source,
});
expect(candidate.manifest.usage?.agent).toMatchObject({
status: "incomplete",
attempts: 2,
rounds: 1,
withUsage: 1,
});
const id = "loopback-responses-incomplete";
const review = join(DIRECTORY, "responses-incomplete-review.json");
writeFileSync(
review,
`${JSON.stringify({
schema: 1,
measurementId: id,
reviewer: { kind: "independent", label: "test fixture reviewer" },
disposition: "approved",
reviewedAt: "2026-09-12T00:35:00.000Z",
summary: "Test-only incomplete Responses receipt.",
})}\n`,
"utf8",
);
expect(() =>
exportMeasurement({
runDir,
id,
reviewFile: review,
measurementsDir: join(DIRECTORY, "incomplete-measurements"),
}),
).toThrow(expect.objectContaining({ code: "CANDIDATE_INCOMPLETE" }));
});
it.each([
"clean",
"transcript",
])("adopts clean Responses evidence, rejects private text: %s", async (name) => {
const text = name === "transcript" ? 'Done. {"transcript": "synthetic private note"}' : null;
const scenario = getScenario("target-right-clip");
const currentSource = captureSourceIdentity();
const source = alternateProvenance(currentSource);
const script = (scenario.demoScript ?? []).map((turn, index) =>
turn.kind === "tools" && index === 0
? { ...turn, opaqueReasoning: "synthetic_measurement_opaque" }
: turn.kind === "text" && text !== null
? { ...turn, text }
: turn,
);
const upstream = await startScriptedModel(script, {
wireApi: "responses",
model: "gpt-5-measurement",
});
const runDir = join(DIRECTORY, `responses-${name}-run`);
const cassetteFile = join(runDir, "main-cassette-rep-0.json");
const headers = publicHeaderProfile({});
const transport = transportIdentity({
wireApi: "responses",
maxOutputTokens: 2048,
publicHeadersSha256: headers.sha256,
});
const recorder = await startRecorder({
upstream: upstream.url,
file: cassetteFile,
scenario: scenario.id,
provider: "loopback",
model: "gpt-5-measurement",
wireApi: "responses",
transport,
publicHeaders: headers,
});
let result: Awaited<ReturnType<typeof runRepetition>>;
try {
result = await runRepetition({
scenario,
rep: 0,
endpoint: recorder,
store: offlineStore({
baseUrl: recorder.url,
allowAgentEdits: true,
model: "gpt-5-measurement",
}),
maxRetries: 0,
});
} finally {
recorder.close();
upstream.close();
}
expect(result.run.ok, result.run.error).toBe(true);
const summary = summarizeScenario({
scenarioId: scenario.id,
title: scenario.title,
tags: scenario.tags,
gate: scenario.gate,
results: [result],
});
const report = buildReport({
label: "responses-loopback-test-only",
fingerprint: fingerprintOf({
wire: result.run.wire,
model: "gpt-5-measurement",
reps: 1,
effectiveSourceSha256: source.effectiveSha256,
}),
scenarios: [summary],
notices: [],
});
const candidate = prepareMeasurementCandidate({
runDir,
scenario,
results: [result],
cassetteFiles: [cassetteFile],
report,
requestedModel: "gpt-5-measurement",
observedModel: "gpt-5-measurement",
provider: "loopback",
endpoint: upstream.url,
source,
});
expect(candidate.manifest.fingerprints.transportSha256).toMatch(/^[a-f0-9]{64}$/);
expect(candidate.manifest.usage?.agent).toMatchObject({
status: "complete",
rounds: 3,
withUsage: 3,
totals: { input_tokens: 303, output_tokens: 33, total_tokens: 336 },
});
const id = "loopback-responses-target-right-clip";
const review = join(DIRECTORY, "responses-review.json");
writeFileSync(
review,
`${JSON.stringify({
schema: 1,
measurementId: id,
reviewer: { kind: "independent", label: "test fixture reviewer" },
disposition: "approved",
reviewedAt: "2026-09-12T00:30:00.000Z",
summary: "Test-only Responses lifecycle receipt; no model-quality claim.",
})}\n`,
"utf8",
);
const measurementsDir = join(DIRECTORY, `responses-${name}-measurements`);
if (text !== null) {
expect(result.run.answer).toBe(text);
const response = readCassette(cassetteFile).rounds.at(-1)?.response;
expect(response?.bodyBase64).toBeTruthy();
expect(Buffer.from(response!.bodyBase64, "base64").toString("utf8")).toContain(
JSON.stringify(text).slice(1, -1),
);
// The real recorder's serialized cassette hides the quoted field from the file scan.
expect(() => assertPublicArtifact(cassetteFile)).not.toThrow();
expect(() => exportMeasurement({ runDir, id, reviewFile: review, measurementsDir })).toThrow(
expect.objectContaining({ code: "PRIVACY_TRANSCRIPT" }),
);
return;
}
const adopted = exportMeasurement({ runDir, id, reviewFile: review, measurementsDir });
const verified = verifyMeasurementDirectory(adopted, { requireApprovedReview: true });
expect(await replayMeasurement(id, measurementsDir)).toEqual(candidate.manifest.results);
const baseline = boundBaselineFromMeasurement(verified);
expect(() => verifyBoundBaseline(baseline, verified)).not.toThrow();
});
it("records, adopts, verifies, replays, and binds one public synthetic measurement", async () => {
const scenario = getScenario("target-right-clip");
const currentSource = captureSourceIdentity();
const source = alternateProvenance(currentSource);
const upstream = await startScriptedModel(scenario.demoScript ?? []);
const runDir = join(DIRECTORY, "run");
const cassetteFile = join(runDir, "main-cassette-rep-0.json");
const recorder = await startRecorder({
upstream: upstream.url,
file: cassetteFile,
scenario: scenario.id,
provider: "loopback",
model: "workbench",
});
let result: Awaited<ReturnType<typeof runRepetition>>;
try {
result = await runRepetition({ scenario, rep: 0, endpoint: recorder });
} finally {
recorder.close();
upstream.close();
}
const summary = summarizeScenario({
scenarioId: scenario.id,
title: scenario.title,
tags: scenario.tags,
gate: scenario.gate,
results: [result],
});
const report = buildReport({
label: "loopback-test-only",
fingerprint: fingerprintOf({
wire: result.run.wire,
model: "workbench",
reps: 1,
effectiveSourceSha256: source.effectiveSha256,
}),
scenarios: [summary],
notices: [],
});
const candidate = prepareMeasurementCandidate({
runDir,
scenario,
results: [result],
cassetteFiles: [cassetteFile],
report,
requestedModel: "workbench",
observedModel: readCassette(cassetteFile).resolvedModel,
provider: "loopback",
endpoint: upstream.url,
source,
});
expect(candidate.manifest.complete).toBe(true);
expect(result.run.wire.systemChars).toBeGreaterThan(0);
expect(candidate.manifest.fingerprints.systemSha256).toBe(result.run.wire.systemSha256);
expect(candidate.manifest.source.commit).not.toBe(currentSource.commit);
expect(candidate.manifest.source.effectiveSha256).toBe(currentSource.effectiveSha256);
expect(candidate.manifest.source.effectiveManifest.length).toBeGreaterThanOrEqual(14);
const id = "loopback-target-right-clip";
const review = join(DIRECTORY, "review.json");
writeFileSync(
review,
`${JSON.stringify(
{
schema: 1,
measurementId: id,
reviewer: { kind: "independent", label: "test fixture reviewer" },
disposition: "approved",
reviewedAt: "2026-09-12T00:10:00.000Z",
summary: "Test-only review fixture; no product or model-quality claim.",
},
null,
"\t",
)}\n`,
"utf8",
);
const measurementsDir = join(DIRECTORY, "measurements");
const adopted = exportMeasurement({
runDir,
id,
reviewFile: review,
measurementsDir,
});
const verified = verifyMeasurementDirectory(adopted, { requireApprovedReview: true });
expect(verified.manifest.results).toEqual(candidate.manifest.results);
expect(await replayMeasurement(id, measurementsDir)).toEqual(candidate.manifest.results);
const baseline = boundBaselineFromMeasurement(verified);
expect(() => verifyBoundBaseline(baseline, verified)).not.toThrow();
expect(() => assertComparable(verified.manifest, verified.manifest)).not.toThrow();
});
it("keeps a judged candidate incomplete until complete judge evidence is attached", async () => {
const scenario = getScenario("no-invented-bounds");
const judged = scenario.judged ?? [];
expect(judged).toHaveLength(2);
const currentSource = captureSourceIdentity();
const source = alternateProvenance(currentSource);
const upstream = await startScriptedModel([
...(scenario.demoScript ?? []),
{
kind: "text",
text: JSON.stringify({
verdict: "conforme",
raison: "Lecture déterministe pour ce test du cycle de mesure.",
}),
},
]);
const runDir = join(DIRECTORY, "judged-run");
const cassetteFile = join(runDir, "main-cassette-rep-0.json");
const mainRecorder = await startRecorder({
upstream: upstream.url,
file: cassetteFile,
scenario: scenario.id,
provider: "loopback",
model: "workbench",
});
let result: Awaited<ReturnType<typeof runRepetition>>;
try {
result = await runRepetition({ scenario, rep: 0, endpoint: mainRecorder });
} finally {
mainRecorder.close();
}
const initialSummary = summarizeScenario({
scenarioId: scenario.id,
title: scenario.title,
tags: scenario.tags,
gate: scenario.gate,
results: [result],
});
const initialReport = buildReport({
label: "judged-loopback-test-only",
fingerprint: fingerprintOf({
wire: result.run.wire,
model: "workbench",
reps: 1,
effectiveSourceSha256: source.effectiveSha256,
}),
scenarios: [initialSummary],
notices: [],
});
const candidate = prepareMeasurementCandidate({
runDir,
scenario,
results: [result],
cassetteFiles: [cassetteFile],
report: initialReport,
requestedModel: "workbench",
observedModel: readCassette(cassetteFile).resolvedModel,
provider: "loopback",
endpoint: upstream.url,
source,
});
expect(candidate.manifest.complete).toBe(false);
const judgeCassette = join(runDir, "judge-cassette.json");
const judgeRecorder = await startRecorder({
upstream: upstream.url,
file: judgeCassette,
scenario: `judge-${scenario.id}`,
provider: "loopback",
model: "workbench",
});
const readings = new Map();
try {
for (const check of judged) {
readings.set(
check.id,
await askJudge({
endpoint: { baseUrl: judgeRecorder.url, model: "workbench" },
rubric: check.rubric,
input: {
prompt: scenario.prompt,
answer: result.run.answer,
facts: check.facts(result.context),
},
}),
);
}
} finally {
judgeRecorder.close();
upstream.close();
}
const finalScored = scoreRun(scenario, result.context, readings);
const finalSummary = summarizeScenario({
scenarioId: scenario.id,
title: scenario.title,
tags: scenario.tags,
gate: scenario.gate,
results: [{ scored: finalScored }],
});
const finalReport = buildReport({
label: "judged-loopback-test-only/final",
fingerprint: fingerprintOf({
wire: result.run.wire,
model: "workbench",
reps: 1,
effectiveSourceSha256: source.effectiveSha256,
}),
scenarios: [finalSummary],
notices: [],
});
const finalized = finalizeJudgedMeasurementCandidate({
runDir,
scenario,
scored: [{ rep: 0, scored: finalScored }],
judgeCassetteFile: judgeCassette,
report: finalReport,
requestedJudgeModel: "workbench",
observedJudgeModel: readCassette(judgeCassette).resolvedModel,
provider: "loopback",
endpoint: upstream.url,
});
expect(finalized.manifest.complete).toBe(true);
expect(finalized.manifest.artifacts.map((artifact) => artifact.role)).toContain(
"judge-cassette",
);
const id = "loopback-bounds-judged";
const review = join(DIRECTORY, "judged-review.json");
writeFileSync(
review,
`${JSON.stringify({
schema: 1,
measurementId: id,
reviewer: { kind: "independent", label: "test fixture reviewer" },
disposition: "approved",
reviewedAt: "2026-09-12T00:20:00.000Z",
summary: "Test-only judged lifecycle receipt; no model-quality claim.",
})}\n`,
"utf8",
);
const measurementsDir = join(DIRECTORY, "judged-measurements");
exportMeasurement({ runDir, id, reviewFile: review, measurementsDir });
expect(await replayMeasurement(id, measurementsDir)).toEqual(finalized.manifest.results);
const truncated = join(DIRECTORY, "truncated-judge");
cpSync(join(measurementsDir, id), truncated, { recursive: true });
const manifestPath = join(truncated, "measurement.json");
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
results: { repetitions: number };
artifacts: Array<{ role: string; path: string; sha256: string }>;
};
const ref = manifest.artifacts.find((artifact) => artifact.role === "judge-cassette");
if (!ref) throw new Error("missing judge cassette");
const cassettePath = join(truncated, ref.path);
const cassette = JSON.parse(readFileSync(cassettePath, "utf8")) as {
rounds: unknown[];
attempts?: unknown[];
};
expect(cassette.rounds).toHaveLength(manifest.results.repetitions * judged.length);
cassette.rounds.push(structuredClone(cassette.rounds[0]));
if (cassette.attempts) cassette.attempts.push(structuredClone(cassette.attempts[0]));
writeFileSync(cassettePath, `${JSON.stringify(cassette, null, "\t")}\n`, "utf8");
ref.sha256 = sha256Bytes(readFileSync(cassettePath));
writeFileSync(manifestPath, `${JSON.stringify(manifest, null, "\t")}\n`, "utf8");
expect(() => verifyMeasurementDirectory(truncated)).toThrow(
expect.objectContaining({
code: "CANDIDATE_INCOMPLETE",
message: expect.stringContaining(
`${manifest.results.repetitions} repetitions × ${judged.length} judged checks`,
),
}),
);
const undercount = join(DIRECTORY, "undercount-judge");
cpSync(join(measurementsDir, id), undercount, { recursive: true });
const underManifestPath = join(undercount, "measurement.json");
const underManifest = JSON.parse(
readFileSync(underManifestPath, "utf8"),
) as MeasurementManifest;
const underRef = underManifest.artifacts.find((artifact) => artifact.role === "judge-cassette");
if (!underRef) throw new Error("missing judge cassette");
const underCassettePath = join(undercount, underRef.path);
const underCassette = readCassette(underCassettePath);
const expectedRounds = underManifest.results.repetitions * judged.length;
expect(expectedRounds).toBeGreaterThan(1);
expect(underCassette.rounds).toHaveLength(expectedRounds);
underCassette.rounds.pop();
underCassette.attempts?.pop();
expect(underCassette.rounds).toHaveLength(expectedRounds - 1);
expect(underCassette.rounds.length).toBeGreaterThan(0);
// This scripted Chat fixture has no usage; retain an accurate summary after deletion.
expect(underCassette.rounds.every((round) => !round.usage)).toBe(true);
if (!underManifest.usage) throw new Error("missing usage summary");
underManifest.usage.judge = {
status: "incomplete",
attempts: underCassette.attempts?.length ?? underCassette.rounds.length,
rounds: underCassette.rounds.length,
withUsage: 0,
missingUsageRounds: underCassette.rounds.map((_, index) => index),
totals: {},
};
writeFileSync(underCassettePath, `${JSON.stringify(underCassette, null, "\t")}\n`, "utf8");
underRef.sha256 = sha256Bytes(readFileSync(underCassettePath));
writeFileSync(underManifestPath, `${JSON.stringify(underManifest, null, "\t")}\n`, "utf8");
expect(() => verifyMeasurementDirectory(undercount)).toThrow(
expect.objectContaining({
code: "CANDIDATE_INCOMPLETE",
message: expect.stringContaining(
`${underManifest.results.repetitions} repetitions × ${judged.length} judged checks`,
),
}),
);
});
});