Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
import { randomUUID } from "node:crypto";
import {
constants,
copyFileSync,
existsSync,
lstatSync,
mkdirSync,
readFileSync,
realpathSync,
renameSync,
rmSync,
writeFileSync,
} from "node:fs";
import { dirname, isAbsolute, posix, relative, resolve, sep } from "node:path";
import { getScenario } from "../scenarios/registry";
import { readCassette, startReplay } from "./cassette";
import { containsSecret } from "./env";
import { offlineStore } from "./harness";
import { askJudge, type JudgeReading } from "./judge";
import {
canonicalJson,
captureSourceIdentity,
type EndpointIdentity,
endpointIdentity,
SHA256_PATTERN,
type SourceIdentity,
SourceIdentityError,
sha256Bytes,
sha256Canonical,
validateSourceIdentity,
} from "./provenance";
import type { WorkbenchReport } from "./report";
import { type RepetitionResult, runRepetition } from "./runner";
import type { Scenario } from "./scenario";
import { type ScoredRun, scoreRun, undecidedAwareAxisMean } from "./score";
import { measurementTransportSha256, type TransportIdentity } from "./transport";
export const MEASUREMENTS_DIR = "workbench/measurements";
export const CANDIDATE_FILE = "measurement-candidate.json";
export const MANIFEST_FILE = "measurement.json";
export type MeasurementErrorCode =
| "UNMEASURED_AXIS"
| "ARGUMENT_ERROR"
| "ARTIFACT_HASH_MISMATCH"
| "ARTIFACT_NOT_REGULAR"
| "ARTIFACT_PATH_INVALID"
| "ARTIFACT_ROLE_INVALID"
| "AXIS_SCORE_MISMATCH"
| "CANDIDATE_INCOMPLETE"
| "CHECK_COUNT_MISMATCH"
| "DUPLICATE_MEASUREMENT"
| "IDENTITY_UNKNOWN"
| "INCOMPATIBLE_IDENTITIES"
| "LEGACY_SOURCE_IDENTITY"
| "MANIFEST_INVALID"
| "MEASUREMENT_ID_INVALID"
| "MEASUREMENT_ID_MISMATCH"
| "MEASUREMENT_NOT_FOUND"
| "ORIGINAL_FIXTURE_UNAVAILABLE"
| "PRIVACY_PRIVATE_PATH"
| "PRIVACY_RAW_CONFIG"
| "PRIVACY_SECRET"
| "PRIVACY_TRANSCRIPT"
| "REPLAY_MISMATCH"
| "REVIEW_INVALID"
| "REVIEW_MISSING"
| "REVIEW_NOT_APPROVED"
| "SCENARIO_NOT_PUBLIC_SYNTHETIC"
| "SOURCE_IDENTITY_MISMATCH"
| "SYMLINK_TRAVERSAL";
export class MeasurementError extends Error {
constructor(
readonly code: MeasurementErrorCode,
message: string,
) {
super(message);
this.name = "MeasurementError";
}
}
const PUBLIC_SYNTHETIC_SOURCES: Record<string, string> = {
"camera-with-track": "workbench/scenarios/camera-track.scn.ts",
"camera-without-track": "workbench/scenarios/camera-track.scn.ts",
consent: "workbench/scenarios/consent.scn.ts",
"cursor-blind": "workbench/scenarios/cursor-question.scn.ts",
"cursor-question": "workbench/scenarios/cursor-question.scn.ts",
"cut-silences-clean": "workbench/scenarios/cut-silences-clean.scn.ts",
"describe-project": "workbench/scenarios/describe-project.scn.ts",
"describe-zooms": "workbench/scenarios/describe-zooms.scn.ts",
"describe-zooms-migrated": "workbench/scenarios/describe-zooms.scn.ts",
"no-invented-bounds": "workbench/scenarios/no-invented-bounds.scn.ts",
"out-of-scope-styling": "workbench/scenarios/out-of-scope-styling.scn.ts",
"remove-one-modifier": "workbench/scenarios/remove-one-modifier.scn.ts",
"reorder-clips": "workbench/scenarios/reorder-clips.scn.ts",
"target-right-clip": "workbench/scenarios/target-right-clip.scn.ts",
"wizard-enhance": "workbench/scenarios/wizard-enhance.scn.ts",
"wizard-enhance-bare": "workbench/scenarios/wizard-enhance-bare.scn.ts",
};
export const PUBLIC_SYNTHETIC_SCENARIOS = new Set(Object.keys(PUBLIC_SYNTHETIC_SOURCES));
export type ArtifactRole =
| "input-fixture"
| "main-cassette"
| "judge-cassette"
| "recorded-checks"
| "report"
| "review-receipt";
const ARTIFACT_ROLES = new Set<ArtifactRole>([
"input-fixture",
"main-cassette",
"judge-cassette",
"recorded-checks",
"report",
"review-receipt",
]);
export interface ArtifactReference {
role: ArtifactRole;
path: string;
sha256: string;
}
export interface RecordedCheck {
id: string;
axis: "behaviour" | "dsl";
weight: number;
ok: boolean;
indeterminate: boolean;
}
export interface RecordedChecks {
schema: 1;
scenarioId: string;
complete: boolean;
repetitions: Array<{ rep: number; checks: RecordedCheck[] }>;
}
export interface CheckCounts {
id: string;
axis: "behaviour" | "dsl";
weight: number;
passed: number;
decided: number;
indeterminate: number;
total: number;
}
export interface AxisCounts {
passed: number;
decided: number;
indeterminate: number;
total: number;
}
export interface MeasurementCounts {
repetitions: number;
axes: { behaviour: AxisCounts; dsl: AxisCounts };
/** Mean of each repetition's weighted axis score, matching report.ts. Wholly
* indeterminate repetitions are excluded from the mean. */
axisScores: { behaviour: number; dsl: number };
/** Repetitions that decided at least one weighted check on the axis. The
* comparison trials use these, never the raw repetition count: an unknown
* repetition is not an implicit success. */
measuredRepetitions: { behaviour: number; dsl: number };
checks: CheckCounts[];
}
export interface MeasurementIdentity {
scenario: { id: string; kind: "public-synthetic" | "original-real"; judged: boolean };
source: SourceIdentity;
input: {
fixtureSha256: string;
documentSha256: string;
telemetrySha256: string;
};
fingerprints: {
promptSha256: string;
systemSha256: string;
toolsSha256: string;
wireSha256: string;
rubricSha256: string;
/** Missing on historical manifests; treated as legacy/unknown. */
transportSha256?: string;
};
models: {
agent: { requested: string; observed: string };
judge: { requested: string; observed: string };
};
provider: EndpointIdentity;
}
export interface ReviewReceipt {
schema: 1;
measurementId: string;
reviewer: { kind: "human" | "independent"; label: string };
disposition: "approved" | "rejected";
reviewedAt: string;
summary: string;
}
export interface MeasurementManifest extends MeasurementIdentity {
schema: 1;
id: string;
createdAt: string;
complete: boolean;
artifacts: ArtifactReference[];
results: MeasurementCounts;
review: {
disposition: ReviewReceipt["disposition"] | "unreviewed";
reviewerKind: ReviewReceipt["reviewer"]["kind"] | "unknown";
};
usage?: {
agent: ObservedUsageSummary;
judge: ObservedUsageSummary | { status: "none" | "pending" };
};
}
export interface ObservedUsageSummary {
status: "complete" | "incomplete";
attempts: number;
rounds: number;
withUsage: number;
missingUsageRounds: number[];
totals: Record<string, number>;
}
export interface PublicSyntheticFixture {
schema: 1;
kind: "public-synthetic";
scenarioId: string;
generator: string;
documentSha256: string;
telemetrySha256: string;
}
function fail(code: MeasurementErrorCode, message: string): never {
throw new MeasurementError(code, message);
}
function requireSourceSchema(source: unknown): SourceIdentity {
try {
return validateSourceIdentity(source);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
fail(
error instanceof SourceIdentityError && message.includes("legacy dirty-only")
? "LEGACY_SOURCE_IDENTITY"
: "IDENTITY_UNKNOWN",
message,
);
}
}
/** The exact source half of the replay gate; HEAD and dirty state are provenance only. */
export function assertReplaySourceCompatible(recorded: unknown, current: unknown): void {
const recordedSource = requireSourceSchema(recorded);
const currentSource = requireSourceSchema(current);
if (recordedSource.effectiveSha256 !== currentSource.effectiveSha256) {
fail("SOURCE_IDENTITY_MISMATCH", "effective source content differs from the measurement");
}
}
function readJson(file: string): unknown {
try {
return JSON.parse(readFileSync(file, "utf8"));
} catch (error) {
fail("MANIFEST_INVALID", `${file}: ${error instanceof Error ? error.message : String(error)}`);
}
}
function writeJson(file: string, value: unknown): void {
const payload = `${JSON.stringify(value, null, "\t")}\n`;
if (containsSecret(payload)) fail("PRIVACY_SECRET", `${file}: payload contains a credential`);
mkdirSync(dirname(file), { recursive: true });
writeFileSync(file, payload, "utf8");
}
function assertString(value: unknown, label: string): asserts value is string {
if (typeof value !== "string" || value.length === 0)
fail("MANIFEST_INVALID", `${label} is required`);
}
function assertSha(value: unknown, label: string): asserts value is string {
if (typeof value !== "string" || !SHA256_PATTERN.test(value)) {
fail("MANIFEST_INVALID", `${label} must be a lowercase SHA256`);
}
}
function validateMeasurementId(
id: string,
code: "ARGUMENT_ERROR" | "MEASUREMENT_ID_INVALID" = "ARGUMENT_ERROR",
): void {
if (!/^[a-z0-9][a-z0-9._-]{0,63}$/.test(id)) {
fail(code, "measurement id must be 1-64 lowercase letters, digits, dot, dash, or underscore");
}
}
function validateRelativePath(path: unknown): asserts path is string {
if (
typeof path !== "string" ||
path.length === 0 ||
isAbsolute(path) ||
path.includes("\\") ||
path !== posix.normalize(path) ||
path.split("/").some((part) => part === "" || part === "." || part === "..")
) {
fail(
"ARTIFACT_PATH_INVALID",
`artifact path must be relative POSIX without dot segments: ${path}`,
);
}
}
function lowerPath(path: string): string {
return process.platform === "win32" ? path.toLowerCase() : path;
}
function isWithin(root: string, candidate: string): boolean {
const rootValue = lowerPath(resolve(root));
const candidateValue = lowerPath(resolve(candidate));
return candidateValue === rootValue || candidateValue.startsWith(`${rootValue}${sep}`);
}
function resolveArtifact(root: string, path: string): string {
validateRelativePath(path);
if (lstatSync(resolve(root)).isSymbolicLink()) {
fail("SYMLINK_TRAVERSAL", "measurement package root is a symlink or junction");
}
const absolute = resolve(root, ...path.split("/"));
if (!isWithin(root, absolute)) fail("ARTIFACT_PATH_INVALID", `${path} escapes its package`);
let cursor = resolve(root);
for (const segment of path.split("/")) {
cursor = resolve(cursor, segment);
let stat: ReturnType<typeof lstatSync>;
try {
stat = lstatSync(cursor);
} catch {
fail("ARTIFACT_NOT_REGULAR", `${path} does not exist`);
}
if (stat.isSymbolicLink()) fail("SYMLINK_TRAVERSAL", `${path} traverses a symlink or junction`);
}
const stat = lstatSync(absolute);
if (!stat.isFile()) fail("ARTIFACT_NOT_REGULAR", `${path} is not a regular file`);
const realRoot = realpathSync(root);
const realFile = realpathSync(absolute);
if (!isWithin(realRoot, realFile))
fail("SYMLINK_TRAVERSAL", `${path} resolves outside its package`);
return absolute;
}
function hashFile(file: string): string {
return sha256Bytes(readFileSync(file));
}
function assertArtifact(root: string, ref: ArtifactReference): string {
if (!ref || typeof ref !== "object")
fail("MANIFEST_INVALID", "artifact reference must be an object");
if (!ARTIFACT_ROLES.has(ref.role))
fail("ARTIFACT_ROLE_INVALID", `unallowlisted role: ${ref.role}`);
validateRelativePath(ref.path);
assertSha(ref.sha256, `${ref.path}.sha256`);
const file = resolveArtifact(root, ref.path);
const actual = hashFile(file);
if (actual !== ref.sha256) {
fail("ARTIFACT_HASH_MISMATCH", `${ref.path}: expected ${ref.sha256}, got ${actual}`);
}
return file;
}
function privacyFailure(payload: string, knownSecrets: string[]): MeasurementError | null {
for (const secret of knownSecrets) {
if (secret.length >= 8 && payload.includes(secret)) {
return new MeasurementError(
"PRIVACY_SECRET",
"artifact contains a caller-supplied credential",
);
}
}
if (
containsSecret(payload) ||
/\b(?:authorization\s*:\s*bearer|api[_-]?key\s*[:=]|x-api-key\s*[:=])\s*\S+/i.test(payload) ||
/"(?:apiKey|authorization)"\s*:/i.test(payload)
) {
return new MeasurementError(
"PRIVACY_SECRET",
"artifact contains a credential or authorization field",
);
}
if (/"(?:baseUrl|rawEndpoint|endpoint|configuration)"\s*:/i.test(payload)) {
return new MeasurementError(
"PRIVACY_RAW_CONFIG",
"artifact contains a raw endpoint or configuration field",
);
}
const pathPayload = payload.replace(/\\\\/g, "\\");
const privatePath = pathPayload.match(
/(?:[A-Za-z]:\\|\\\\[A-Za-z0-9._-]+\\[A-Za-z0-9$._-]+|\/(?:Users|home|tmp|opt|srv|mnt|Volumes|root|data|private\/var|var\/(?:folders|lib|tmp))\/)/i,
);
if (privatePath) {
return new MeasurementError(
"PRIVACY_PRIVATE_PATH",
`artifact contains an OS absolute private path (${privatePath[0]})`,
);
}
if (/"(?:transcripts?|documents?|rawDocument)"\s*:/i.test(payload)) {
return new MeasurementError(
"PRIVACY_TRANSCRIPT",
"artifact contains a forbidden transcript/document field",
);
}
return null;
}
export function assertPublicArtifact(file: string, knownSecrets: string[] = []): void {
const payload = readFileSync(file, "utf8");
const issue = privacyFailure(payload, knownSecrets);
if (issue) throw new MeasurementError(issue.code, `${file}: ${issue.message}`);
}
function artifactRef(root: string, file: string, role: ArtifactRole): ArtifactReference {
const path = relative(root, file).replace(/\\/g, "/");
validateRelativePath(path);
return { role, path, sha256: hashFile(file) };
}
export function recordedChecksFromScored(
scenarioId: string,
entries: Array<{ rep: number; scored: ScoredRun }>,
complete: boolean,
): RecordedChecks {
return {
schema: 1,
scenarioId,
complete,
repetitions: entries.map(({ rep, scored }) => ({
rep,
checks: [
...scored.behaviour.results.map((check) => ({
id: check.id,
axis: "behaviour" as const,
weight: check.weight,
ok: check.ok,
indeterminate: check.indeterminate,
})),
...scored.dsl.results.map((check) => ({
id: check.id,
axis: "dsl" as const,
weight: check.weight,
ok: check.ok,
indeterminate: check.indeterminate,
})),
],
})),
};
}
export function recomputeMeasurementCounts(recorded: RecordedChecks): MeasurementCounts {
if (
recorded.schema !== 1 ||
!recorded.scenarioId ||
!Array.isArray(recorded.repetitions) ||
recorded.repetitions.length === 0
) {
fail("MANIFEST_INVALID", "recorded checks are empty or malformed");
}
const reps = new Set<number>();
const byCheck = new Map<string, CheckCounts>();
let roster: string[] | null = null;
for (const repetition of recorded.repetitions) {
if (
!Number.isInteger(repetition.rep) ||
reps.has(repetition.rep) ||
!Array.isArray(repetition.checks)
) {
fail("CHECK_COUNT_MISMATCH", `duplicate or invalid repetition ${repetition.rep}`);
}
for (const check of repetition.checks) {
if (
typeof check.id !== "string" ||
check.id.length === 0 ||
(check.axis !== "behaviour" && check.axis !== "dsl") ||
typeof check.weight !== "number" ||
!Number.isFinite(check.weight) ||
check.weight <= 0 ||
typeof check.ok !== "boolean" ||
typeof check.indeterminate !== "boolean"
) {
fail("CHECK_COUNT_MISMATCH", `repetition ${repetition.rep} has a malformed check`);
}
}
reps.add(repetition.rep);
const keys = repetition.checks
.map((check) => `${check.axis}:${check.id}:${check.weight}`)
.sort();
if (new Set(keys).size !== keys.length) {
fail("CHECK_COUNT_MISMATCH", `repetition ${repetition.rep} has duplicate checks`);
}
if (roster === null) roster = keys;
else if (canonicalJson(roster) !== canonicalJson(keys)) {
fail("CHECK_COUNT_MISMATCH", `repetition ${repetition.rep} changed the check denominator`);
}
for (const check of repetition.checks) {
if (check.indeterminate && check.ok) {
fail("CHECK_COUNT_MISMATCH", `${check.id} is both passing and indeterminate`);
}
const key = `${check.axis}:${check.id}`;
const entry = byCheck.get(key) ?? {
id: check.id,
axis: check.axis,
weight: check.weight,
passed: 0,
decided: 0,
indeterminate: 0,
total: 0,
};
entry.total += 1;
if (check.indeterminate) entry.indeterminate += 1;
else {
entry.decided += 1;
if (check.ok) entry.passed += 1;
}
byCheck.set(key, entry);
}
}
const checks = [...byCheck.values()].sort((left, right) =>
`${left.axis}:${left.id}`.localeCompare(`${right.axis}:${right.id}`),
);
const axis = (name: "behaviour" | "dsl"): AxisCounts =>
checks
.filter((check) => check.axis === name)
.reduce(
(sum, check) => ({
passed: sum.passed + check.passed,
decided: sum.decided + check.decided,
indeterminate: sum.indeterminate + check.indeterminate,
total: sum.total + check.total,
}),
{ passed: 0, decided: 0, indeterminate: 0, total: 0 },
);
const axisResults = (name: "behaviour" | "dsl") =>
recorded.repetitions.map((repetition) =>
repetition.checks.filter((check) => check.axis === name),
);
// The measurement rule lives in one place (`undecidedAwareAxisMean`): a wholly
// indeterminate repetition-axis decides nothing and leaves the mean, and an
// axis with nothing decidable scores 0 — never `runChecks`'s placeholder 1.
const behaviourScore = undecidedAwareAxisMean(axisResults("behaviour"));
const dslScore = undecidedAwareAxisMean(axisResults("dsl"));
return {
repetitions: recorded.repetitions.length,
axes: { behaviour: axis("behaviour"), dsl: axis("dsl") },
axisScores: { behaviour: behaviourScore.score, dsl: dslScore.score },
measuredRepetitions: { behaviour: behaviourScore.measured, dsl: dslScore.measured },
checks,
};
}
/** One trial per measured repetition, weighted the same way as `axisScores`. The
* trial count stays FRACTIONAL on purpose: weighted checks commonly produce means
* like 0.625, and rounding k into whole successes before the Newcombe comparison
* misstates the point estimate (0.625 over two repetitions would read as 0.5).
* The Wilson arithmetic below holds for non-integer k, and only repetitions that
* decided weighted checks count as trials — an unknown repetition is not an
* implicit success. */
export function weightedAxisTrials(
results: Pick<MeasurementCounts, "repetitions" | "axisScores" | "measuredRepetitions">,
axis: "behaviour" | "dsl",
): { k: number; n: number } {
const n = results.measuredRepetitions?.[axis] ?? results.repetitions;
return { k: results.axisScores[axis] * n, n };
}
export function boundMeasurementFingerprints(
scenario: Scenario,
wire: {
systemSha256?: string | null;
toolsSha256?: string | null;
toolNames?: string[];
},
): Pick<MeasurementIdentity["fingerprints"], "promptSha256" | "rubricSha256" | "wireSha256"> {
return {
promptSha256: sha256Bytes(scenario.prompt.replace(/\r\n?/g, "\n")),
rubricSha256: scenarioRubricSha256(scenario),
wireSha256: sha256Canonical({
systemSha256: wire.systemSha256 ?? null,
toolsSha256: wire.toolsSha256 ?? null,
toolNames: wire.toolNames ?? [],
}),
};
}
function scenarioRubricSha256(scenario: Scenario): string {
return sha256Canonical({
behaviour: scenario.behaviour.map((check) => ({
id: check.id,
weight: check.weight,
})),
judged: (scenario.judged ?? []).map((check) => ({
id: check.id,
weight: check.weight,
rubric: check.rubric,
})),
dsl: scenario.dsl.map((check) => ({
id: check.id,
weight: check.weight,
})),
});
}
function publicFixtureFor(scenario: Scenario): PublicSyntheticFixture {
if (!PUBLIC_SYNTHETIC_SCENARIOS.has(scenario.id)) {
fail(
scenario.id.startsWith("real-")
? "ORIGINAL_FIXTURE_UNAVAILABLE"
: "SCENARIO_NOT_PUBLIC_SYNTHETIC",
`${scenario.id}: only registered public synthetic scenarios may be exported`,
);
}
const documentSha256 = sha256Canonical(scenario.document());
const telemetrySha256 = sha256Canonical(scenario.cursorTelemetry?.() ?? null);
return {
schema: 1,
kind: "public-synthetic",
scenarioId: scenario.id,
generator: PUBLIC_SYNTHETIC_SOURCES[scenario.id],
documentSha256,
telemetrySha256,
};
}
function unknown(value: string | null | undefined): string {
return value && value.trim().length > 0 ? value : "unknown";
}
function wireIdentity(results: RepetitionResult[]): MeasurementIdentity["fingerprints"] {
const first = results[0]?.run.wire;
const contract = {
systemSha256: first?.systemSha256 ?? null,
toolsSha256: first?.toolsSha256 ?? null,
toolNames: first?.toolNames ?? [],
};
for (const result of results) {
const next = {
systemSha256: result.run.wire.systemSha256 ?? null,
toolsSha256: result.run.wire.toolsSha256 ?? null,
toolNames: result.run.wire.toolNames ?? [],
};
if (canonicalJson(next) !== canonicalJson(contract)) {
fail("INCOMPATIBLE_IDENTITIES", "repetitions do not share a request-contract wire identity");
}
}
return {
promptSha256: "unknown",
systemSha256: unknown(first?.systemSha256),
toolsSha256: unknown(first?.toolsSha256),
wireSha256: sha256Canonical(contract),
rubricSha256: "unknown",
};
}
function transportFingerprint(
agent: TransportIdentity[],
judge: TransportIdentity | "no-judge",
): string {
try {
return measurementTransportSha256(agent, judge);
} catch (error) {
fail(
"INCOMPATIBLE_IDENTITIES",
error instanceof Error ? error.message : "incompatible agent transport identities",
);
}
}
function cassetteTransportProfiles(files: string[]): TransportIdentity[] | null {
const profiles = files.map((file) => readCassette(file).transport);
if (profiles.every((profile) => profile === undefined)) return null;
if (profiles.some((profile) => profile === undefined)) {
fail("INCOMPATIBLE_IDENTITIES", "measurement mixes legacy and transport-bound cassettes");
}
return profiles as TransportIdentity[];
}
function observedUsage(files: string[]): ObservedUsageSummary {
const totals: Record<string, number> = {};
const missingUsageRounds: number[] = [];
let attempts = 0;
let rounds = 0;
let withUsage = 0;
for (const file of files) {
const cassette = readCassette(file);
attempts += cassette.attempts?.length ?? cassette.rounds.length;
for (const round of cassette.rounds) {
const index = rounds++;
if (!round.usage) {
missingUsageRounds.push(index);
continue;
}
withUsage += 1;
for (const [name, value] of Object.entries(round.usage)) {
if (typeof value === "number" && Number.isFinite(value)) {
totals[name] = (totals[name] ?? 0) + value;
}
}
}
}
return {
status: missingUsageRounds.length === 0 && attempts === rounds ? "complete" : "incomplete",
attempts,
rounds,
withUsage,
missingUsageRounds,
totals,
};
}
export interface PrepareMeasurementCandidateOptions {
runDir: string;
scenario: Scenario;
results: RepetitionResult[];
cassetteFiles: string[];
report: WorkbenchReport;
requestedModel: string;
observedModel?: string | null;
provider: string;
endpoint: string;
source?: SourceIdentity;
now?: Date;
}
/** Called by the real recording CLI; its run directory is the export input. */
export function prepareMeasurementCandidate(options: PrepareMeasurementCandidateOptions): {
file: string;
manifest: MeasurementManifest;
} {
if (options.results.length === 0 || options.cassetteFiles.length !== options.results.length) {
fail("CANDIDATE_INCOMPLETE", "a candidate needs one recorded cassette per repetition");
}
const runDir = resolve(options.runDir);
mkdirSync(runDir, { recursive: true });
const fixture = publicFixtureFor(options.scenario);
const fixtureFile = resolve(runDir, "input-fixture.json");
const checksFile = resolve(runDir, "recorded-checks.json");
const reportFile = resolve(runDir, "measurement-report.json");
writeJson(fixtureFile, fixture);
const complete = (options.scenario.judged ?? []).length === 0;
const recorded = recordedChecksFromScored(
options.scenario.id,
options.results.map((result) => ({ rep: result.rep, scored: result.scored })),
complete,
);
writeJson(checksFile, recorded);
writeJson(reportFile, options.report);
const cassetteRefs = options.cassetteFiles.map((file) => {
const absolute = resolve(file);
if (!isWithin(runDir, absolute)) {
fail("ARTIFACT_PATH_INVALID", `cassette must be recorded inside ${runDir}: ${file}`);
}
return artifactRef(runDir, absolute, "main-cassette");
});
const fingerprints = wireIdentity(options.results);
const bound = boundMeasurementFingerprints(options.scenario, options.results[0]?.run.wire ?? {});
fingerprints.promptSha256 = bound.promptSha256;
fingerprints.wireSha256 = bound.wireSha256;
fingerprints.rubricSha256 = bound.rubricSha256;
const agentTransports = cassetteTransportProfiles(options.cassetteFiles);
if (agentTransports) {
fingerprints.transportSha256 = transportFingerprint(agentTransports, "no-judge");
}
const manifest: MeasurementManifest = {
schema: 1,
id: "candidate",
createdAt: (options.now ?? new Date()).toISOString(),
complete,
scenario: {
id: options.scenario.id,
kind: "public-synthetic",
judged: (options.scenario.judged ?? []).length > 0,
},
source: options.source ?? captureSourceIdentity(),
input: {
fixtureSha256: hashFile(fixtureFile),
documentSha256: fixture.documentSha256,
telemetrySha256: fixture.telemetrySha256,
},
fingerprints,
models: {
agent: {
requested: unknown(options.requestedModel),
observed: unknown(options.observedModel),
},
judge: complete
? { requested: "none", observed: "none" }
: { requested: "unknown", observed: "unknown" },
},
provider: endpointIdentity(options.provider, options.endpoint),
artifacts: [
artifactRef(runDir, fixtureFile, "input-fixture"),
...cassetteRefs,
artifactRef(runDir, checksFile, "recorded-checks"),
artifactRef(runDir, reportFile, "report"),
],
results: recomputeMeasurementCounts(recorded),
review: { disposition: "unreviewed", reviewerKind: "unknown" },
usage: {
agent: observedUsage(options.cassetteFiles),
judge: complete ? { status: "none" } : { status: "pending" },
},
};
const file = resolve(runDir, CANDIDATE_FILE);
writeJson(file, manifest);
return { file, manifest };
}
export function finalizeJudgedMeasurementCandidate(options: {
runDir: string;
scenario: Scenario;
scored: Array<{ rep: number; scored: ScoredRun }>;
judgeCassetteFile: string;
report: WorkbenchReport;
requestedJudgeModel: string;
observedJudgeModel?: string | null;
provider: string;
endpoint: string;
}): { file: string; manifest: MeasurementManifest } {
const runDir = resolve(options.runDir);
const file = resolve(runDir, CANDIDATE_FILE);
const candidate = assertManifestShape(readJson(file));
if (candidate.scenario.id !== options.scenario.id || !candidate.scenario.judged) {
fail("MANIFEST_INVALID", "judge result does not match a judged measurement candidate");
}
const currentSource = captureSourceIdentity();
assertReplaySourceCompatible(candidate.source, currentSource);
if (
canonicalJson(candidate.provider) !==
canonicalJson(endpointIdentity(options.provider, options.endpoint))
) {
fail("INCOMPATIBLE_IDENTITIES", "judge provider/endpoint differs from the agent measurement");
}
if (!isWithin(runDir, resolve(options.judgeCassetteFile))) {
fail(
"ARTIFACT_PATH_INVALID",
"judge cassette must be recorded inside the candidate run directory",
);
}
const recorded = recordedChecksFromScored(options.scenario.id, options.scored, true);
const checksFile = resolve(runDir, "recorded-checks.json");
const reportFile = resolve(runDir, "measurement-report.json");
writeJson(checksFile, recorded);
writeJson(reportFile, options.report);
const preserved = candidate.artifacts.filter(
(ref) =>
ref.role !== "recorded-checks" && ref.role !== "report" && ref.role !== "judge-cassette",
);
const agentCassetteFiles = candidate.artifacts
.filter((ref) => ref.role === "main-cassette")
.map((ref) => resolveArtifact(runDir, ref.path));
const agentTransports = cassetteTransportProfiles(agentCassetteFiles);
const judgeTransport = readCassette(resolve(options.judgeCassetteFile)).transport;
if ((agentTransports === null) !== (judgeTransport === undefined)) {
fail("INCOMPATIBLE_IDENTITIES", "agent and judge transport identity generations differ");
}
const manifest: MeasurementManifest = {
...candidate,
complete: true,
fingerprints: {
...candidate.fingerprints,
...(agentTransports && judgeTransport
? { transportSha256: transportFingerprint(agentTransports, judgeTransport) }
: {}),
},
models: {
...candidate.models,
judge: {
requested: unknown(options.requestedJudgeModel),
observed: unknown(options.observedJudgeModel),
},
},
artifacts: [
...preserved,
artifactRef(runDir, resolve(options.judgeCassetteFile), "judge-cassette"),
artifactRef(runDir, checksFile, "recorded-checks"),
artifactRef(runDir, reportFile, "report"),
],
results: recomputeMeasurementCounts(recorded),
usage: {
agent: candidate.usage?.agent ?? observedUsage(agentCassetteFiles),
judge: observedUsage([resolve(options.judgeCassetteFile)]),
},
};
writeJson(file, manifest);
return { file, manifest };
}
function validateReview(value: unknown, id: string): ReviewReceipt {
const receipt = value as Partial<ReviewReceipt>;
const keys = value !== null && typeof value === "object" ? Object.keys(value).sort() : [];
const reviewerKeys =
receipt.reviewer !== null && typeof receipt.reviewer === "object"
? Object.keys(receipt.reviewer).sort()
: [];
if (
canonicalJson(keys) !==
canonicalJson(
["disposition", "measurementId", "reviewedAt", "reviewer", "schema", "summary"].sort(),
) ||
canonicalJson(reviewerKeys) !== canonicalJson(["kind", "label"].sort()) ||
receipt.schema !== 1 ||
receipt.measurementId !== id ||
(receipt.reviewer?.kind !== "human" && receipt.reviewer?.kind !== "independent") ||
typeof receipt.reviewer?.label !== "string" ||
receipt.reviewer.label.trim().length === 0 ||
(receipt.disposition !== "approved" && receipt.disposition !== "rejected") ||
typeof receipt.reviewedAt !== "string" ||
Number.isNaN(Date.parse(receipt.reviewedAt)) ||
typeof receipt.summary !== "string" ||
receipt.summary.trim().length === 0
) {
fail(
"REVIEW_INVALID",
"review receipt must identify the measurement, reviewer, disposition, time, and summary",
);
}
return receipt as ReviewReceipt;
}
function assertManifestShape(value: unknown): MeasurementManifest {
const manifest = value as Partial<MeasurementManifest>;
if (
manifest.schema !== 1 ||
typeof manifest.id !== "string" ||
typeof manifest.createdAt !== "string" ||
typeof manifest.complete !== "boolean" ||
!manifest.scenario ||
!manifest.source ||
!manifest.input ||
!manifest.fingerprints ||
!manifest.models ||
!manifest.provider ||
!Array.isArray(manifest.artifacts) ||
!manifest.results ||
!manifest.review
) {
fail("MANIFEST_INVALID", "measurement manifest is missing required version 1 fields");
}
validateMeasurementId(manifest.id, "MEASUREMENT_ID_INVALID");
assertString(manifest.scenario.id, "scenario.id");
if (!/^[a-z0-9][a-z0-9-]*$/.test(manifest.scenario.id)) {
fail("MANIFEST_INVALID", "scenario.id must be kebab-case");
}
if (manifest.scenario.kind !== "public-synthetic" && manifest.scenario.kind !== "original-real") {
fail("MANIFEST_INVALID", "scenario.kind is invalid");
}
if (typeof manifest.scenario.judged !== "boolean") {
fail("MANIFEST_INVALID", "scenario.judged must be boolean");
}
try {
validateSourceIdentity(manifest.source);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
fail(
error instanceof SourceIdentityError && message.includes("legacy dirty-only")
? "LEGACY_SOURCE_IDENTITY"
: "MANIFEST_INVALID",
message,
);
}
for (const [key, value] of Object.entries(manifest.input)) assertSha(value, `input.${key}`);
// Historical manifests omit transportSha256. The other five are required even when
// a later equality check would compare two missing values as equal.
for (const key of [
"promptSha256",
"systemSha256",
"toolsSha256",
"wireSha256",
"rubricSha256",
] as const) {
const value = manifest.fingerprints[key];
if (value !== "unknown") assertSha(value, `fingerprints.${key}`);
}
if (
manifest.fingerprints.transportSha256 !== undefined &&
manifest.fingerprints.transportSha256 !== "unknown"
) {
assertSha(manifest.fingerprints.transportSha256, "fingerprints.transportSha256");
}
assertString(manifest.models.agent.requested, "models.agent.requested");
assertString(manifest.models.agent.observed, "models.agent.observed");
assertString(manifest.models.judge.requested, "models.judge.requested");
assertString(manifest.models.judge.observed, "models.judge.observed");
assertString(manifest.provider.provider, "provider.provider");
assertSha(manifest.provider.endpointSha256, "provider.endpointSha256");
return manifest as MeasurementManifest;
}
function roleRefs(manifest: MeasurementManifest, role: ArtifactRole): ArtifactReference[] {
return manifest.artifacts.filter((ref) => ref.role === role);
}
function assertRequiredRoles(manifest: MeasurementManifest): void {
for (const ref of manifest.artifacts) {
if (!ARTIFACT_ROLES.has(ref.role))
fail("ARTIFACT_ROLE_INVALID", `unallowlisted role: ${ref.role}`);
}
for (const role of ["input-fixture", "main-cassette", "recorded-checks", "report"] as const) {
if (roleRefs(manifest, role).length === 0)
fail("CANDIDATE_INCOMPLETE", `missing ${role} artifact`);
}
const judgeCassettes = roleRefs(manifest, "judge-cassette");
if (judgeCassettes.length > 1) {
fail("CANDIDATE_INCOMPLETE", "measurement must reference at most one judge cassette");
}
if (manifest.scenario.judged) {
if (manifest.complete && judgeCassettes.length === 0) {
fail("CANDIDATE_INCOMPLETE", "judged measurement is missing its judge cassette");
}
} else if (judgeCassettes.length !== 0) {
fail("CANDIDATE_INCOMPLETE", "unjudged measurement must not include a judge cassette");
}
}
function assertBoundFingerprints(manifest: MeasurementManifest, report: WorkbenchReport): void {
const scenario = getScenario(manifest.scenario.id);
const bound = boundMeasurementFingerprints(scenario, report.fingerprint);
if (manifest.fingerprints.promptSha256 !== bound.promptSha256) {
fail("INCOMPATIBLE_IDENTITIES", "prompt fingerprint does not match the registered scenario");
}
if (manifest.fingerprints.rubricSha256 !== bound.rubricSha256) {
fail("INCOMPATIBLE_IDENTITIES", "rubric fingerprint does not match the registered scenario");
}
if (manifest.fingerprints.wireSha256 !== bound.wireSha256) {
fail("INCOMPATIBLE_IDENTITIES", "wire fingerprint does not match the bound report contract");
}
}
function assertCountsAgainstReport(manifest: MeasurementManifest, report: WorkbenchReport): void {
const scenario = report.scenarios.find((entry) => entry.scenarioId === manifest.scenario.id);
if (!scenario) fail("CHECK_COUNT_MISMATCH", "report does not contain the measured scenario");
const infrastructureFailures =
(scenario.failureClasses.TRANSPORT ?? 0) + (scenario.failureClasses.TIMEOUT ?? 0);
if (infrastructureFailures > 0) {
fail(
"CANDIDATE_INCOMPLETE",
"measurement contains infrastructure failure classes and cannot be adopted",
);
}
if (
report.scenarios.length !== 1 ||
scenario.reps !== manifest.results.repetitions ||
scenario.checks.length !== manifest.results.checks.length
) {
fail("CHECK_COUNT_MISMATCH", "report scenario/check denominators do not match the measurement");
}
const reportModel = manifest.scenario.judged
? manifest.models.judge.requested
: manifest.models.agent.requested;
if (
report.fingerprint.systemSha256 !== manifest.fingerprints.systemSha256 ||
report.fingerprint.toolsSha256 !== manifest.fingerprints.toolsSha256 ||
report.fingerprint.model !== reportModel ||
report.fingerprint.reps !== manifest.results.repetitions ||
report.fingerprint.effectiveSourceSha256 !== manifest.source.effectiveSha256
) {
fail("INCOMPATIBLE_IDENTITIES", "report fingerprint does not match the measurement identity");
}
for (const count of manifest.results.checks) {
const reported = scenario.checks.find(
(entry) => entry.id === count.id && entry.axis === count.axis,
);
if (
!reported ||
reported.weight !== count.weight ||
reported.passed !== count.passed ||
reported.total !== count.total ||
reported.indeterminate !== count.indeterminate
) {
fail("CHECK_COUNT_MISMATCH", `report summary changed ${count.axis}/${count.id}`);
}
}
if (
Math.abs(scenario.behaviour.rate - manifest.results.axisScores.behaviour) > 1e-12 ||
Math.abs(scenario.dsl.rate - manifest.results.axisScores.dsl) > 1e-12
) {
fail("AXIS_SCORE_MISMATCH", "report weighted behaviour/DSL scores changed");
}
}
/** Base64 bodies are opaque to the file-level scan. Inspect decoded request
* documents and Responses SSE payloads with the existing privacy rules.
* Scan keys and string values unescaped, not JSON serialization (whose escape
* sequences can also look like drive paths). */
export function assertCassettePrivacy(
cassette: ReturnType<typeof readCassette>,
knownSecrets: string[],
label: string,
): void {
const scan = (decoded: string): void => {
let scannable: string;
try {
const fragments: string[] = [];
const collect = (node: unknown): void => {
if (typeof node === "string") {
fragments.push(node);
return;
}
if (Array.isArray(node)) {
for (const item of node) collect(item);
return;
}
if (node && typeof node === "object") {
for (const [key, value] of Object.entries(node)) {
fragments.push(`"${key}":`);
collect(value);
}
}
};
collect(JSON.parse(decoded));
scannable = fragments.join("\n");
} catch {
scannable = decoded;
}
const issue = privacyFailure(scannable, knownSecrets);
if (issue) throw new MeasurementError(issue.code, `${label}: ${issue.message}`);
};
for (const round of cassette.rounds) {
if (round.requestBodyBase64) {
scan(Buffer.from(round.requestBodyBase64, "base64").toString("utf8"));
}
if (cassette.wireApi === "responses" && round.response) {
const decoded = Buffer.from(round.response.bodyBase64, "base64").toString("utf8");
for (const line of decoded.split("\n")) {
if (line.startsWith("data:")) scan(line.slice(5).trim());
}
}
}
}
function assertCassettes(
manifest: MeasurementManifest,
root: string,
knownSecrets: string[] = [],
): void {
const assertOne = (ref: ArtifactReference, judged: boolean): void => {
let cassette: ReturnType<typeof readCassette>;
try {
cassette = readCassette(resolveArtifact(root, ref.path));
} catch (error) {
fail("MANIFEST_INVALID", `${ref.path} is not a readable cassette: ${String(error)}`);
}
const expectedScenario = judged ? `judge-${manifest.scenario.id}` : manifest.scenario.id;
const expectedModel = judged
? manifest.models.judge.requested
: manifest.models.agent.requested;
const observedModel = judged ? manifest.models.judge.observed : manifest.models.agent.observed;
if (
cassette.scenario !== expectedScenario ||
cassette.provider !== manifest.provider.provider ||
cassette.model !== expectedModel ||
!Array.isArray(cassette.rounds) ||
cassette.rounds.length === 0
) {
fail("MANIFEST_INVALID", `${ref.path} cassette identity is inconsistent with the manifest`);
}
if (observedModel !== "unknown" && cassette.resolvedModel !== observedModel) {
fail("MANIFEST_INVALID", `${ref.path} observed model is inconsistent with the manifest`);
}
if (
cassette.rounds.some((round) => {
if (cassette.wireApi === "responses") {
return (
round.terminalStatus !== "completed" ||
!round.response ||
round.response.status < 200 ||
round.response.status >= 300
);
}
return (
typeof round.httpStatus === "number" &&
(round.httpStatus < 200 || round.httpStatus >= 300)
);
})
) {
fail(
"CANDIDATE_INCOMPLETE",
`${ref.path} contains a failed ${cassette.wireApi === "responses" ? "Responses" : "Chat"} round`,
);
}
if (
cassette.wireApi === "responses" &&
(!cassette.attempts ||
cassette.attempts.length !== cassette.rounds.length ||
cassette.attempts.some((attempt) => attempt.status !== "recorded"))
) {
fail(
"CANDIDATE_INCOMPLETE",
`${ref.path} contains an unmatched rejected or failed Responses attempt`,
);
}
if (judged) {
const judgedChecks = getScenario(manifest.scenario.id).judged ?? [];
const expectedRounds = manifest.results.repetitions * judgedChecks.length;
if (cassette.rounds.length !== expectedRounds) {
fail(
"CANDIDATE_INCOMPLETE",
`${ref.path} must retain ${expectedRounds} judge rounds (${manifest.results.repetitions} repetitions × ${judgedChecks.length} judged checks)`,
);
}
}
assertCassettePrivacy(cassette, knownSecrets, ref.path);
};
for (const ref of roleRefs(manifest, "main-cassette")) assertOne(ref, false);
for (const ref of roleRefs(manifest, "judge-cassette")) assertOne(ref, true);
}
function assertTransportFingerprint(manifest: MeasurementManifest, root: string): void {
const agentFiles = roleRefs(manifest, "main-cassette").map((ref) =>
resolveArtifact(root, ref.path),
);
const agent = cassetteTransportProfiles(agentFiles);
const judgeRef = roleRefs(manifest, "judge-cassette")[0];
const judge = judgeRef ? readCassette(resolveArtifact(root, judgeRef.path)).transport : undefined;
if (agent === null && judge === undefined) {
if (manifest.fingerprints.transportSha256 !== undefined) {
fail("INCOMPATIBLE_IDENTITIES", "legacy cassettes cannot claim a transport fingerprint");
}
return;
}
if (agent === null || (judgeRef && judge === undefined)) {
fail("INCOMPATIBLE_IDENTITIES", "measurement mixes legacy and transport-bound cassettes");
}
const expected = transportFingerprint(agent, judge ?? "no-judge");
if (manifest.fingerprints.transportSha256 !== expected) {
fail("INCOMPATIBLE_IDENTITIES", "measurement transport fingerprint does not match cassettes");
}
}
function assertObservedUsage(manifest: MeasurementManifest, root: string): void {
if (!manifest.usage) return;
const agent = observedUsage(
roleRefs(manifest, "main-cassette").map((ref) => resolveArtifact(root, ref.path)),
);
if (canonicalJson(agent) !== canonicalJson(manifest.usage.agent)) {
fail("MANIFEST_INVALID", "agent observed usage summary does not match cassettes");
}
const judgeRefs = roleRefs(manifest, "judge-cassette");
const judge =
judgeRefs.length === 0
? manifest.complete
? { status: "none" as const }
: { status: "pending" as const }
: observedUsage(judgeRefs.map((ref) => resolveArtifact(root, ref.path)));
if (canonicalJson(judge) !== canonicalJson(manifest.usage.judge)) {
fail("MANIFEST_INVALID", "judge observed usage summary does not match cassettes");
}
}
function assertFixture(manifest: MeasurementManifest, root: string): void {
const refs = roleRefs(manifest, "input-fixture");
if (refs.length !== 1) fail("CANDIDATE_INCOMPLETE", "exactly one input fixture is required");
const fixture = readJson(resolveArtifact(root, refs[0].path)) as Partial<PublicSyntheticFixture>;
const keys = Object.keys(fixture).sort();
const expectedKeys = [
"documentSha256",
"generator",
"kind",
"scenarioId",
"schema",
"telemetrySha256",
].sort();
if (canonicalJson(keys) !== canonicalJson(expectedKeys)) {
fail("MANIFEST_INVALID", "public synthetic fixture has fields outside its exact schema");
}
if (
fixture.schema !== 1 ||
fixture.kind !== "public-synthetic" ||
fixture.scenarioId !== manifest.scenario.id ||
fixture.generator !== PUBLIC_SYNTHETIC_SOURCES[manifest.scenario.id]
) {
fail("MANIFEST_INVALID", "public synthetic fixture identity does not match the manifest");
}
assertSha(fixture.documentSha256, "fixture.documentSha256");
assertSha(fixture.telemetrySha256, "fixture.telemetrySha256");
if (
manifest.input.fixtureSha256 !== refs[0].sha256 ||
manifest.input.documentSha256 !== fixture.documentSha256 ||
manifest.input.telemetrySha256 !== fixture.telemetrySha256
) {
fail("MANIFEST_INVALID", "fixture/document/telemetry identity does not match the manifest");
}
}
export interface VerifiedMeasurement {
directory: string;
manifest: MeasurementManifest;
recorded: RecordedChecks;
review: ReviewReceipt;
}
export function verifyMeasurementDirectory(
directory: string,
options: { knownSecrets?: string[]; requireApprovedReview?: boolean } = {},
): VerifiedMeasurement {
const root = resolve(directory);
if (!existsSync(root)) fail("MEASUREMENT_NOT_FOUND", `${directory} does not exist`);
const manifestFile = resolve(root, MANIFEST_FILE);
assertPublicArtifact(manifestFile, options.knownSecrets);
const manifest = assertManifestShape(readJson(manifestFile));
if (!manifest.complete) fail("CANDIDATE_INCOMPLETE", `${manifest.id} is not complete`);
if (
manifest.scenario.kind !== "public-synthetic" ||
!PUBLIC_SYNTHETIC_SCENARIOS.has(manifest.scenario.id)
) {
fail(
"SCENARIO_NOT_PUBLIC_SYNTHETIC",
`${manifest.scenario.id} is not registered public synthetic input`,
);
}
assertRequiredRoles(manifest);
const paths = new Set<string>();
for (const ref of manifest.artifacts) {
if (paths.has(ref.path)) fail("MANIFEST_INVALID", `duplicate artifact path ${ref.path}`);
paths.add(ref.path);
const file = assertArtifact(root, ref);
assertPublicArtifact(file, options.knownSecrets);
}
assertFixture(manifest, root);
assertCassettes(manifest, root, options.knownSecrets ?? []);
assertTransportFingerprint(manifest, root);
assertObservedUsage(manifest, root);
if (roleRefs(manifest, "main-cassette").length !== manifest.results.repetitions) {
fail("CANDIDATE_INCOMPLETE", "main cassette count must equal the recorded repetition count");
}
const checksRefs = roleRefs(manifest, "recorded-checks");
if (checksRefs.length !== 1)
fail("CANDIDATE_INCOMPLETE", "exactly one recorded-checks artifact is required");
const recorded = readJson(resolveArtifact(root, checksRefs[0].path)) as RecordedChecks;
if (!recorded.complete) fail("CANDIDATE_INCOMPLETE", "recorded checks are not complete");
if (recorded.scenarioId !== manifest.scenario.id) {
fail(
"CHECK_COUNT_MISMATCH",
"recorded-checks scenario does not match the measurement scenario",
);
}
const recomputed = recomputeMeasurementCounts(recorded);
if (canonicalJson(recomputed) !== canonicalJson(manifest.results)) {
fail("CHECK_COUNT_MISMATCH", "manifest k/n does not match recorded check outcomes");
}
const reportRefs = roleRefs(manifest, "report");
if (reportRefs.length !== 1)
fail("CANDIDATE_INCOMPLETE", "exactly one report artifact is required");
const report = readJson(resolveArtifact(root, reportRefs[0].path)) as WorkbenchReport;
assertCountsAgainstReport(manifest, report);
assertBoundFingerprints(manifest, report);
const reviewRefs = roleRefs(manifest, "review-receipt");
if (reviewRefs.length !== 1)
fail("REVIEW_MISSING", "exactly one explicit review receipt is required");
const review = validateReview(readJson(resolveArtifact(root, reviewRefs[0].path)), manifest.id);
if (
manifest.review.disposition !== review.disposition ||
manifest.review.reviewerKind !== review.reviewer.kind
) {
fail("REVIEW_INVALID", "manifest review summary does not match its receipt");
}
if (options.requireApprovedReview && review.disposition !== "approved") {
fail(
"REVIEW_NOT_APPROVED",
`${manifest.id} was reviewed with disposition ${review.disposition}`,
);
}
return { directory: root, manifest, recorded, review };
}
export function exportMeasurement(options: {
runDir: string;
id: string;
reviewFile: string;
measurementsDir?: string;
knownSecrets?: string[];
}): string {
validateMeasurementId(options.id);
const runDir = resolve(options.runDir);
if (!existsSync(runDir)) fail("MEASUREMENT_NOT_FOUND", `${options.runDir} does not exist`);
const candidate = assertManifestShape(readJson(resolve(runDir, CANDIDATE_FILE)));
if (!candidate.complete)
fail("CANDIDATE_INCOMPLETE", `${candidate.scenario.id} still needs its judge pass`);
if (
candidate.scenario.kind !== "public-synthetic" ||
!PUBLIC_SYNTHETIC_SCENARIOS.has(candidate.scenario.id)
) {
fail("SCENARIO_NOT_PUBLIC_SYNTHETIC", `${candidate.scenario.id} cannot be exported`);
}
assertRequiredRoles(candidate);
const reviewPath = resolve(options.reviewFile);
if (!existsSync(reviewPath)) fail("REVIEW_MISSING", `${options.reviewFile} does not exist`);
assertPublicArtifact(reviewPath, options.knownSecrets);
const review = validateReview(readJson(reviewPath), options.id);
const sourceFiles = candidate.artifacts.map((ref) => ({
ref,
file: assertArtifact(runDir, ref),
}));
for (const entry of sourceFiles) assertPublicArtifact(entry.file, options.knownSecrets);
const root = resolve(options.measurementsDir ?? MEASUREMENTS_DIR);
const destination = resolve(root, options.id);
if (!isWithin(root, destination))
fail("ARTIFACT_PATH_INVALID", "measurement id escapes destination");
if (existsSync(destination)) fail("DUPLICATE_MEASUREMENT", `${options.id} already exists`);
const temporary = resolve(root, `.tmp-${options.id}-${randomUUID()}`);
mkdirSync(temporary, { recursive: true });
try {
const refs: ArtifactReference[] = [];
for (const entry of sourceFiles) {
const target = resolve(temporary, ...entry.ref.path.split("/"));
mkdirSync(dirname(target), { recursive: true });
copyFileSync(entry.file, target, constants.COPYFILE_EXCL);
const copiedHash = hashFile(target);
if (copiedHash !== entry.ref.sha256) {
fail("ARTIFACT_HASH_MISMATCH", `${entry.ref.path} changed during adoption`);
}
refs.push(entry.ref);
}
const reviewName = "review.json";
const reviewTarget = resolve(temporary, reviewName);
copyFileSync(reviewPath, reviewTarget, constants.COPYFILE_EXCL);
refs.push(artifactRef(temporary, reviewTarget, "review-receipt"));
const manifest: MeasurementManifest = {
...candidate,
id: options.id,
artifacts: refs,
review: { disposition: review.disposition, reviewerKind: review.reviewer.kind },
};
writeJson(resolve(temporary, MANIFEST_FILE), manifest);
verifyMeasurementDirectory(temporary, { knownSecrets: options.knownSecrets });
mkdirSync(root, { recursive: true });
renameSync(temporary, destination);
} catch (error) {
rmSync(temporary, { recursive: true, force: true });
throw error;
}
return destination;
}
const IDENTITY_FIELDS: Array<[string, (identity: MeasurementIdentity) => string]> = [
["scenario", (x) => `${x.scenario.kind}:${x.scenario.id}:${x.scenario.judged}`],
["source.effectiveSha256", (x) => x.source.effectiveSha256],
["fixture", (x) => x.input.fixtureSha256],
["document", (x) => x.input.documentSha256],
["telemetry", (x) => x.input.telemetrySha256],
["prompt", (x) => x.fingerprints.promptSha256],
["system", (x) => x.fingerprints.systemSha256],
["tools", (x) => x.fingerprints.toolsSha256],
["wire", (x) => x.fingerprints.wireSha256],
["rubric", (x) => x.fingerprints.rubricSha256],
["transport", (x) => x.fingerprints.transportSha256 ?? "unknown"],
["provider", (x) => x.provider.provider],
["endpoint", (x) => x.provider.endpointSha256],
["agent.requested", (x) => x.models.agent.requested],
["agent.observed", (x) => x.models.agent.observed],
["judge.requested", (x) => x.models.judge.requested],
["judge.observed", (x) => x.models.judge.observed],
];
export function identityDifferences(
left: MeasurementIdentity,
right: MeasurementIdentity,
): string[] {
return IDENTITY_FIELDS.filter(([, read]) => read(left) !== read(right)).map(([name]) => name);
}
export function assertComparable(left: MeasurementIdentity, right: MeasurementIdentity): void {
requireSourceSchema(left.source);
requireSourceSchema(right.source);
for (const [name, read] of IDENTITY_FIELDS) {
if (
name === "transport" &&
left.fingerprints.transportSha256 === undefined &&
right.fingerprints.transportSha256 === undefined
) {
continue;
}
if (read(left) === "unknown" || read(right) === "unknown") {
fail("IDENTITY_UNKNOWN", `${name} is unknown and cannot establish comparability`);
}
}
const differences = identityDifferences(left, right);
if (differences.length > 0) {
fail(
"INCOMPATIBLE_IDENTITIES",
`non-comparable measurement identities: ${differences.join(", ")}`,
);
}
}
function manifestPath(id: string, measurementsDir = MEASUREMENTS_DIR): string {
validateMeasurementId(id);
return resolve(measurementsDir, id);
}
export function verifyMeasurementId(
id: string,
measurementsDir = MEASUREMENTS_DIR,
): VerifiedMeasurement {
const verified = verifyMeasurementDirectory(manifestPath(id, measurementsDir));
if (verified.manifest.id !== id) {
fail(
"MEASUREMENT_ID_MISMATCH",
`requested measurement ${id} contains manifest id ${verified.manifest.id}`,
);
}
return verified;
}
export async function replayMeasurement(
id: string,
measurementsDir = MEASUREMENTS_DIR,
): Promise<MeasurementCounts> {
const verified = verifyMeasurementId(id, measurementsDir);
const current = captureSourceIdentity();
assertReplaySourceCompatible(verified.manifest.source, current);
const scenario = getScenario(verified.manifest.scenario.id);
if ((scenario.judged ?? []).length > 0 !== verified.manifest.scenario.judged) {
fail(
"SOURCE_IDENTITY_MISMATCH",
"current scenario judging contract differs from the measurement",
);
}
const mainRefs = roleRefs(verified.manifest, "main-cassette");
const results: RepetitionResult[] = [];
for (const [rep, ref] of mainRefs.entries()) {
const cassetteFile = resolveArtifact(verified.directory, ref.path);
const cassette = readCassette(cassetteFile);
const replay = await startReplay({
file: cassetteFile,
onStale: "throw",
});
try {
results.push(
await runRepetition({
scenario,
rep,
endpoint: replay,
store: offlineStore({
baseUrl: replay.url,
allowAgentEdits: scenario.allowAgentEdits ?? true,
model: cassette.model,
}),
maxRetries: cassette.wireApi === "responses" ? 0 : undefined,
}),
);
} finally {
replay.close();
}
replay.assertFresh();
}
const judged = scenario.judged ?? [];
let scored = results.map((result) => ({ rep: result.rep, scored: result.scored }));
if (judged.length > 0) {
const refs = roleRefs(verified.manifest, "judge-cassette");
if (refs.length !== 1)
fail("CANDIDATE_INCOMPLETE", "judged replay needs exactly one judge cassette");
const judgeCassette = readCassette(resolveArtifact(verified.directory, refs[0].path));
const replay = await startReplay({
file: resolveArtifact(verified.directory, refs[0].path),
onStale: "throw",
});
try {
scored = [];
for (const result of results) {
const readings = new Map<string, JudgeReading>();
for (const check of judged) {
const reading = await askJudge({
endpoint: {
baseUrl: replay.url,
model: judgeCassette.model,
wireApi: judgeCassette.wireApi ?? "chat-completions",
},
rubric: check.rubric,
input: {
prompt: scenario.prompt,
answer: result.run.answer,
facts: check.facts(result.context),
},
});
readings.set(check.id, reading);
}
scored.push({ rep: result.rep, scored: scoreRun(scenario, result.context, readings) });
}
} finally {
replay.close();
}
replay.assertFresh();
}
const actual = recomputeMeasurementCounts(recordedChecksFromScored(scenario.id, scored, true));
if (canonicalJson(actual) !== canonicalJson(verified.manifest.results)) {
fail("REPLAY_MISMATCH", "offline cassette replay did not reproduce recorded check outcomes");
}
return actual;
}
export interface BoundBaseline {
schema: 2;
binding: "measurement-v1";
measurementId: string;
recordedAt: string;
effectiveSourceSha256: string;
identity: MeasurementIdentity;
counts: MeasurementCounts;
expectedFailures: string[];
}
export function boundBaselineFromMeasurement(verified: VerifiedMeasurement): BoundBaseline {
if (verified.review.disposition !== "approved") {
fail("REVIEW_NOT_APPROVED", `${verified.manifest.id} is not approved for a baseline claim`);
}
assertComparable(verified.manifest, verified.manifest);
return {
schema: 2,
binding: "measurement-v1",
measurementId: verified.manifest.id,
recordedAt: verified.manifest.createdAt,
effectiveSourceSha256: verified.manifest.source.effectiveSha256,
identity: {
scenario: verified.manifest.scenario,
source: verified.manifest.source,
input: verified.manifest.input,
fingerprints: verified.manifest.fingerprints,
models: verified.manifest.models,
provider: verified.manifest.provider,
},
counts: verified.manifest.results,
expectedFailures: verified.manifest.results.checks
.filter((check) => check.decided > check.passed)
.map((check) => check.id)
.sort(),
};
}
export function writeBoundBaseline(file: string, baseline: BoundBaseline): void {
if (existsSync(file)) fail("DUPLICATE_MEASUREMENT", `${file} already exists`);
writeJson(file, baseline);
}
export function verifyBoundBaseline(baseline: BoundBaseline, verified: VerifiedMeasurement): void {
if (baseline.schema !== 2 || baseline.binding !== "measurement-v1") {
fail("MANIFEST_INVALID", "baseline is legacy/unbound, so its numbers cannot be verified");
}
if (baseline.measurementId !== verified.manifest.id) {
fail("INCOMPATIBLE_IDENTITIES", "baseline names a different measurement");
}
if (
baseline.effectiveSourceSha256 !== baseline.identity.source.effectiveSha256 ||
baseline.effectiveSourceSha256 !== verified.manifest.source.effectiveSha256
) {
fail("SOURCE_IDENTITY_MISMATCH", "baseline effective source does not match its measurement");
}
assertComparable(baseline.identity, verified.manifest);
if (canonicalJson(baseline.counts) !== canonicalJson(verified.manifest.results)) {
fail("CHECK_COUNT_MISMATCH", "baseline k/n does not match its bound measurement");
}
const expected = boundBaselineFromMeasurement(verified);
if (canonicalJson(baseline.expectedFailures) !== canonicalJson(expected.expectedFailures)) {
fail("CHECK_COUNT_MISMATCH", "baseline expectedFailures do not match its bound measurement");
}
if (baseline.recordedAt !== expected.recordedAt) {
fail("INCOMPATIBLE_IDENTITIES", "baseline recordedAt does not match its measurement");
}
if (canonicalJson(baseline.identity) !== canonicalJson(expected.identity)) {
fail("INCOMPATIBLE_IDENTITIES", "baseline identity is not rebound from the measurement");
}
}