Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
import { randomUUID } from "node:crypto";
import {
mkdirSync,
mkdtempSync,
readFileSync,
renameSync,
rmSync,
symlinkSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { readCassette } from "../lib/cassette";
import {
type ArtifactReference,
assertCassettePrivacy,
assertComparable,
boundBaselineFromMeasurement,
boundMeasurementFingerprints,
exportMeasurement,
MeasurementError,
type MeasurementManifest,
type RecordedChecks,
recomputeMeasurementCounts,
verifyBoundBaseline,
verifyMeasurementDirectory,
verifyMeasurementId,
weightedAxisTrials,
} from "../lib/measurement";
import {
MANDATORY_SOURCE_ANCHORS,
sha256Bytes,
sha256Canonical,
sourceIdentityFromManifests,
} from "../lib/provenance";
import { newcombeDelta, wilson95 } from "../lib/stats";
import { runMeasurementCli } from "../measurement-cli";
import { getScenario } from "../scenarios/registry";
const roots: string[] = [];
const cliOutputs: string[] = [];
const HASH = (letter: string) => letter.repeat(64);
function writeJson(file: string, value: unknown): void {
writeFileSync(file, `${JSON.stringify(value, null, "\t")}\n`, "utf8");
}
function testSourceIdentity() {
return sourceIdentityFromManifests({
commit: HASH("a").slice(0, 40),
dirtyManifest: [],
effectiveManifest: MANDATORY_SOURCE_ANCHORS.map((path) => ({
path,
kind: "text" as const,
sha256: sha256Bytes(`test source ${path}\n`),
})),
});
}
function createCandidate(root: string): {
runDir: string;
measurementsDir: string;
review: string;
manifest: MeasurementManifest;
} {
const runDir = join(root, "run");
const measurementsDir = join(root, "measurements");
mkdirSync(runDir, { recursive: true });
const recorded: RecordedChecks = {
schema: 1,
scenarioId: "target-right-clip",
complete: true,
repetitions: [
{
rep: 0,
checks: [
{
id: "beh.remove",
axis: "behaviour",
weight: 1,
ok: true,
indeterminate: false,
},
{
id: "dsl.remove",
axis: "dsl",
weight: 3,
ok: false,
indeterminate: false,
},
],
},
],
};
const counts = recomputeMeasurementCounts(recorded);
const source = testSourceIdentity();
const fixture = {
schema: 1,
kind: "public-synthetic",
scenarioId: "target-right-clip",
generator: "workbench/scenarios/target-right-clip.scn.ts",
documentSha256: HASH("b"),
telemetrySha256: HASH("c"),
};
const report = {
label: "test-only",
createdAt: "2026-09-12T00:00:00.000Z",
fingerprint: {
systemSha256: HASH("d"),
systemChars: 1,
toolsSha256: HASH("e"),
toolNames: [],
model: "loopback-model",
gitSha: "abc1234",
gitDirty: true,
effectiveSourceSha256: source.effectiveSha256,
overlayId: null,
reps: 1,
},
minDetectableEffect: 1,
scenarios: [
{
scenarioId: "target-right-clip",
title: "test",
tags: [],
gate: 0,
reps: 1,
behaviour: { ...wilson95(1, 1), rate: counts.axisScores.behaviour },
dsl: { ...wilson95(0, 1), rate: counts.axisScores.dsl },
gateScoreMean: 0,
passRate: wilson95(0, 1),
failureClasses: { NONE: 1 },
checks: counts.checks.map((check) => ({
id: check.id,
axis: check.axis,
weight: check.weight,
passed: check.passed,
total: check.total,
indeterminate: check.indeterminate,
wilson: wilson95(check.passed, check.decided),
expected: false,
evidence: [],
})),
unmeasuredAxes: [],
msMean: 1,
},
],
notices: [],
};
const files: Array<{ name: string; role: ArtifactReference["role"]; value: unknown }> = [
{ name: "input-fixture.json", role: "input-fixture", value: fixture },
{
name: "main-cassette.json",
role: "main-cassette",
value: {
scenario: "target-right-clip",
provider: "loopback",
model: "loopback-model",
resolvedModel: "loopback-model",
recordedAt: "2026-09-12",
rounds: [
{
round: 0,
requestHash: "0123456789abcdef",
digest: { systemChars: 1, toolCount: 1, roles: ["user"], lastUserText: "test" },
sse: "data: [DONE]\\n\\n",
},
],
},
},
{ name: "recorded-checks.json", role: "recorded-checks", value: recorded },
{ name: "measurement-report.json", role: "report", value: report },
];
const artifacts = files.map(({ name, role, value }) => {
const file = join(runDir, name);
writeJson(file, value);
return { role, path: name, sha256: sha256Bytes(readFileSync(file)) };
});
const manifest: MeasurementManifest = {
schema: 1,
id: "candidate",
createdAt: "2026-09-12T00:00:00.000Z",
complete: true,
scenario: { id: "target-right-clip", kind: "public-synthetic", judged: false },
source,
input: {
fixtureSha256: artifacts[0].sha256,
documentSha256: fixture.documentSha256,
telemetrySha256: fixture.telemetrySha256,
},
fingerprints: {
...boundMeasurementFingerprints(getScenario("target-right-clip"), {
systemSha256: HASH("d"),
toolsSha256: HASH("e"),
toolNames: [],
}),
systemSha256: HASH("d"),
toolsSha256: HASH("e"),
},
models: {
agent: { requested: "loopback-model", observed: "loopback-model" },
judge: { requested: "none", observed: "none" },
},
provider: { provider: "loopback", endpointSha256: HASH("6") },
artifacts,
results: counts,
review: { disposition: "unreviewed", reviewerKind: "unknown" },
};
writeJson(join(runDir, "measurement-candidate.json"), manifest);
const review = join(root, "review.json");
writeJson(review, {
schema: 1,
measurementId: "valid-measurement",
reviewer: { kind: "independent", label: "test fixture reviewer" },
disposition: "approved",
reviewedAt: "2026-09-12T00:10:00.000Z",
summary: "Test-only review fixture for measurement mechanism validation.",
});
return { runDir, measurementsDir, review, manifest };
}
function root(): string {
const value = mkdtempSync(join(tmpdir(), "wb-measurement-"));
roots.push(value);
return value;
}
function expectCode(action: () => unknown, code: string): void {
try {
action();
throw new Error("expected measurement error");
} catch (error) {
expect(error).toBeInstanceOf(MeasurementError);
expect((error as MeasurementError).code).toBe(code);
}
}
function rewriteCandidate(runDir: string, change: (manifest: MeasurementManifest) => void): void {
const file = join(runDir, "measurement-candidate.json");
const manifest = JSON.parse(readFileSync(file, "utf8")) as MeasurementManifest;
change(manifest);
writeJson(file, manifest);
}
function exportValid(base: ReturnType<typeof createCandidate>): string {
return exportMeasurement({
runDir: base.runDir,
id: "valid-measurement",
reviewFile: base.review,
measurementsDir: base.measurementsDir,
});
}
afterEach(() => {
for (const value of roots.splice(0)) rmSync(value, { recursive: true, force: true });
for (const value of cliOutputs.splice(0)) rmSync(value, { recursive: true, force: true });
});
describe("versioned measurement export and verification", () => {
it("scans decoded Responses request bodies for private material", () => {
const base64 = (text: string) => Buffer.from(text, "utf8").toString("base64");
const cassette = {
scenario: "privacy-probe",
provider: "openai-compatible",
model: "workbench-loopback-fixture",
resolvedModel: "workbench",
recordedAt: "2026-09-20T00:00:00.000Z",
rounds: [
{
round: 0,
requestHash: "0".repeat(64),
digest: { systemChars: 1, toolCount: 0, roles: [], lastUserText: "" },
sse: "",
requestBodyBase64: base64(JSON.stringify({ prompt: "look at /Users/alice/private.txt" })),
},
],
attempts: [],
wireApi: "responses",
} as unknown as ReturnType<typeof readCassette>;
expect(() => assertCassettePrivacy(cassette, [], "probe.json")).toThrow(
expect.objectContaining({ code: "PRIVACY_PRIVATE_PATH" }),
);
const clean = {
...cassette,
rounds: [
{
...cassette.rounds[0],
requestBodyBase64: base64(JSON.stringify({ prompt: "clean request" })),
},
],
} as unknown as ReturnType<typeof readCassette>;
expect(() => assertCassettePrivacy(clean, [], "probe.json")).not.toThrow();
});
it("keeps fractional weighted scores exact through the comparison trials", () => {
const checks = (heavyPasses: boolean) => [
{ id: "beh.light", axis: "behaviour" as const, weight: 1, ok: true, indeterminate: false },
{
id: "beh.heavy",
axis: "behaviour" as const,
weight: 3,
ok: heavyPasses,
indeterminate: false,
},
{ id: "beh.unread", axis: "behaviour" as const, weight: 10, ok: false, indeterminate: true },
{ id: "dsl.complete", axis: "dsl" as const, weight: 2, ok: true, indeterminate: false },
];
const counts = recomputeMeasurementCounts({
schema: 1,
scenarioId: "fractional-probe",
complete: true,
repetitions: [
{ rep: 0, checks: checks(false) },
{ rep: 1, checks: checks(true) },
],
});
expect(counts.axisScores.behaviour).toBe(0.625);
const trials = weightedAxisTrials(counts, "behaviour");
expect(trials).toEqual({ k: 1.25, n: 2 });
// The comparison must read the real 0.625, not the rounded 0.5.
const delta = newcombeDelta({ k: 1, n: 2 }, trials);
expect(delta.point).toBeCloseTo(0.625 - 0.5, 12);
});
it("scores a wholly indeterminate repetition-axis as undecided, not perfect", () => {
const counts = recomputeMeasurementCounts({
schema: 1,
scenarioId: "undecided-probe",
complete: true,
repetitions: [
{
rep: 0,
checks: [
{ id: "beh.a", axis: "behaviour", weight: 1, ok: false, indeterminate: true },
{ id: "dsl.a", axis: "dsl", weight: 1, ok: false, indeterminate: true },
],
},
{
rep: 1,
checks: [
{ id: "beh.a", axis: "behaviour", weight: 1, ok: true, indeterminate: false },
{ id: "dsl.a", axis: "dsl", weight: 1, ok: false, indeterminate: true },
],
},
],
});
// rep 0 decided nothing: it must not contribute a placeholder perfect score.
expect(counts.axisScores.behaviour).toBe(1);
expect(counts.measuredRepetitions.behaviour).toBe(1);
// dsl decided nothing in either repetition: score 0, never a nominal 100%.
expect(counts.axisScores.dsl).toBe(0);
expect(counts.measuredRepetitions.dsl).toBe(0);
});
it("recomputes the existing weighted axis mean while excluding indeterminate weight", () => {
const checks = (heavyPasses: boolean) => [
{
id: "beh.light",
axis: "behaviour" as const,
weight: 1,
ok: true,
indeterminate: false,
},
{
id: "beh.heavy",
axis: "behaviour" as const,
weight: 3,
ok: heavyPasses,
indeterminate: false,
},
{
id: "beh.unread",
axis: "behaviour" as const,
weight: 10,
ok: false,
indeterminate: true,
},
{
id: "dsl.complete",
axis: "dsl" as const,
weight: 2,
ok: true,
indeterminate: false,
},
];
const counts = recomputeMeasurementCounts({
schema: 1,
scenarioId: "weighted-probe",
complete: true,
repetitions: [
{ rep: 0, checks: checks(false) },
{ rep: 1, checks: checks(true) },
],
});
// Unweighted behaviour k/n is 3/4 = .75. Existing score semantics are
// mean((1/4), (4/4)) = .625; the undecided weight 10 is in neither term.
expect(counts.axes.behaviour).toEqual({
passed: 3,
decided: 4,
indeterminate: 2,
total: 6,
});
expect(counts.axisScores.behaviour).toBe(0.625);
expect(counts.axisScores.dsl).toBe(1);
// The trial stays fractional: rounding 0.625 x 2 into whole successes read a
// 0.625 mean as 0.5 in the Newcombe comparison.
expect(weightedAxisTrials(counts, "behaviour")).toEqual({ k: 1.25, n: 2 });
expect(weightedAxisTrials(counts, "dsl")).toEqual({ k: 2, n: 2 });
});
it("accepts the exact finite CLI argument shapes without reading environment configuration", async () => {
const base = createCandidate(root());
const id = `test-cli-${randomUUID()}`;
writeJson(base.review, {
schema: 1,
measurementId: id,
reviewer: { kind: "independent", label: "test fixture reviewer" },
disposition: "approved",
reviewedAt: "2026-09-12T00:10:00.000Z",
summary: "Test-only review fixture for CLI validation.",
});
cliOutputs.push(join(process.cwd(), "workbench", "measurements", id));
const output = vi.spyOn(process.stdout, "write").mockImplementation(() => true);
const errors = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
try {
expect(
await runMeasurementCli([
"export",
"--run",
base.runDir,
"--id",
id,
"--review",
base.review,
]),
).toBe(0);
expect(await runMeasurementCli(["verify", "--id", id])).toBe(0);
expect(
await runMeasurementCli([
"baseline",
"--id",
id,
"--out",
join(base.runDir, "bound-baseline.json"),
]),
).toBe(0);
expect(await runMeasurementCli(["compare", "--left", id, "--right", id])).toBe(0);
expect(await runMeasurementCli(["verify", "--left", "wrong"])).toBe(1);
expect(errors).toHaveBeenCalledWith(expect.stringContaining("ARGUMENT_ERROR"));
expect(output).toHaveBeenCalled();
} finally {
output.mockRestore();
errors.mockRestore();
}
});
it("atomically adopts an allowlisted candidate and refuses overwrite", () => {
const base = createCandidate(root());
const directory = exportValid(base);
const verified = verifyMeasurementDirectory(directory, { requireApprovedReview: true });
expect(verified.manifest.id).toBe("valid-measurement");
expect(verified.manifest.review).toEqual({
disposition: "approved",
reviewerKind: "independent",
});
expectCode(() => exportValid(base), "DUPLICATE_MEASUREMENT");
});
it("rejects an invalid embedded id and a requested-folder id mismatch", () => {
const invalid = createCandidate(root());
const invalidDirectory = exportValid(invalid);
const manifestFile = join(invalidDirectory, "measurement.json");
const manifest = JSON.parse(readFileSync(manifestFile, "utf8")) as MeasurementManifest;
manifest.id = "../not-portable";
writeJson(manifestFile, manifest);
expectCode(() => verifyMeasurementDirectory(invalidDirectory), "MEASUREMENT_ID_INVALID");
const mismatch = createCandidate(root());
const adopted = exportValid(mismatch);
const requestedId = "requested-measurement";
const renamed = join(mismatch.measurementsDir, requestedId);
renameSync(adopted, renamed);
expectCode(
() => verifyMeasurementId(requestedId, mismatch.measurementsDir),
"MEASUREMENT_ID_MISMATCH",
);
});
it("rejects missing review, traversal, and unallowlisted roles", () => {
const missing = createCandidate(root());
expectCode(
() =>
exportMeasurement({
runDir: missing.runDir,
id: "valid-measurement",
reviewFile: join(missing.runDir, "absent.json"),
measurementsDir: missing.measurementsDir,
}),
"REVIEW_MISSING",
);
const traversal = createCandidate(root());
rewriteCandidate(traversal.runDir, (manifest) => {
manifest.artifacts[0].path = "../outside.json";
});
expectCode(() => exportValid(traversal), "ARTIFACT_PATH_INVALID");
const role = createCandidate(root());
rewriteCandidate(role.runDir, (manifest) => {
(manifest.artifacts[0] as { role: string }).role = "raw-config";
});
expectCode(() => exportValid(role), "ARTIFACT_ROLE_INVALID");
});
it("rejects hash and recomputed count tampering", () => {
const hash = createCandidate(root());
writeFileSync(join(hash.runDir, "main-cassette.json"), "tampered\n", "utf8");
expectCode(() => exportValid(hash), "ARTIFACT_HASH_MISMATCH");
const counts = createCandidate(root());
rewriteCandidate(counts.runDir, (manifest) => {
manifest.results.axes.behaviour.passed = 0;
});
expectCode(() => exportValid(counts), "CHECK_COUNT_MISMATCH");
expect(() =>
readFileSync(join(counts.measurementsDir, "valid-measurement", "measurement.json")),
).toThrow();
const extraJudge = createCandidate(root());
rewriteCandidate(extraJudge.runDir, (manifest) => {
const main = manifest.artifacts.find((entry) => entry.role === "main-cassette");
if (main) {
manifest.artifacts.push({
role: "judge-cassette",
path: main.path,
sha256: main.sha256,
});
}
});
expectCode(() => exportValid(extraJudge), "CANDIDATE_INCOMPLETE");
const foreignChecks = createCandidate(root());
const checksFile = join(foreignChecks.runDir, "recorded-checks.json");
const recorded = JSON.parse(readFileSync(checksFile, "utf8")) as RecordedChecks;
recorded.scenarioId = "another-scenario";
writeJson(checksFile, recorded);
rewriteCandidate(foreignChecks.runDir, (manifest) => {
const artifact = manifest.artifacts.find((entry) => entry.path === "recorded-checks.json");
if (artifact) artifact.sha256 = sha256Bytes(readFileSync(checksFile));
});
expectCode(() => exportValid(foreignChecks), "CHECK_COUNT_MISMATCH");
const transport = createCandidate(root());
const reportFile = join(transport.runDir, "measurement-report.json");
const report = JSON.parse(readFileSync(reportFile, "utf8")) as {
scenarios: Array<{ failureClasses: Record<string, number> }>;
};
report.scenarios[0].failureClasses = { TRANSPORT: 1 };
writeJson(reportFile, report);
rewriteCandidate(transport.runDir, (manifest) => {
const artifact = manifest.artifacts.find((entry) => entry.path === "measurement-report.json");
if (artifact) artifact.sha256 = sha256Bytes(readFileSync(reportFile));
});
expectCode(() => exportValid(transport), "CANDIDATE_INCOMPLETE");
const source = createCandidate(root());
rewriteCandidate(source.runDir, (manifest) => {
manifest.source.effectiveManifest[0].sha256 = HASH("0");
});
expectCode(() => exportValid(source), "MANIFEST_INVALID");
const legacy = createCandidate(root());
rewriteCandidate(legacy.runDir, (manifest) => {
manifest.source = {
commit: manifest.source.commit,
contentManifest: manifest.source.dirtyManifest,
contentSha256: manifest.source.dirtySha256,
} as unknown as typeof manifest.source;
});
expectCode(() => exportValid(legacy), "LEGACY_SOURCE_IDENTITY");
});
it("rejects fixture and report identity mismatches even when their artifact hashes are updated", () => {
const fixture = createCandidate(root());
const fixtureFile = join(fixture.runDir, "input-fixture.json");
const fixtureValue = JSON.parse(readFileSync(fixtureFile, "utf8")) as {
documentSha256: string;
};
fixtureValue.documentSha256 = HASH("9");
writeJson(fixtureFile, fixtureValue);
rewriteCandidate(fixture.runDir, (manifest) => {
const ref = manifest.artifacts.find((entry) => entry.path === "input-fixture.json")!;
ref.sha256 = sha256Bytes(readFileSync(fixtureFile));
manifest.input.fixtureSha256 = ref.sha256;
});
expectCode(() => exportValid(fixture), "MANIFEST_INVALID");
const report = createCandidate(root());
const reportFile = join(report.runDir, "measurement-report.json");
const reportValue = JSON.parse(readFileSync(reportFile, "utf8")) as {
fingerprint: { model: string };
};
reportValue.fingerprint.model = "different-model";
writeJson(reportFile, reportValue);
rewriteCandidate(report.runDir, (manifest) => {
manifest.artifacts.find((entry) => entry.path === "measurement-report.json")!.sha256 =
sha256Bytes(readFileSync(reportFile));
});
expectCode(() => exportValid(report), "INCOMPATIBLE_IDENTITIES");
const axis = createCandidate(root());
const axisReportFile = join(axis.runDir, "measurement-report.json");
const axisReport = JSON.parse(readFileSync(axisReportFile, "utf8")) as {
scenarios: Array<{ behaviour: { rate: number } }>;
};
axisReport.scenarios[0].behaviour.rate = 0.5;
writeJson(axisReportFile, axisReport);
rewriteCandidate(axis.runDir, (manifest) => {
manifest.artifacts.find((entry) => entry.path === "measurement-report.json")!.sha256 =
sha256Bytes(readFileSync(axisReportFile));
});
expectCode(() => exportValid(axis), "AXIS_SCORE_MISMATCH");
});
it.each([
["Authorization: Bearer test-secret-value", "PRIVACY_SECRET"],
['{"note":"C:\\\\Users\\\\private\\\\fixture.json"}', "PRIVACY_PRIVATE_PATH"],
['{"transcript":"private words"}', "PRIVACY_TRANSCRIPT"],
['{"baseUrl":"https://private.invalid/v1"}', "PRIVACY_RAW_CONFIG"],
])("rejects unsafe exported material (%s)", (payload, code) => {
const base = createCandidate(root());
const file = join(base.runDir, "main-cassette.json");
writeFileSync(file, payload, "utf8");
rewriteCandidate(base.runDir, (manifest) => {
manifest.artifacts.find((ref) => ref.path === "main-cassette.json")!.sha256 = sha256Bytes(
readFileSync(file),
);
});
expectCode(() => exportValid(base), code);
});
it("rejects a credential literal supplied by the caller even without a familiar prefix", () => {
const base = createCandidate(root());
const file = join(base.runDir, "main-cassette.json");
const secret = "opaque-credential-literal-459";
writeFileSync(file, `provider response ${secret}\n`, "utf8");
rewriteCandidate(base.runDir, (manifest) => {
manifest.artifacts.find((ref) => ref.path === "main-cassette.json")!.sha256 = sha256Bytes(
readFileSync(file),
);
});
expectCode(
() =>
exportMeasurement({
runDir: base.runDir,
id: "valid-measurement",
reviewFile: base.review,
measurementsDir: base.measurementsDir,
knownSecrets: [secret],
}),
"PRIVACY_SECRET",
);
});
it.skipIf(process.platform === "win32")("rejects a directory symlink artifact on posix", () => {
const base = createCandidate(root());
const outsideDirectory = join(dirname(base.runDir), "outside");
mkdirSync(outsideDirectory);
const outside = join(outsideDirectory, "outside.json");
writeFileSync(outside, "{}\n", "utf8");
const link = join(base.runDir, "linked");
symlinkSync(outsideDirectory, link, "dir");
rewriteCandidate(base.runDir, (manifest) => {
manifest.artifacts[0] = {
role: "input-fixture",
path: "linked/outside.json",
sha256: sha256Bytes(readFileSync(outside)),
};
});
expectCode(() => exportValid(base), "SYMLINK_TRAVERSAL");
});
it.skipIf(process.platform !== "win32")("rejects a junction artifact on windows", () => {
const base = createCandidate(root());
const outsideDirectory = join(dirname(base.runDir), "outside");
mkdirSync(outsideDirectory);
const outside = join(outsideDirectory, "outside.json");
writeFileSync(outside, "{}\n", "utf8");
const link = join(base.runDir, "linked");
symlinkSync(outsideDirectory, link, "junction");
rewriteCandidate(base.runDir, (manifest) => {
manifest.artifacts[0] = {
role: "input-fixture",
path: "linked/outside.json",
sha256: sha256Bytes(readFileSync(outside)),
};
});
expectCode(() => exportValid(base), "SYMLINK_TRAVERSAL");
});
});
describe("measurement identity and bound baseline", () => {
it("recomputes weighted axis means while excluding indeterminate weight", () => {
const counts = recomputeMeasurementCounts({
schema: 1,
scenarioId: "weighted-probe",
complete: true,
repetitions: [
{
rep: 0,
checks: [
{ id: "beh.light", axis: "behaviour", weight: 1, ok: true, indeterminate: false },
{ id: "beh.heavy", axis: "behaviour", weight: 3, ok: false, indeterminate: false },
{ id: "beh.unknown", axis: "behaviour", weight: 9, ok: false, indeterminate: true },
{ id: "dsl.pass", axis: "dsl", weight: 2, ok: true, indeterminate: false },
],
},
],
});
// Unweighted behaviour is 1/2; the existing weighted metric is 1/(1+3).
expect(counts.axes.behaviour).toEqual({ passed: 1, decided: 2, indeterminate: 1, total: 3 });
expect(counts.axisScores).toEqual({ behaviour: 0.25, dsl: 1 });
expect(counts.checks.find((check) => check.id === "beh.unknown")?.weight).toBe(9);
});
it("keeps the wire fingerprint independent of sample size", () => {
const contract = { systemSha256: HASH("d"), toolsSha256: HASH("e"), toolNames: ["read"] };
expect(sha256Canonical([contract, contract])).not.toBe(sha256Canonical(contract));
expect(sha256Canonical(contract)).toBe(
sha256Canonical({
systemSha256: HASH("d"),
toolsSha256: HASH("e"),
toolNames: ["read"],
}),
);
});
it("refuses unknown or incompatible identities and never silently compares them", () => {
const base = createCandidate(root());
const left = base.manifest;
const committed = structuredClone(left);
committed.source = sourceIdentityFromManifests({
commit: HASH("b").slice(0, 40),
dirtyManifest: [left.source.effectiveManifest[0]],
effectiveManifest: left.source.effectiveManifest,
});
expect(() => assertComparable(left, committed)).not.toThrow();
const unknown = structuredClone(left);
unknown.models.agent.observed = "unknown";
expectCode(() => assertComparable(left, unknown), "IDENTITY_UNKNOWN");
const changed = structuredClone(left);
changed.input.documentSha256 = HASH("9");
expectCode(() => assertComparable(left, changed), "INCOMPATIBLE_IDENTITIES");
const legacy = structuredClone(left) as unknown as {
source: unknown;
};
legacy.source = {
commit: HASH("a").slice(0, 40),
contentManifest: [],
contentSha256: HASH("1"),
};
expectCode(
() => assertComparable(legacy as unknown as typeof left, left),
"LEGACY_SOURCE_IDENTITY",
);
});
it("binds k/n to approved evidence and rejects a changed baseline summary", () => {
const directory = exportValid(createCandidate(root()));
const verified = verifyMeasurementDirectory(directory, { requireApprovedReview: true });
const baseline = boundBaselineFromMeasurement(verified);
expect(() => verifyBoundBaseline(baseline, verified)).not.toThrow();
const changed = structuredClone(baseline);
changed.counts.axes.dsl.decided += 1;
expectCode(() => verifyBoundBaseline(changed, verified), "CHECK_COUNT_MISMATCH");
const changedSource = structuredClone(baseline);
changedSource.effectiveSourceSha256 = HASH("0");
expectCode(() => verifyBoundBaseline(changedSource, verified), "SOURCE_IDENTITY_MISMATCH");
const changedFailures = structuredClone(baseline);
changedFailures.expectedFailures = ["whatever-I-want"];
expectCode(() => verifyBoundBaseline(changedFailures, verified), "CHECK_COUNT_MISMATCH");
expect(() => assertComparable(baseline.identity, verified.manifest)).not.toThrow();
});
it("requires every fingerprint except a historical missing transport hash", () => {
const adopted = exportValid(createCandidate(root()));
const adoptedManifest = JSON.parse(
readFileSync(join(adopted, "measurement.json"), "utf8"),
) as MeasurementManifest;
expect(adoptedManifest.fingerprints.transportSha256).toBeUndefined();
expect(() => verifyMeasurementDirectory(adopted)).not.toThrow();
const required = [
"promptSha256",
"systemSha256",
"toolsSha256",
"wireSha256",
"rubricSha256",
] as const;
for (const key of required) {
const base = createCandidate(root());
const scenario = getScenario("target-right-clip");
const reportFile = join(base.runDir, "measurement-report.json");
const report = JSON.parse(readFileSync(reportFile, "utf8")) as {
fingerprint: {
systemSha256?: string;
toolsSha256?: string;
toolNames?: string[];
};
};
if (key === "systemSha256" || key === "toolsSha256") {
delete report.fingerprint[key];
writeJson(reportFile, report);
}
rewriteCandidate(base.runDir, (manifest) => {
if (key === "systemSha256" || key === "toolsSha256") {
manifest.fingerprints.wireSha256 = boundMeasurementFingerprints(scenario, {
systemSha256: report.fingerprint.systemSha256,
toolsSha256: report.fingerprint.toolsSha256,
toolNames: report.fingerprint.toolNames ?? [],
}).wireSha256;
const reportRef = manifest.artifacts.find(
(entry) => entry.path === "measurement-report.json",
);
if (!reportRef) throw new Error("missing report artifact");
reportRef.sha256 = sha256Bytes(readFileSync(reportFile));
}
delete (manifest.fingerprints as Partial<MeasurementManifest["fingerprints"]>)[key];
});
try {
exportValid(base);
throw new Error(`expected ${key} to be required`);
} catch (error) {
expect(error).toBeInstanceOf(MeasurementError);
expect((error as MeasurementError).code).toBe("MANIFEST_INVALID");
expect((error as MeasurementError).message).toContain(`fingerprints.${key}`);
}
}
const transport = createCandidate(root());
rewriteCandidate(transport.runDir, (manifest) => {
manifest.fingerprints.transportSha256 = "not-a-sha";
});
try {
exportValid(transport);
throw new Error("expected a present transport fingerprint to be checked");
} catch (error) {
expect(error).toBeInstanceOf(MeasurementError);
expect((error as MeasurementError).code).toBe("MANIFEST_INVALID");
expect((error as MeasurementError).message).toContain("fingerprints.transportSha256");
}
});
it("rejects a rewritten prompt, wire, or rubric fingerprint", () => {
const fingerprints = createCandidate(root());
rewriteCandidate(fingerprints.runDir, (manifest) => {
manifest.fingerprints.promptSha256 = HASH("1");
});
expectCode(() => exportValid(fingerprints), "INCOMPATIBLE_IDENTITIES");
const wire = createCandidate(root());
rewriteCandidate(wire.runDir, (manifest) => {
manifest.fingerprints.wireSha256 = HASH("4");
});
expectCode(() => exportValid(wire), "INCOMPATIBLE_IDENTITIES");
const rubric = createCandidate(root());
rewriteCandidate(rubric.runDir, (manifest) => {
manifest.fingerprints.rubricSha256 = HASH("5");
});
expectCode(() => exportValid(rubric), "INCOMPATIBLE_IDENTITIES");
});
it("never turns an explicit rejected review into baseline approval", () => {
const base = createCandidate(root());
const receipt = JSON.parse(readFileSync(base.review, "utf8")) as { disposition: string };
receipt.disposition = "rejected";
writeJson(base.review, receipt);
const verified = verifyMeasurementDirectory(exportValid(base));
expect(verified.review.disposition).toBe("rejected");
expectCode(() => boundBaselineFromMeasurement(verified), "REVIEW_NOT_APPROVED");
});
});