Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(workbench): score undecided repetitions and compare fractional trials correctly

超超超超超级喜欢你的达妮娅7eaef4dd412d
diff --git a/workbench/l0/measurements.wb.ts b/workbench/l0/measurements.wb.ts
index 13ee7c25a..ddbca7d55 100644
--- a/workbench/l0/measurements.wb.ts
+++ b/workbench/l0/measurements.wb.ts
@@ -32,7 +32,7 @@ import {
sha256Canonical,
sourceIdentityFromManifests,
} from "../lib/provenance";
-import { wilson95 } from "../lib/stats";
+import { newcombeDelta, wilson95 } from "../lib/stats";
import { runMeasurementCli } from "../measurement-cli";
import { getScenario } from "../scenarios/registry";
@@ -256,6 +256,65 @@ afterEach(() => {
});
describe("versioned measurement export and verification", () => {
+ it("keeps fractional weighted scores exact through the comparison trials", () => {
+ const checks = (heavyPasses: boolean) => [
+ { id: "beh.light", axis: "behaviour" as const, weight: 1, ok: true, indeterminate: false },
+ {
+ id: "beh.heavy",
+ axis: "behaviour" as const,
+ weight: 3,
+ ok: heavyPasses,
+ indeterminate: false,
+ },
+ { id: "beh.unread", axis: "behaviour" as const, weight: 10, ok: false, indeterminate: true },
+ { id: "dsl.complete", axis: "dsl" as const, weight: 2, ok: true, indeterminate: false },
+ ];
+ const counts = recomputeMeasurementCounts({
+ schema: 1,
+ scenarioId: "fractional-probe",
+ complete: true,
+ repetitions: [
+ { rep: 0, checks: checks(false) },
+ { rep: 1, checks: checks(true) },
+ ],
+ });
+ expect(counts.axisScores.behaviour).toBe(0.625);
+ const trials = weightedAxisTrials(counts, "behaviour");
+ expect(trials).toEqual({ k: 1.25, n: 2 });
+ // The comparison must read the real 0.625, not the rounded 0.5.
+ const delta = newcombeDelta({ k: 1, n: 2 }, trials);
+ expect(delta.point).toBeCloseTo(0.625 - 0.5, 12);
+ });
+
+ it("scores a wholly indeterminate repetition-axis as undecided, not perfect", () => {
+ const counts = recomputeMeasurementCounts({
+ schema: 1,
+ scenarioId: "undecided-probe",
+ complete: true,
+ repetitions: [
+ {
+ rep: 0,
+ checks: [
+ { id: "beh.a", axis: "behaviour", weight: 1, ok: false, indeterminate: true },
+ { id: "dsl.a", axis: "dsl", weight: 1, ok: false, indeterminate: true },
+ ],
+ },
+ {
+ rep: 1,
+ checks: [
+ { id: "beh.a", axis: "behaviour", weight: 1, ok: true, indeterminate: false },
+ { id: "dsl.a", axis: "dsl", weight: 1, ok: false, indeterminate: true },
+ ],
+ },
+ ],
+ });
+ // rep 0 decided nothing: it must not contribute a placeholder perfect score.
+ expect(counts.axisScores.behaviour).toBe(1);
+ expect(counts.measuredRepetitions.behaviour).toBe(1);
+ // dsl decided nothing in either repetition: score 0, never a nominal 100%.
+ expect(counts.axisScores.dsl).toBe(0);
+ expect(counts.measuredRepetitions.dsl).toBe(0);
+ });
it("recomputes the existing weighted axis mean while excluding indeterminate weight", () => {
const checks = (heavyPasses: boolean) => [
{
@@ -306,7 +365,9 @@ describe("versioned measurement export and verification", () => {
});
expect(counts.axisScores.behaviour).toBe(0.625);
expect(counts.axisScores.dsl).toBe(1);
- expect(weightedAxisTrials(counts, "behaviour")).toEqual({ k: 1, n: 2 });
+ // The trial stays fractional: rounding 0.625 x 2 into whole successes read a
+ // 0.625 mean as 0.5 in the Newcombe comparison.
+ expect(weightedAxisTrials(counts, "behaviour")).toEqual({ k: 1.25, n: 2 });
expect(weightedAxisTrials(counts, "dsl")).toEqual({ k: 2, n: 2 });
});
diff --git a/workbench/l0/stats-report.wb.ts b/workbench/l0/stats-report.wb.ts
index 5e26af0e8..da9f00e95 100644
--- a/workbench/l0/stats-report.wb.ts
+++ b/workbench/l0/stats-report.wb.ts
@@ -5,9 +5,81 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { ENV_KEYS } from "../lib/env";
-import { buildReport, renderMarkdown, writeReportFile } from "../lib/report";
+import { recomputeMeasurementCounts } from "../lib/measurement";
+import { buildReport, renderMarkdown, summarizeScenario, writeReportFile } from "../lib/report";
import { minDetectableEffect, newcombeDelta, wilson95 } from "../lib/stats";
+describe("axis score consistency (report vs manifest)", () => {
+ it("summarizes wholly indeterminate repetitions out of the axis mean", () => {
+ const scoredRun = (check: { ok: boolean; indeterminate: boolean }) =>
+ ({
+ behaviour: {
+ score: check.indeterminate ? 1 : check.ok ? 1 : 0,
+ decidedWeight: check.indeterminate ? 0 : 1,
+ undecidedWeight: check.indeterminate ? 1 : 0,
+ measured: !check.indeterminate,
+ results: [
+ {
+ id: "beh.a",
+ weight: 1,
+ ok: check.ok,
+ indeterminate: check.indeterminate,
+ expected: false,
+ },
+ ],
+ },
+ dsl: {
+ score: 1,
+ decidedWeight: 1,
+ undecidedWeight: 0,
+ measured: true,
+ results: [{ id: "dsl.a", weight: 1, ok: true, indeterminate: false, expected: false }],
+ },
+ gateScore: 1,
+ passed: true,
+ ms: 1,
+ failureClass: "NONE",
+ }) as never;
+ const summary = summarizeScenario({
+ scenarioId: "probe",
+ title: "probe",
+ tags: [],
+ gate: 0,
+ results: [
+ // runChecks reports score 1 for the wholly indeterminate repetition —
+ // the placeholder that must not leak into the report mean.
+ { scored: scoredRun({ ok: false, indeterminate: true }) },
+ { scored: scoredRun({ ok: true, indeterminate: false }) },
+ ],
+ });
+ const counts = recomputeMeasurementCounts({
+ schema: 1,
+ scenarioId: "probe",
+ complete: true,
+ repetitions: [
+ {
+ rep: 0,
+ checks: [
+ { id: "beh.a", axis: "behaviour", weight: 1, ok: false, indeterminate: true },
+ { id: "dsl.a", axis: "dsl", weight: 1, ok: true, indeterminate: false },
+ ],
+ },
+ {
+ rep: 1,
+ checks: [
+ { id: "beh.a", axis: "behaviour", weight: 1, ok: true, indeterminate: false },
+ { id: "dsl.a", axis: "dsl", weight: 1, ok: true, indeterminate: false },
+ ],
+ },
+ ],
+ });
+ // Report and manifest must agree, or finalize/export fails its own
+ // AXIS_SCORE_MISMATCH check on freshly generated evidence.
+ expect(summary.behaviour.rate).toBe(counts.axisScores.behaviour);
+ expect(summary.dsl.rate).toBe(counts.axisScores.dsl);
+ });
+});
+
describe("wilson95", () => {
it("stays inside [0,1] at the edges, where Wald does not", () => {
const none = wilson95(0, 3);
diff --git a/workbench/lib/measurement.ts b/workbench/lib/measurement.ts
index 6643a6cc8..867d2ca92 100644
--- a/workbench/lib/measurement.ts
+++ b/workbench/lib/measurement.ts
@@ -32,7 +32,7 @@ import {
import type { WorkbenchReport } from "./report";
import { type RepetitionResult, runRepetition } from "./runner";
import type { Scenario } from "./scenario";
-import { type ScoredRun, scoreRun } from "./score";
+import { type ScoredRun, scoreRun, undecidedAwareAxisMean } from "./score";
import { measurementTransportSha256, type TransportIdentity } from "./transport";
export const MEASUREMENTS_DIR = "workbench/measurements";
@@ -40,6 +40,7 @@ export const CANDIDATE_FILE = "measurement-candidate.json";
export const MANIFEST_FILE = "measurement.json";
export type MeasurementErrorCode =
+ | "UNMEASURED_AXIS"
| "ARGUMENT_ERROR"
| "ARTIFACT_HASH_MISMATCH"
| "ARTIFACT_NOT_REGULAR"
@@ -158,8 +159,13 @@ export interface AxisCounts {
export interface MeasurementCounts {
repetitions: number;
axes: { behaviour: AxisCounts; dsl: AxisCounts };
- /** Mean of each repetition's weighted axis score, matching report.ts. */
+ /** Mean of each repetition's weighted axis score, matching report.ts. Wholly
+ * indeterminate repetitions are excluded from the mean. */
axisScores: { behaviour: number; dsl: number };
+ /** Repetitions that decided at least one weighted check on the axis. The
+ * comparison trials use these, never the raw repetition count: an unknown
+ * repetition is not an implicit success. */
+ measuredRepetitions: { behaviour: number; dsl: number };
checks: CheckCounts[];
}
@@ -535,37 +541,37 @@ export function recomputeMeasurementCounts(recorded: RecordedChecks): Measuremen
}),
{ passed: 0, decided: 0, indeterminate: 0, total: 0 },
);
- const scoreFor = (
- repetition: RecordedChecks["repetitions"][number],
- name: "behaviour" | "dsl",
- ): number => {
- const checks = repetition.checks.filter((check) => check.axis === name);
- const decidedWeight = checks.reduce(
- (sum, check) => sum + (check.indeterminate ? 0 : check.weight),
- 0,
+ const axisResults = (name: "behaviour" | "dsl") =>
+ recorded.repetitions.map((repetition) =>
+ repetition.checks.filter((check) => check.axis === name),
);
- if (decidedWeight === 0) return 1;
- const passedWeight = checks.reduce((sum, check) => sum + (check.ok ? check.weight : 0), 0);
- return passedWeight / decidedWeight;
- };
- const meanScore = (name: "behaviour" | "dsl"): number =>
- recorded.repetitions.reduce((sum, repetition) => sum + scoreFor(repetition, name), 0) /
- recorded.repetitions.length;
+ // The measurement rule lives in one place (`undecidedAwareAxisMean`): a wholly
+ // indeterminate repetition-axis decides nothing and leaves the mean, and an
+ // axis with nothing decidable scores 0 — never `runChecks`'s placeholder 1.
+ const behaviourScore = undecidedAwareAxisMean(axisResults("behaviour"));
+ const dslScore = undecidedAwareAxisMean(axisResults("dsl"));
return {
repetitions: recorded.repetitions.length,
axes: { behaviour: axis("behaviour"), dsl: axis("dsl") },
- axisScores: { behaviour: meanScore("behaviour"), dsl: meanScore("dsl") },
+ axisScores: { behaviour: behaviourScore.score, dsl: dslScore.score },
+ measuredRepetitions: { behaviour: behaviourScore.measured, dsl: dslScore.measured },
checks,
};
}
-/** One independent trial per repetition, weighted the same way as `axisScores`. */
+/** One trial per measured repetition, weighted the same way as `axisScores`. The
+ * trial count stays FRACTIONAL on purpose: weighted checks commonly produce means
+ * like 0.625, and rounding k into whole successes before the Newcombe comparison
+ * misstates the point estimate (0.625 over two repetitions would read as 0.5).
+ * The Wilson arithmetic below holds for non-integer k, and only repetitions that
+ * decided weighted checks count as trials — an unknown repetition is not an
+ * implicit success. */
export function weightedAxisTrials(
- results: Pick<MeasurementCounts, "repetitions" | "axisScores">,
+ results: Pick<MeasurementCounts, "repetitions" | "axisScores" | "measuredRepetitions">,
axis: "behaviour" | "dsl",
): { k: number; n: number } {
- const n = results.repetitions;
- return { k: Math.round(results.axisScores[axis] * n), n };
+ const n = results.measuredRepetitions?.[axis] ?? results.repetitions;
+ return { k: results.axisScores[axis] * n, n };
}
export function boundMeasurementFingerprints(
diff --git a/workbench/lib/report.ts b/workbench/lib/report.ts
index 263fd53f7..a3cc4026e 100644
--- a/workbench/lib/report.ts
+++ b/workbench/lib/report.ts
@@ -18,6 +18,7 @@ import { mkdirSync, writeFileSync } from "node:fs";
import { dirname } from "node:path";
import { containsSecret } from "./env";
import type { CheckResult, ScoredRun } from "./score";
+import { undecidedAwareAxisMean } from "./score";
import {
formatInterval,
formatPercent,
@@ -197,8 +198,11 @@ export function summarizeScenario(options: {
const mean = (values: number[]) =>
values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length;
- const behaviourMean = mean(results.map((r) => r.scored.behaviour.score));
- const dslMean = mean(results.map((r) => r.scored.dsl.score));
+ // Axis rates use the measurement rule (undecidedAwareAxisMean): a wholly
+ // indeterminate repetition must not contribute `runChecks`'s placeholder 1,
+ // or the report disagrees with the manifest and export fails its own check.
+ const behaviourAxis = undecidedAwareAxisMean(results.map((r) => r.scored.behaviour.results));
+ const dslAxis = undecidedAwareAxisMean(results.map((r) => r.scored.dsl.results));
const passes = results.filter((r) => r.scored.passed).length;
return {
@@ -209,8 +213,14 @@ export function summarizeScenario(options: {
reps: n,
// Axis scores are weighted means, not counts; Wilson is carried on the
// pass counts and the rate field holds the weighted mean.
- behaviour: { ...wilson95(Math.round(behaviourMean * n), n), rate: behaviourMean },
- dsl: { ...wilson95(Math.round(dslMean * n), n), rate: dslMean },
+ behaviour: {
+ ...wilson95(Math.round(behaviourAxis.score * behaviourAxis.measured), behaviourAxis.measured),
+ rate: behaviourAxis.score,
+ },
+ dsl: {
+ ...wilson95(Math.round(dslAxis.score * dslAxis.measured), dslAxis.measured),
+ rate: dslAxis.score,
+ },
gateScoreMean: mean(results.map((r) => r.scored.gateScore)),
passRate: wilson95(passes, n),
failureClasses,
diff --git a/workbench/lib/score.ts b/workbench/lib/score.ts
index f5d99be5b..45700877f 100644
--- a/workbench/lib/score.ts
+++ b/workbench/lib/score.ts
@@ -103,6 +103,31 @@ export const STRUCTURAL_CHECKS: Check[] = [
},
];
+/** Weighted mean of each repetition's axis score with the measurement rule: a
+ * wholly indeterminate repetition decides nothing and leaves the mean, and an
+ * axis with nothing decidable scores 0. Shared by the manifest recompute and
+ * the report summary so the two can never disagree — `runChecks`'s `score: 1`
+ * placeholder for zero decided weight is a placeholder, never a measurement. */
+export function undecidedAwareAxisMean(
+ perRep: ReadonlyArray<ReadonlyArray<Pick<CheckResult, "ok" | "indeterminate" | "weight">>>,
+): { score: number; measured: number } {
+ const scores: number[] = [];
+ for (const results of perRep) {
+ const decidedWeight = results.reduce(
+ (sum, check) => sum + (check.indeterminate ? 0 : check.weight),
+ 0,
+ );
+ if (decidedWeight === 0) continue;
+ const passedWeight = results.reduce((sum, check) => sum + (check.ok ? check.weight : 0), 0);
+ scores.push(passedWeight / decidedWeight);
+ }
+ if (scores.length === 0) return { score: 0, measured: 0 };
+ return {
+ score: scores.reduce((sum, score) => sum + score, 0) / scores.length,
+ measured: scores.length,
+ };
+}
+
export function runChecks(
checks: Check[],
context: EvalContext,
diff --git a/workbench/measurement-cli.ts b/workbench/measurement-cli.ts
index 010e3e883..21c210c2b 100644
--- a/workbench/measurement-cli.ts
+++ b/workbench/measurement-cli.ts
@@ -2,13 +2,32 @@ import {
assertComparable,
boundBaselineFromMeasurement,
exportMeasurement,
+ type MeasurementCounts,
MeasurementError,
replayMeasurement,
verifyMeasurementId,
weightedAxisTrials,
writeBoundBaseline,
} from "./lib/measurement";
-import { newcombeDelta } from "./lib/stats";
+import { type Delta, newcombeDelta } from "./lib/stats";
+
+/** A wholly unmeasured axis has no observations on at least one side: a zero-trial
+ * Wilson interval would present "unknown" as a quantitative delta. */
+function compareTrials(
+ left: MeasurementCounts,
+ right: MeasurementCounts,
+ axis: "behaviour" | "dsl",
+): Delta {
+ const leftTrials = weightedAxisTrials(left, axis);
+ const rightTrials = weightedAxisTrials(right, axis);
+ if (leftTrials.n === 0 || rightTrials.n === 0) {
+ throw new MeasurementError(
+ "UNMEASURED_AXIS",
+ `${axis}: ${leftTrials.n === 0 ? "left" : "right"} side has no measured repetitions`,
+ );
+ }
+ return newcombeDelta(leftTrials, rightTrials);
+}
type Command = "export" | "verify" | "replay" | "baseline" | "compare" | "help";
@@ -146,14 +165,8 @@ export async function runMeasurementCli(argv: string[]): Promise<number> {
comparable: true,
leftScores: left.manifest.results.axisScores,
rightScores: right.manifest.results.axisScores,
- behaviour: newcombeDelta(
- weightedAxisTrials(left.manifest.results, "behaviour"),
- weightedAxisTrials(right.manifest.results, "behaviour"),
- ),
- dsl: newcombeDelta(
- weightedAxisTrials(left.manifest.results, "dsl"),
- weightedAxisTrials(right.manifest.results, "dsl"),
- ),
+ behaviour: compareTrials(left.manifest.results, right.manifest.results, "behaviour"),
+ dsl: compareTrials(left.manifest.results, right.manifest.results, "dsl"),
});
return 0;
}