Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(workbench): score undecided repetitions and compare fractional trials correctly
超超超超超级喜欢你的达妮娅7eaef4dd412d
diff --git a/workbench/l0/measurements.wb.ts b/workbench/l0/measurements.wb.tsindex 13ee7c25a..ddbca7d55 100644--- a/workbench/l0/measurements.wb.ts+++ b/workbench/l0/measurements.wb.ts@@ -32,7 +32,7 @@ import {sha256Canonical,sourceIdentityFromManifests,} from "../lib/provenance";-import { wilson95 } from "../lib/stats";+import { newcombeDelta, wilson95 } from "../lib/stats";import { runMeasurementCli } from "../measurement-cli";import { getScenario } from "../scenarios/registry";@@ -256,6 +256,65 @@ afterEach(() => {});describe("versioned measurement export and verification", () => {+ it("keeps fractional weighted scores exact through the comparison trials", () => {+ const checks = (heavyPasses: boolean) => [+ { id: "beh.light", axis: "behaviour" as const, weight: 1, ok: true, indeterminate: false },+ {+ id: "beh.heavy",+ axis: "behaviour" as const,+ weight: 3,+ ok: heavyPasses,+ indeterminate: false,+ },+ { id: "beh.unread", axis: "behaviour" as const, weight: 10, ok: false, indeterminate: true },+ { id: "dsl.complete", axis: "dsl" as const, weight: 2, ok: true, indeterminate: false },+ ];+ const counts = recomputeMeasurementCounts({+ schema: 1,+ scenarioId: "fractional-probe",+ complete: true,+ repetitions: [+ { rep: 0, checks: checks(false) },+ { rep: 1, checks: checks(true) },+ ],+ });+ expect(counts.axisScores.behaviour).toBe(0.625);+ const trials = weightedAxisTrials(counts, "behaviour");+ expect(trials).toEqual({ k: 1.25, n: 2 });+ // The comparison must read the real 0.625, not the rounded 0.5.+ const delta = newcombeDelta({ k: 1, n: 2 }, trials);+ expect(delta.point).toBeCloseTo(0.625 - 0.5, 12);+ });++ it("scores a wholly indeterminate repetition-axis as undecided, not perfect", () => {+ const counts = recomputeMeasurementCounts({+ schema: 1,+ scenarioId: "undecided-probe",+ complete: true,+ repetitions: [+ {+ rep: 0,+ checks: [+ { id: "beh.a", axis: "behaviour", weight: 1, ok: false, indeterminate: true },+ { id: "dsl.a", axis: "dsl", weight: 1, ok: false, indeterminate: true },+ ],+ },+ {+ rep: 1,+ checks: [+ { id: "beh.a", axis: "behaviour", weight: 1, ok: true, indeterminate: false },+ { id: "dsl.a", axis: "dsl", weight: 1, ok: false, indeterminate: true },+ ],+ },+ ],+ });+ // rep 0 decided nothing: it must not contribute a placeholder perfect score.+ expect(counts.axisScores.behaviour).toBe(1);+ expect(counts.measuredRepetitions.behaviour).toBe(1);+ // dsl decided nothing in either repetition: score 0, never a nominal 100%.+ expect(counts.axisScores.dsl).toBe(0);+ expect(counts.measuredRepetitions.dsl).toBe(0);+ });it("recomputes the existing weighted axis mean while excluding indeterminate weight", () => {const checks = (heavyPasses: boolean) => [{@@ -306,7 +365,9 @@ describe("versioned measurement export and verification", () => {});expect(counts.axisScores.behaviour).toBe(0.625);expect(counts.axisScores.dsl).toBe(1);- expect(weightedAxisTrials(counts, "behaviour")).toEqual({ k: 1, n: 2 });+ // The trial stays fractional: rounding 0.625 x 2 into whole successes read a+ // 0.625 mean as 0.5 in the Newcombe comparison.+ expect(weightedAxisTrials(counts, "behaviour")).toEqual({ k: 1.25, n: 2 });expect(weightedAxisTrials(counts, "dsl")).toEqual({ k: 2, n: 2 });});diff --git a/workbench/l0/stats-report.wb.ts b/workbench/l0/stats-report.wb.tsindex 5e26af0e8..da9f00e95 100644--- a/workbench/l0/stats-report.wb.ts+++ b/workbench/l0/stats-report.wb.ts@@ -5,9 +5,81 @@ import { tmpdir } from "node:os";import { join } from "node:path";import { describe, expect, it } from "vitest";import { ENV_KEYS } from "../lib/env";-import { buildReport, renderMarkdown, writeReportFile } from "../lib/report";+import { recomputeMeasurementCounts } from "../lib/measurement";+import { buildReport, renderMarkdown, summarizeScenario, writeReportFile } from "../lib/report";import { minDetectableEffect, newcombeDelta, wilson95 } from "../lib/stats";+describe("axis score consistency (report vs manifest)", () => {+ it("summarizes wholly indeterminate repetitions out of the axis mean", () => {+ const scoredRun = (check: { ok: boolean; indeterminate: boolean }) =>+ ({+ behaviour: {+ score: check.indeterminate ? 1 : check.ok ? 1 : 0,+ decidedWeight: check.indeterminate ? 0 : 1,+ undecidedWeight: check.indeterminate ? 1 : 0,+ measured: !check.indeterminate,+ results: [+ {+ id: "beh.a",+ weight: 1,+ ok: check.ok,+ indeterminate: check.indeterminate,+ expected: false,+ },+ ],+ },+ dsl: {+ score: 1,+ decidedWeight: 1,+ undecidedWeight: 0,+ measured: true,+ results: [{ id: "dsl.a", weight: 1, ok: true, indeterminate: false, expected: false }],+ },+ gateScore: 1,+ passed: true,+ ms: 1,+ failureClass: "NONE",+ }) as never;+ const summary = summarizeScenario({+ scenarioId: "probe",+ title: "probe",+ tags: [],+ gate: 0,+ results: [+ // runChecks reports score 1 for the wholly indeterminate repetition —+ // the placeholder that must not leak into the report mean.+ { scored: scoredRun({ ok: false, indeterminate: true }) },+ { scored: scoredRun({ ok: true, indeterminate: false }) },+ ],+ });+ const counts = recomputeMeasurementCounts({+ schema: 1,+ scenarioId: "probe",+ complete: true,+ repetitions: [+ {+ rep: 0,+ checks: [+ { id: "beh.a", axis: "behaviour", weight: 1, ok: false, indeterminate: true },+ { id: "dsl.a", axis: "dsl", weight: 1, ok: true, indeterminate: false },+ ],+ },+ {+ rep: 1,+ checks: [+ { id: "beh.a", axis: "behaviour", weight: 1, ok: true, indeterminate: false },+ { id: "dsl.a", axis: "dsl", weight: 1, ok: true, indeterminate: false },+ ],+ },+ ],+ });+ // Report and manifest must agree, or finalize/export fails its own+ // AXIS_SCORE_MISMATCH check on freshly generated evidence.+ expect(summary.behaviour.rate).toBe(counts.axisScores.behaviour);+ expect(summary.dsl.rate).toBe(counts.axisScores.dsl);+ });+});+describe("wilson95", () => {it("stays inside [0,1] at the edges, where Wald does not", () => {const none = wilson95(0, 3);diff --git a/workbench/lib/measurement.ts b/workbench/lib/measurement.tsindex 6643a6cc8..867d2ca92 100644--- a/workbench/lib/measurement.ts+++ b/workbench/lib/measurement.ts@@ -32,7 +32,7 @@ import {import type { WorkbenchReport } from "./report";import { type RepetitionResult, runRepetition } from "./runner";import type { Scenario } from "./scenario";-import { type ScoredRun, scoreRun } from "./score";+import { type ScoredRun, scoreRun, undecidedAwareAxisMean } from "./score";import { measurementTransportSha256, type TransportIdentity } from "./transport";export const MEASUREMENTS_DIR = "workbench/measurements";@@ -40,6 +40,7 @@ export const CANDIDATE_FILE = "measurement-candidate.json";export const MANIFEST_FILE = "measurement.json";export type MeasurementErrorCode =+ | "UNMEASURED_AXIS"| "ARGUMENT_ERROR"| "ARTIFACT_HASH_MISMATCH"| "ARTIFACT_NOT_REGULAR"@@ -158,8 +159,13 @@ export interface AxisCounts {export interface MeasurementCounts {repetitions: number;axes: { behaviour: AxisCounts; dsl: AxisCounts };- /** Mean of each repetition's weighted axis score, matching report.ts. */+ /** Mean of each repetition's weighted axis score, matching report.ts. Wholly+ * indeterminate repetitions are excluded from the mean. */axisScores: { behaviour: number; dsl: number };+ /** Repetitions that decided at least one weighted check on the axis. The+ * comparison trials use these, never the raw repetition count: an unknown+ * repetition is not an implicit success. */+ measuredRepetitions: { behaviour: number; dsl: number };checks: CheckCounts[];}@@ -535,37 +541,37 @@ export function recomputeMeasurementCounts(recorded: RecordedChecks): Measuremen}),{ passed: 0, decided: 0, indeterminate: 0, total: 0 },);- const scoreFor = (- repetition: RecordedChecks["repetitions"][number],- name: "behaviour" | "dsl",- ): number => {- const checks = repetition.checks.filter((check) => check.axis === name);- const decidedWeight = checks.reduce(- (sum, check) => sum + (check.indeterminate ? 0 : check.weight),- 0,+ const axisResults = (name: "behaviour" | "dsl") =>+ recorded.repetitions.map((repetition) =>+ repetition.checks.filter((check) => check.axis === name),);- if (decidedWeight === 0) return 1;- const passedWeight = checks.reduce((sum, check) => sum + (check.ok ? check.weight : 0), 0);- return passedWeight / decidedWeight;- };- const meanScore = (name: "behaviour" | "dsl"): number =>- recorded.repetitions.reduce((sum, repetition) => sum + scoreFor(repetition, name), 0) /- recorded.repetitions.length;+ // The measurement rule lives in one place (`undecidedAwareAxisMean`): a wholly+ // indeterminate repetition-axis decides nothing and leaves the mean, and an+ // axis with nothing decidable scores 0 — never `runChecks`'s placeholder 1.+ const behaviourScore = undecidedAwareAxisMean(axisResults("behaviour"));+ const dslScore = undecidedAwareAxisMean(axisResults("dsl"));return {repetitions: recorded.repetitions.length,axes: { behaviour: axis("behaviour"), dsl: axis("dsl") },- axisScores: { behaviour: meanScore("behaviour"), dsl: meanScore("dsl") },+ axisScores: { behaviour: behaviourScore.score, dsl: dslScore.score },+ measuredRepetitions: { behaviour: behaviourScore.measured, dsl: dslScore.measured },checks,};}-/** One independent trial per repetition, weighted the same way as `axisScores`. */+/** One trial per measured repetition, weighted the same way as `axisScores`. The+ * trial count stays FRACTIONAL on purpose: weighted checks commonly produce means+ * like 0.625, and rounding k into whole successes before the Newcombe comparison+ * misstates the point estimate (0.625 over two repetitions would read as 0.5).+ * The Wilson arithmetic below holds for non-integer k, and only repetitions that+ * decided weighted checks count as trials — an unknown repetition is not an+ * implicit success. */export function weightedAxisTrials(- results: Pick<MeasurementCounts, "repetitions" | "axisScores">,+ results: Pick<MeasurementCounts, "repetitions" | "axisScores" | "measuredRepetitions">,axis: "behaviour" | "dsl",): { k: number; n: number } {- const n = results.repetitions;- return { k: Math.round(results.axisScores[axis] * n), n };+ const n = results.measuredRepetitions?.[axis] ?? results.repetitions;+ return { k: results.axisScores[axis] * n, n };}export function boundMeasurementFingerprints(diff --git a/workbench/lib/report.ts b/workbench/lib/report.tsindex 263fd53f7..a3cc4026e 100644--- a/workbench/lib/report.ts+++ b/workbench/lib/report.ts@@ -18,6 +18,7 @@ import { mkdirSync, writeFileSync } from "node:fs";import { dirname } from "node:path";import { containsSecret } from "./env";import type { CheckResult, ScoredRun } from "./score";+import { undecidedAwareAxisMean } from "./score";import {formatInterval,formatPercent,@@ -197,8 +198,11 @@ export function summarizeScenario(options: {const mean = (values: number[]) =>values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length;- const behaviourMean = mean(results.map((r) => r.scored.behaviour.score));- const dslMean = mean(results.map((r) => r.scored.dsl.score));+ // Axis rates use the measurement rule (undecidedAwareAxisMean): a wholly+ // indeterminate repetition must not contribute `runChecks`'s placeholder 1,+ // or the report disagrees with the manifest and export fails its own check.+ const behaviourAxis = undecidedAwareAxisMean(results.map((r) => r.scored.behaviour.results));+ const dslAxis = undecidedAwareAxisMean(results.map((r) => r.scored.dsl.results));const passes = results.filter((r) => r.scored.passed).length;return {@@ -209,8 +213,14 @@ export function summarizeScenario(options: {reps: n,// Axis scores are weighted means, not counts; Wilson is carried on the// pass counts and the rate field holds the weighted mean.- behaviour: { ...wilson95(Math.round(behaviourMean * n), n), rate: behaviourMean },- dsl: { ...wilson95(Math.round(dslMean * n), n), rate: dslMean },+ behaviour: {+ ...wilson95(Math.round(behaviourAxis.score * behaviourAxis.measured), behaviourAxis.measured),+ rate: behaviourAxis.score,+ },+ dsl: {+ ...wilson95(Math.round(dslAxis.score * dslAxis.measured), dslAxis.measured),+ rate: dslAxis.score,+ },gateScoreMean: mean(results.map((r) => r.scored.gateScore)),passRate: wilson95(passes, n),failureClasses,diff --git a/workbench/lib/score.ts b/workbench/lib/score.tsindex f5d99be5b..45700877f 100644--- a/workbench/lib/score.ts+++ b/workbench/lib/score.ts@@ -103,6 +103,31 @@ export const STRUCTURAL_CHECKS: Check[] = [},];+/** Weighted mean of each repetition's axis score with the measurement rule: a+ * wholly indeterminate repetition decides nothing and leaves the mean, and an+ * axis with nothing decidable scores 0. Shared by the manifest recompute and+ * the report summary so the two can never disagree — `runChecks`'s `score: 1`+ * placeholder for zero decided weight is a placeholder, never a measurement. */+export function undecidedAwareAxisMean(+ perRep: ReadonlyArray<ReadonlyArray<Pick<CheckResult, "ok" | "indeterminate" | "weight">>>,+): { score: number; measured: number } {+ const scores: number[] = [];+ for (const results of perRep) {+ const decidedWeight = results.reduce(+ (sum, check) => sum + (check.indeterminate ? 0 : check.weight),+ 0,+ );+ if (decidedWeight === 0) continue;+ const passedWeight = results.reduce((sum, check) => sum + (check.ok ? check.weight : 0), 0);+ scores.push(passedWeight / decidedWeight);+ }+ if (scores.length === 0) return { score: 0, measured: 0 };+ return {+ score: scores.reduce((sum, score) => sum + score, 0) / scores.length,+ measured: scores.length,+ };+}+export function runChecks(checks: Check[],context: EvalContext,diff --git a/workbench/measurement-cli.ts b/workbench/measurement-cli.tsindex 010e3e883..21c210c2b 100644--- a/workbench/measurement-cli.ts+++ b/workbench/measurement-cli.ts@@ -2,13 +2,32 @@ import {assertComparable,boundBaselineFromMeasurement,exportMeasurement,+ type MeasurementCounts,MeasurementError,replayMeasurement,verifyMeasurementId,weightedAxisTrials,writeBoundBaseline,} from "./lib/measurement";-import { newcombeDelta } from "./lib/stats";+import { type Delta, newcombeDelta } from "./lib/stats";++/** A wholly unmeasured axis has no observations on at least one side: a zero-trial+ * Wilson interval would present "unknown" as a quantitative delta. */+function compareTrials(+ left: MeasurementCounts,+ right: MeasurementCounts,+ axis: "behaviour" | "dsl",+): Delta {+ const leftTrials = weightedAxisTrials(left, axis);+ const rightTrials = weightedAxisTrials(right, axis);+ if (leftTrials.n === 0 || rightTrials.n === 0) {+ throw new MeasurementError(+ "UNMEASURED_AXIS",+ `${axis}: ${leftTrials.n === 0 ? "left" : "right"} side has no measured repetitions`,+ );+ }+ return newcombeDelta(leftTrials, rightTrials);+}type Command = "export" | "verify" | "replay" | "baseline" | "compare" | "help";@@ -146,14 +165,8 @@ export async function runMeasurementCli(argv: string[]): Promise<number> {comparable: true,leftScores: left.manifest.results.axisScores,rightScores: right.manifest.results.axisScores,- behaviour: newcombeDelta(- weightedAxisTrials(left.manifest.results, "behaviour"),- weightedAxisTrials(right.manifest.results, "behaviour"),- ),- dsl: newcombeDelta(- weightedAxisTrials(left.manifest.results, "dsl"),- weightedAxisTrials(right.manifest.results, "dsl"),- ),+ behaviour: compareTrials(left.manifest.results, right.manifest.results, "behaviour"),+ dsl: compareTrials(left.manifest.results, right.manifest.results, "dsl"),});return 0;}