Open source CapCut alternative.
TypeScript
test(site): the perf eval counts frames missed and streams its fixtures
David Le93bae3273fd8
diff --git a/docs/guides/cut/README.md b/docs/guides/cut/README.mdindex 56fbbb5d..d7b9380f 100644--- a/docs/guides/cut/README.md+++ b/docs/guides/cut/README.md@@ -78,7 +78,7 @@ The stage sits under a camera. The picture opens fitted to its pane, and that fiThe playhead lives apart from the document. It moves sixty times a second and the document does not, so a moved playhead wakes the surfaces that draw it and nothing else — and a paused editor with nothing changing schedules no frames at all.-Whether any of this is true of a given build is measured rather than assumed: `npm run eval:cut-perf` drives a real editor in a real browser against fixture montages and reports how long the picture takes to answer a moved playhead, whether any frame crossing a cut was the wrong one, and whether a fast-cut montage replayed for minutes still holds the picture to the clock the way its first pass did. Its committed report is the baseline a change is diffed against.+Whether any of this is true of a given build is measured rather than assumed: `npm run eval:cut-perf` drives a real editor in a real browser against fixture montages and reports whether the picture answers a moved playhead on the next frame painted (counted in frames, so a sixty-hertz and a hundred-and-twenty-hertz display read the same), whether any frame crossing a cut was the wrong one, and whether a fast-cut montage replayed for minutes still holds the picture to the clock the way its first pass did. Its committed report is the baseline a change is diffed against.The machine it measures on is part of the measurement. A developer desktop has decode slots to spare and never falls behind, so the preview's caps never bind there and a change that would ruin a laptop reads as a clean sheet. `--machine` puts the page on a smaller one: it takes away hardware decode slots, so decoders past the limit deliver their frames late the way software decoding does, it slows the main thread to a fraction of this one, and it paces the media over a link the way a cloud project reads it. That last one is its own axis: a reader short of bytes and a reader short of decode both show a picture drifting behind the sound, each takes its own fix, and `--net` and `--rtt` open the link up on a failing case to say which it was. The fixtures cover the shape that binding actually happens in — one long clip on the timeline, read the way a cloud project reads it, played cold and played again after the editor has been closed and reopened. `npm run eval:cut-perf-lowend` is the profile the reports come from. What the gates ask there is what a viewer would ask — did the frames arrive, did the picture hitch a quarter second behind the sound, and was the last pass as good as the first — and each machine has its own budget for those, because a number that is honest on a desktop is fantasy on a fifteen-watt laptop.diff --git a/evals/cut-perf.latest-report.json b/evals/cut-perf.latest-report.jsonindex cfe5b7b4..00f39443 100644--- a/evals/cut-perf.latest-report.json+++ b/evals/cut-perf.latest-report.json@@ -15,16 +15,17 @@"decayFloor": 0.02},"gate": {- "scrubP50Ms": 10,- "scrubP95Ms": 16.7,- "jumpP50Ms": 20,- "jumpP95Ms": 32,+ "scrubLateP50": 0,+ "scrubLateP95": 1,+ "jumpLateP50": 1,+ "jumpLateP95": 2,"boundaryDrops": 0,"longTaskMs": 16,"filmstripSlackS": 0.2,"lagP95S": 0.05,"decayRatio": 1.5,"warmMb": 400,+ "memMb": 1600,"stallShare": 0.002,"lateShare": 0.02,"decayFloor": 0.02@@ -39,11 +40,79 @@"bucket": "scrub","pass": true,"notes": [],+ "late": {+ "p50": 0,+ "p95": 1,+ "mean": 0.2,+ "max": 2,+ "n": 41+ },+ "frameMs": 16.7,+ "scrub": {+ "p50": 15.7,+ "p95": 33.5,+ "mean": 18.89,+ "max": 47.8,+ "n": 41+ },+ "longTasks": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "max": 0,+ "n": 0+ },+ "liveSamples": 48,+ "machine": "desktop"+ },+ {+ "name": "drag-backward-within-a-clip",+ "bucket": "scrub",+ "pass": true,+ "notes": [],+ "late": {+ "p50": 0,+ "p95": 1,+ "mean": 0.15,+ "max": 2,+ "n": 41+ },+ "frameMs": 16.7,+ "scrub": {+ "p50": 15.9,+ "p95": 32.4,+ "mean": 18.11,+ "max": 48.2,+ "n": 41+ },+ "longTasks": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "max": 0,+ "n": 0+ },+ "liveSamples": 69,+ "machine": "desktop"+ },+ {+ "name": "drag-backward-across-a-cut",+ "bucket": "scrub",+ "pass": true,+ "notes": [],+ "late": {+ "p50": 0,+ "p95": 0,+ "mean": 0.05,+ "max": 1,+ "n": 41+ },+ "frameMs": 16.7,"scrub": {- "p50": 7.4,- "p95": 15.9,- "mean": 8,- "max": 16.3,+ "p50": 15.7,+ "p95": 16.6,+ "mean": 16.25,+ "max": 32.2,"n": 41},"longTasks": {@@ -53,7 +122,37 @@"max": 0,"n": 0},- "liveSamples": 63,+ "liveSamples": 101,+ "machine": "desktop"+ },+ {+ "name": "drag-backward-long-clip",+ "bucket": "scrub",+ "pass": true,+ "notes": [],+ "late": {+ "p50": 0,+ "p95": 0,+ "mean": 0.04,+ "max": 2,+ "n": 81+ },+ "frameMs": 16.7,+ "scrub": {+ "p50": 15.6,+ "p95": 16.6,+ "mean": 15.75,+ "max": 44.2,+ "n": 81+ },+ "longTasks": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "max": 0,+ "n": 0+ },+ "liveSamples": 48,"machine": "desktop"},{@@ -61,11 +160,19 @@"bucket": "scrub","pass": true,"notes": [],+ "late": {+ "p50": 0,+ "p95": 1,+ "mean": 0.17,+ "max": 1,+ "n": 41+ },+ "frameMs": 16.7,"scrub": {- "p50": 7.4,- "p95": 15.8,- "mean": 8.24,- "max": 15.9,+ "p50": 15.9,+ "p95": 31.8,+ "mean": 18.15,+ "max": 32.7,"n": 41},"longTasks": {@@ -75,7 +182,7 @@"max": 0,"n": 0},- "liveSamples": 46,+ "liveSamples": 65,"machine": "desktop"},{@@ -83,11 +190,19 @@"bucket": "scrub","pass": true,"notes": [],+ "late": {+ "p50": 0,+ "p95": 1,+ "mean": 0.17,+ "max": 1,+ "n": 41+ },+ "frameMs": 16.7,"scrub": {- "p50": 7.4,- "p95": 15.5,- "mean": 7.68,- "max": 15.8,+ "p50": 15.8,+ "p95": 32.4,+ "mean": 18.37,+ "max": 33,"n": 41},"longTasks": {@@ -97,7 +212,7 @@"max": 0,"n": 0},- "liveSamples": 60,+ "liveSamples": 54,"machine": "desktop"},{@@ -105,11 +220,19 @@"bucket": "scrub","pass": true,"notes": [],+ "late": {+ "p50": 1,+ "p95": 1,+ "mean": 0.9,+ "max": 1,+ "n": 41+ },+ "frameMs": 16.7,"scrub": {- "p50": 15.5,- "p95": 16,- "mean": 14.23,- "max": 24.3,+ "p50": 32.2,+ "p95": 32.9,+ "mean": 30.51,+ "max": 33.1,"n": 41},"longTasks": {@@ -119,7 +242,7 @@"max": 0,"n": 0},- "liveSamples": 71,+ "liveSamples": 60,"machine": "desktop"},{@@ -128,7 +251,7 @@"pass": true,"notes": [],"drops": 0,- "presented": 2075,+ "presented": 1038,"boundaryDrops": 0,"longTasks": {"p50": 0,@@ -145,8 +268,8 @@"bucket": "boundary","pass": true,"notes": [],- "drops": 1,- "presented": 2077,+ "drops": 0,+ "presented": 1038,"boundaryDrops": 0,"longTasks": {"p50": 0,@@ -164,13 +287,13 @@"pass": true,"notes": [],"drops": 0,- "presented": 1618,+ "presented": 808,"lag": {"p50": 0.02,"p95": 0.03,"mean": 0.02,"max": 0.03,- "n": 1618+ "n": 808},"stallShare": 0,"decay": {@@ -192,6 +315,7 @@"liveSources": 12,"keptSources": 20,"warmMb": 354,+ "memMb": 1108,"machine": "desktop"},{@@ -199,19 +323,19 @@"bucket": "playback","pass": true,"notes": [],- "drops": 5,- "presented": 6493,+ "drops": 0,+ "presented": 3240,"lag": {"p50": 0.02,"p95": 0.03,"mean": 0.02,"max": 0.03,- "n": 6493+ "n": 3240},"stallShare": 0,"decay": {"first": 0,- "last": 0.001+ "last": 0},"clock": {"lead": 0.04,@@ -228,6 +352,118 @@"liveSources": 12,"keptSources": 20,"warmMb": 354,+ "memMb": 1108,+ "machine": "desktop"+ },+ {+ "name": "play-replay-from-end",+ "bucket": "playback",+ "pass": true,+ "notes": [],+ "drops": 0,+ "presented": 222,+ "lag": {+ "p50": 0.02,+ "p95": 0.03,+ "mean": 0.01,+ "max": 0.03,+ "n": 222+ },+ "stallShare": 0,+ "decay": {+ "first": 0,+ "last": 0+ },+ "clock": {+ "lead": 0.04,+ "reported": 0.04+ },+ "longTasks": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "max": 0,+ "n": 0+ },+ "liveSamples": 60,+ "liveSources": 6,+ "keptSources": 6,+ "warmMb": 0,+ "memMb": 389,+ "machine": "desktop"+ },+ {+ "name": "play-user-project-cold",+ "bucket": "playback",+ "pass": true,+ "notes": [],+ "drops": 0,+ "presented": 1029,+ "lag": {+ "p50": 0.02,+ "p95": 0.03,+ "mean": 0.02,+ "max": 0.03,+ "n": 1029+ },+ "stallShare": 0,+ "decay": {+ "first": 0,+ "last": 0+ },+ "clock": {+ "lead": 0.04,+ "reported": 0.04+ },+ "longTasks": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "max": 0,+ "n": 0+ },+ "liveSamples": 60,+ "liveSources": 6,+ "keptSources": 6,+ "warmMb": 0,+ "memMb": 418,+ "machine": "desktop"+ },+ {+ "name": "play-user-project-replay",+ "bucket": "playback",+ "pass": true,+ "notes": [],+ "drops": 0,+ "presented": 1027,+ "lag": {+ "p50": 0.02,+ "p95": 0.03,+ "mean": 0.02,+ "max": 0.03,+ "n": 1027+ },+ "stallShare": 0,+ "decay": {+ "first": 0,+ "last": 0+ },+ "clock": {+ "lead": 0.04,+ "reported": 0.04+ },+ "longTasks": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "max": 0,+ "n": 0+ },+ "liveSamples": 60,+ "liveSources": 6,+ "keptSources": 6,+ "warmMb": 0,+ "memMb": 418,"machine": "desktop"},{@@ -235,19 +471,19 @@"bucket": "playback","pass": true,"notes": [],- "drops": 3,- "presented": 6891,+ "drops": 0,+ "presented": 3444,"lag": {"p50": 0.02,"p95": 0.03,"mean": 0.02,"max": 0.03,- "n": 6891+ "n": 3444},"stallShare": 0,"decay": {"first": 0,- "last": 0.001+ "last": 0},"clock": {"lead": 0.04,@@ -264,6 +500,7 @@"liveSources": 1,"keptSources": 1,"warmMb": 0,+ "memMb": 117,"machine": "desktop"},{@@ -271,19 +508,19 @@"bucket": "playback","pass": true,"notes": [],- "drops": 10,- "presented": 6889,+ "drops": 0,+ "presented": 3444,"lag": {"p50": 0.01,"p95": 0.02,"mean": 0.01,"max": 0.02,- "n": 6889+ "n": 3444},"stallShare": 0,"decay": {- "first": 0.001,- "last": 0.002+ "first": 0,+ "last": 0},"clock": {"lead": 0.04,@@ -300,28 +537,27 @@"liveSources": 1,"keptSources": 1,"warmMb": 0,+ "memMb": 117,"machine": "desktop"},{"name": "play-cold-bytes","bucket": "playback",- "pass": false,- "notes": [- "7% of frames did not arrive on time"- ],- "drops": 490,- "presented": 6890,+ "pass": true,+ "notes": [],+ "drops": 0,+ "presented": 3444,"lag": {"p50": 0.02,"p95": 0.03,"mean": 0.02,- "max": 0.97,- "n": 6410+ "max": 0.03,+ "n": 3444},- "stallShare": 0.0006,+ "stallShare": 0,"decay": {- "first": 0.211,- "last": 0.002+ "first": 0,+ "last": 0},"clock": {"lead": 0.04,@@ -338,6 +574,7 @@"liveSources": 1,"keptSources": 1,"warmMb": 0,+ "memMb": 117,"machine": "desktop"},{@@ -346,13 +583,13 @@"pass": true,"notes": [],"drops": 0,- "presented": 6884,+ "presented": 3443,"lag": {"p50": 0.02,"p95": 0.03,"mean": 0.02,"max": 0.03,- "n": 6884+ "n": 3443},"stallShare": 0,"decay": {@@ -374,6 +611,44 @@"liveSources": 1,"keptSources": 1,"warmMb": 0,+ "memMb": 134,+ "machine": "desktop"+ },+ {+ "name": "play-source-1080p60",+ "bucket": "playback",+ "pass": true,+ "notes": [],+ "drops": 0,+ "presented": 3439,+ "lag": {+ "p50": 0.01,+ "p95": 0.01,+ "mean": 0.01,+ "max": 0.02,+ "n": 3439+ },+ "stallShare": 0,+ "decay": {+ "first": 0,+ "last": 0+ },+ "clock": {+ "lead": 0.04,+ "reported": 0.04+ },+ "longTasks": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "max": 0,+ "n": 0+ },+ "liveSamples": 10,+ "liveSources": 1,+ "keptSources": 1,+ "warmMb": 0,+ "memMb": 144,"machine": "desktop"},{@@ -381,19 +656,19 @@"bucket": "playback","pass": true,"notes": [],- "drops": 3,- "presented": 6886,+ "drops": 0,+ "presented": 3444,"lag": {"p50": 0.02,"p95": 0.03,"mean": 0.02,"max": 0.03,- "n": 6886+ "n": 3444},"stallShare": 0,"decay": {"first": 0,- "last": 0.001+ "last": 0},"clock": {"lead": 0.04,@@ -410,6 +685,7 @@"liveSources": 1,"keptSources": 1,"warmMb": 0,+ "memMb": 117,"machine": "desktop"},{@@ -418,13 +694,13 @@"pass": true,"notes": [],"drops": 0,- "presented": 3236,+ "presented": 1620,"lag": {"p50": 0.02,"p95": 0.03,"mean": 0.02,"max": 0.03,- "n": 3236+ "n": 1620},"stallShare": 0,"decay": {@@ -446,6 +722,7 @@"liveSources": 12,"keptSources": 20,"warmMb": 354,+ "memMb": 1108,"machine": "desktop"},{@@ -462,10 +739,10 @@"notes": [],"tileErr": {"p50": 0.04,- "p95": 0.07,+ "p95": 0.05,"mean": 0.03,"max": 0.13,- "n": 35+ "n": 43}},{diff --git a/site/scripts/eval-cut-perf.ts b/site/scripts/eval-cut-perf.tsindex bae954e6..a58a00e6 100644--- a/site/scripts/eval-cut-perf.ts+++ b/site/scripts/eval-cut-perf.ts@@ -8,9 +8,11 @@* engine's own frame trace (`window.__cutPerf`, installed by devHooks.ts). Two* numbers come out of it:*- * scrub-to-pixel — from the moment a time is asked for to the moment a frame- * for that time is painted. Held or black frames don't count;- * only real pixels answer a scrub.+ * scrub-to-pixel — whether the first frame painted after a time is asked for+ * shows that time, counted in frames missed. Held or black+ * frames don't count; only real pixels answer a scrub. Frames+ * rather than milliseconds, so the report reads the same on+ * a sixty-hertz and a hundred-and-twenty-hertz display.* frames at a cut — every boundary in the fixture is crossed while playing and* the trace is checked for a source frame shown twice or* skipped over.@@ -31,19 +33,24 @@* Fixtures build deterministically into dist/cut-perf/ (gitignored) from the* bundled stock clips, so the montage's cut times are known exactly.*- * Needs `next dev` on :3000 and the dev auth bypass user. Spends no credits.+ * Needs `next dev` on :3000, the dev auth bypass user, and `openssl` on the+ * path for the fixture server's certificate. Spends no credits.** npm run eval:cut-perf [--only <case>] [--runs N] [--out path]- * [--enforce-budgets] [--headed]+ * [--enforce-budgets] [--headed] [--dump-trace]* [--machine desktop|laptop|ryzen-5500u|potato] [--cpu N]* [--net <kbps>] [--rtt <ms>]*/-import { chromium, type Browser, type Page, type Request, type Route } from "playwright";-import { spawn } from "node:child_process";-import { existsSync } from "node:fs";-import { mkdir, readFile, writeFile } from "node:fs/promises";+import { chromium, type Browser, type Page } from "playwright";+import { execFile, spawn } from "node:child_process";+import { once } from "node:events";+import { createReadStream, existsSync } from "node:fs";+import { mkdir, readFile, stat, writeFile } from "node:fs/promises";+import { createServer as createHttpServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";+import { createServer as createHttpsServer } from "node:https";import path from "node:path";+import { promisify } from "node:util";import { CUT_MEDIA_ORIGIN } from "../src/cut/lib/hosts";const SITE = path.resolve(import.meta.dir, "..");@@ -205,14 +212,14 @@ const DEV_USER = "donkey-dev-auth-bypass";// overhead. A production build has less of it; the gate is set where the dev// build can hold it under ordinary machine load.const BASE_GATE = {- // Every step of a drag lands inside one frame at 60Hz, tail included. That is- // the whole claim.- scrubP50Ms: 10,- scrubP95Ms: 16.7,+ // Every step of a drag is on the next frame painted, and the tail is at+ // most one frame behind it. That is the whole claim, in frames missed.+ scrubLateP50: 0,+ scrubLateP95: 1,// A jump lands where no buffer could have reached, so it pays for a decode// from the nearest keyframe. Held to what that honestly costs.- jumpP50Ms: 20,- jumpP95Ms: 32,+ jumpLateP50: 1,+ jumpLateP95: 2,// The one that has to be exact: crossing a join must never show a frame that// does not belong at that instant.boundaryDrops: 0,@@ -259,14 +266,14 @@ const GATE = {// the same thing.longTaskMs: BASE_GATE.longTaskMs * MACHINE.cpu,// A step of a drag is the same work on any machine: read the ring, compose,- // paint. What changes is how long a millisecond of it takes, so the budgets- // scale with the throttle for the same reason the long-task one does — a- // tenth of a machine answering a drag in a tenth of the rate is the drag- // behaving, and a fixed number would only say the profile is slow.- scrubP50Ms: BASE_GATE.scrubP50Ms * MACHINE.cpu,- scrubP95Ms: BASE_GATE.scrubP95Ms * MACHINE.cpu,- jumpP50Ms: BASE_GATE.jumpP50Ms * MACHINE.cpu,- jumpP95Ms: BASE_GATE.jumpP95Ms * MACHINE.cpu,+ // paint. What changes is how long a millisecond of it takes, so the frames a+ // step may miss scale with the throttle for the same reason the long-task+ // budget does — a tenth of a machine answering a drag in a tenth of the rate+ // is the drag behaving, and a fixed number would only say the profile is slow.+ scrubLateP50: Math.ceil(BASE_GATE.scrubLateP50 * MACHINE.cpu),+ scrubLateP95: Math.ceil(BASE_GATE.scrubLateP95 * MACHINE.cpu),+ jumpLateP50: Math.ceil(BASE_GATE.jumpLateP50 * MACHINE.cpu),+ jumpLateP95: Math.ceil(BASE_GATE.jumpLateP95 * MACHINE.cpu),};/** How far behind the sound the picture has to be for that frame to read as a@@ -294,6 +301,7 @@ interface SeekRecord {t: number;at: number;latencyMs: number | null;+ lateFrames: number | null;}interface Trace {presents: PresentRecord[];@@ -767,7 +775,32 @@ const seedTimeCoded = (src: string, cuts: number[]) => async (page: Page): Promi);const startTrace = (page: Page) =>- page.evaluate(() => (window as unknown as { __cutPerf: { start(): void } }).__cutPerf.start());+ page.evaluate(() => {+ // Room for every media request a long play makes, so the dump below can+ // put the walk's waits beside the requests they waited on.+ performance.setResourceTimingBufferSize(5000);+ (window as unknown as { __cutPerf: { start(): void } }).__cutPerf.start();+ });++/** The display's frame period as the page sees it: the median gap between+ * animation frames. Reported beside the frame counts so a reader can put+ * milliseconds to them. */+const frameMs = (page: Page) =>+ page.evaluate(+ () =>+ new Promise<number>((resolve) => {+ const ts: number[] = [];+ const step = (t: number) => {+ ts.push(t);+ if (ts.length < 30) requestAnimationFrame(step);+ else {+ const d = ts.slice(1).map((v, i) => v - ts[i]).sort((a, b) => a - b);+ resolve(+d[Math.floor(d.length / 2)].toFixed(2));+ }+ };+ requestAnimationFrame(step);+ })+ );const stopTrace = (page: Page) =>page.evaluate(@@ -857,6 +890,12 @@ interface CaseResult {bucket: Bucket;pass: boolean;notes: string[];+ /** Frames each step of a drag missed before its picture was painted. */+ late?: Agg;+ /** The display's frame period during the case, in milliseconds. */+ frameMs?: number;+ /** Milliseconds from the ask to the answering paint. Carries the wait for the+ * next frame, so it reads with `frameMs` beside it. */scrub?: Agg;drops?: number;presented?: number;@@ -937,6 +976,7 @@ interface ScrubShape {transitions: boolean;/** How the gesture moves. */gesture: (page: Page, fx: Fixture) => Promise<void>;+ /** Frames a step may miss, at the median and at the tail. */budget: { p50: number; p95: number };/** The project the gesture runs on; the montage of stock clips when unset. */seed?: (page: Page) => Promise<Fixture>;@@ -952,25 +992,29 @@ const scrubCase = (shape: ScrubShape): EvalCase => ({// A pass before the trace opens the decoders and lets the dev build settle,// so the numbers describe scrubbing rather than first load.await sweep(page, fx, 0, fx.duration - 0.05, 8);+ const frame = await frameMs(page);await startTrace(page);await shape.gesture(page, fx);if (has("--detail")) await dumpDetail(page, 0);const trace = await stopTrace(page);const notes: string[] = [];if (!trace) return { name: shape.name, bucket: "scrub", pass: false, notes: ["no trace"] };- const answered = trace.seeks.filter((s) => s.latencyMs !== null).map((s) => s.latencyMs!);+ // The raw trace beside the report, for reading which paint answered which+ // seek when the aggregates say a step is slow.+ if (has("--dump-trace")) await writeFile(path.join(OUT, `trace-${shape.name}.json`), JSON.stringify(trace));+ const answered = trace.seeks.filter((s) => s.latencyMs !== null);const unanswered = trace.seeks.length - answered.length;if (has("--detail")) {- const slow = [...trace.seeks]- .filter((s) => s.latencyMs !== null)- .sort((a, b) => b.latencyMs! - a.latencyMs!)+ const slow = [...answered]+ .sort((a, b) => b.lateFrames! - a.lateFrames! || b.latencyMs! - a.latencyMs!).slice(0, 8)- .map((s) => `${s.t.toFixed(2)}s=${s.latencyMs!.toFixed(1)}ms`);+ .map((s) => `${s.t.toFixed(2)}s=${s.lateFrames}f/${s.latencyMs!.toFixed(1)}ms`);console.log(`[seeks] slowest: ${slow.join(" ")}`);const missed = trace.seeks.filter((s) => s.latencyMs === null).map((s) => s.t.toFixed(2));if (missed.length) console.log(`[seeks] unanswered: ${missed.join(" ")}`);}- const a = agg(answered);+ const late = agg(answered.map((s) => s.lateFrames!));+ const ms = agg(answered.map((s) => s.latencyMs!));// Superseded seeks never resolve and that is correct — but a gesture that// waits a frame per step should answer nearly all of them.if (unanswered > trace.seeks.length * 0.2) {@@ -978,14 +1022,16 @@ const scrubCase = (shape: ScrubShape): EvalCase => ({}const lt = agg(trace.longTasks.map((l) => l.ms));if (lt.max > GATE.longTaskMs) notes.push(`long task ${lt.max}ms`);- if (a.p95 > shape.budget.p95) notes.push(`p95 ${a.p95}ms over ${shape.budget.p95}ms`);- if (a.p50 > shape.budget.p50) notes.push(`p50 ${a.p50}ms over ${shape.budget.p50}ms`);+ if (late.p95 > shape.budget.p95) notes.push(`p95 ${late.p95} frames late over ${shape.budget.p95}`);+ if (late.p50 > shape.budget.p50) notes.push(`p50 ${late.p50} frames late over ${shape.budget.p50}`);return {name: shape.name,bucket: "scrub",pass: notes.length === 0,notes,- scrub: a,+ late,+ frameMs: frame,+ scrub: ms,longTasks: lt,liveSamples: Math.max(0, ...trace.liveSamples),machine: MACHINE.name,@@ -1128,6 +1174,21 @@ async function judgePlay(const trace = await stopTrace(page);const notes: string[] = [];if (!trace) return { name, bucket: "playback", pass: false, notes: ["no trace"] };+ if (has("--dump-trace")) {+ await writeFile(path.join(OUT, `trace-${name}.json`), JSON.stringify(trace));+ // The engine's own log beside it: walks sent, slow pulls, tick gaps.+ const log = await page.evaluate(+ () => (window as unknown as { __cutEngineLog?: unknown }).__cutEngineLog ?? []+ );+ await writeFile(path.join(OUT, `log-${name}.json`), JSON.stringify(log));+ // And the media requests the page made, on the same clock.+ const net = await page.evaluate(() =>+ performance+ .getEntriesByType("resource")+ .map((e) => ({ name: e.name, start: e.startTime, ms: e.duration, bytes: (e as PerformanceResourceTiming).transferSize }))+ );+ await writeFile(path.join(OUT, `net-${name}.json`), JSON.stringify(net));+ }const decoders = await page.evaluate(() =>(@@ -1856,7 +1917,7 @@ const CASES: EvalCase[] = [name: "drag-within-a-clip",transitions: false,gesture: (page, fx) => sweep(page, fx, 0.4, 2, 40),- budget: { p50: GATE.scrubP50Ms, p95: GATE.scrubP95Ms },+ budget: { p50: GATE.scrubLateP50, p95: GATE.scrubLateP95 },}),// The same drag the other way. Frames decode forward from a keyframe, so a// pointer moving against that direction is served from a cache of the span@@ -1865,13 +1926,13 @@ const CASES: EvalCase[] = [name: "drag-backward-within-a-clip",transitions: false,gesture: (page, fx) => sweep(page, fx, 2.4, -2, 40),- budget: { p50: GATE.scrubP50Ms, p95: GATE.scrubP95Ms },+ budget: { p50: GATE.scrubLateP50, p95: GATE.scrubLateP95 },}),scrubCase({name: "drag-backward-across-a-cut",transitions: false,gesture: (page, fx) => sweep(page, fx, fx.cuts[0] + 1, -2, 40),- budget: { p50: GATE.scrubP50Ms, p95: GATE.scrubP95Ms },+ budget: { p50: GATE.scrubLateP50, p95: GATE.scrubLateP95 },}),// Backward through a long file read as cloud media: keyframes two seconds// apart, bytes off the link through the chunk cache, which is what a@@ -1881,19 +1942,19 @@ const CASES: EvalCase[] = [transitions: false,seed: seedLongClip(30, true),gesture: (page, fx) => sweep(page, fx, 12, -4, 80),- budget: { p50: GATE.scrubP50Ms, p95: GATE.scrubP95Ms },+ budget: { p50: GATE.scrubLateP50, p95: GATE.scrubLateP95 },}),scrubCase({name: "drag-across-a-cut",transitions: false,gesture: (page, fx) => sweep(page, fx, fx.cuts[0] - 1, 2, 40),- budget: { p50: GATE.scrubP50Ms, p95: GATE.scrubP95Ms },+ budget: { p50: GATE.scrubLateP50, p95: GATE.scrubLateP95 },}),scrubCase({name: "drag-across-a-transition",transitions: true,gesture: (page, fx) => sweep(page, fx, fx.cuts[0] - 1, 2, 40),- budget: { p50: GATE.scrubP50Ms, p95: GATE.scrubP95Ms },+ budget: { p50: GATE.scrubLateP50, p95: GATE.scrubLateP95 },}),// Landing somewhere else entirely costs a decode from the nearest keyframe,// which no arrangement of buffers can avoid. Budgeted for what that costs.@@ -1901,7 +1962,7 @@ const CASES: EvalCase[] = [name: "jump-across-the-cut",transitions: false,gesture: (page, fx) => sweep(page, fx, 0, fx.duration - 0.05, 40),- budget: { p50: GATE.jumpP50Ms, p95: GATE.jumpP95Ms },+ budget: { p50: GATE.jumpLateP50, p95: GATE.jumpLateP95 },}),playbackCase("play-hard-cuts", false),playbackCase("play-with-transitions", true),@@ -1969,13 +2030,12 @@ const CASES: EvalCase[] = [*/let linkKbps = MACHINE.netKbps;-/** Pause for the profile's round trip, then for as long as the link takes to- * carry `bytes`. Zero on either leaves the response alone, which is what keeps- * the desktop baseline still. */-const charge = (bytes: number): Promise<void> => {- const ms = MACHINE.rttMs + (linkKbps > 0 ? (bytes * 8) / linkKbps : 0);- return ms > 0 ? new Promise((r) => setTimeout(r, ms)) : Promise.resolve();-};+/** Bytes handed to the page at a time over a paced link. Small enough that a+ * reader waiting on the first chunk of a long run has it after a fraction of+ * a second, the way a real server's stream delivers it. */+const PIECE = 64 * 1024;++const sleep = (ms: number) => (ms > 0 ? new Promise<void>((r) => setTimeout(r, ms)) : Promise.resolve());/** Headers a cross-origin media read needs back. The chunk cache reads* `Content-Range` to learn the object's size, and a browser hides it from a@@ -1987,41 +2047,113 @@ const CORS = {};/**- * One fixture file. A `Range` header is not a simple header, so a cross-origin- * read preflights and that is answered here too.+ * The fixture files, served by a real HTTP server on this machine.+ *+ * A fulfilled route hands the page a body only once the whole of it is+ * here, and a sixteen-megabyte run then reaches the reader waiting on its+ * first chunk all at once, seconds after a stream would have delivered that+ * chunk. The chunk cache is built for the stream: it settles each chunk of a+ * run as its bytes land. So the files are served the way a server serves+ * them, a range at a time as bytes, and the page's requests are pointed here+ * by the route without the page knowing.+ *+ * `/disk/` is a file the browser already holds — an import in OPFS, a local+ * project's own bytes — and is read as fast as the disk gives it. `/link/` is+ * cloud media read over the wire, and only those reads are charged the+ * profile's link: the round trip first, then the bytes at the link's rate,+ * piece by piece, so the first piece arrives at the round trip and the last+ * when the link has carried them all.*- * `paced` is what separates the two media residencies this serves. A file the- * browser already holds — an import in OPFS, a local project's own bytes — is- * a disk read, and that is every fixture addressed at `/__cutperf/`. Cloud- * media is read over the wire, and only those reads are charged the profile's- * link.+ * A route may only send a request to a URL of the same protocol, and the+ * page addresses local fixtures on its own http origin and cloud media on the+ * https media origin, so the one handler listens on both: plain for the disk+ * files, and behind a self-signed certificate the browser is told to accept+ * for the link.*/-async function serveFixture(route: Route, request: Request, paced: boolean): Promise<void> {- if (request.method() === "OPTIONS") {- await route.fulfill({ status: 204, headers: CORS });- return;+interface FixtureServers {+ disk: Server;+ link: Server;+}++async function fixtureCert(): Promise<{ key: Buffer; cert: Buffer }> {+ const keyPath = path.join(OUT, "fixture-key.pem");+ const certPath = path.join(OUT, "fixture-cert.pem");+ if (!existsSync(keyPath) || !existsSync(certPath)) {+ await promisify(execFile)("openssl", [+ "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "3650",+ "-subj", "/CN=127.0.0.1", "-keyout", keyPath, "-out", certPath,+ ]);}- const name = path.basename(new URL(request.url()).pathname);- const body = await readFile(path.join(OUT, name));- const range = /^bytes=(\d+)-(\d*)$/.exec(request.headers()["range"] ?? "");- const from = range ? Number(range[1]) : 0;- const to = range- ? range[2]- ? Math.min(Number(range[2]), body.length - 1)- : body.length - 1- : body.length - 1;- const slice = body.subarray(from, to + 1);- if (paced) await charge(slice.length);- await route.fulfill({- status: range ? 206 : 200,- body: slice,- headers: {+ return { key: await readFile(keyPath), cert: await readFile(certPath) };+}++async function startFixtureServers(): Promise<FixtureServers> {+ const listen = (server: Server) =>+ new Promise<Server>((resolve) => {+ server.listen(0, "127.0.0.1", () => resolve(server));+ // The servers live as long as the browser does and must not keep the+ // process up once the run is over.+ server.unref();+ });+ const disk = await listen(createHttpServer(serveFixture));+ const link = await listen(createHttpsServer(await fixtureCert(), serveFixture));+ return { disk, link };+}++async function serveFixture(req: IncomingMessage, res: ServerResponse): Promise<void> {+ {+ const url = new URL(req.url ?? "/", "http://fixture");+ const [, kind, name] = url.pathname.split("/");+ if (req.method === "OPTIONS") {+ res.writeHead(204, CORS);+ res.end();+ return;+ }+ const file = path.join(OUT, path.basename(name ?? ""));+ const size = await stat(file)+ .then((st) => st.size)+ .catch(() => null);+ if (size === null || (kind !== "disk" && kind !== "link")) {+ res.writeHead(404, CORS);+ res.end();+ return;+ }+ const range = /^bytes=(\d+)-(\d*)$/.exec(req.headers.range ?? "");+ const from = range ? Number(range[1]) : 0;+ const to = range ? (range[2] ? Math.min(Number(range[2]), size - 1) : size - 1) : size - 1;+ res.writeHead(range ? 206 : 200, {...CORS,"Content-Type": "video/mp4","Accept-Ranges": "bytes",- ...(range ? { "Content-Range": `bytes ${from}-${to}/${body.length}` } : {}),- },- });+ "Content-Length": String(to - from + 1),+ ...(range ? { "Content-Range": `bytes ${from}-${to}/${size}` } : {}),+ });+ const paced = kind === "link" && (linkKbps > 0 || MACHINE.rttMs > 0);+ const stream = createReadStream(file, { start: from, end: to, highWaterMark: paced ? PIECE : 1 << 20 });+ try {+ if (paced) await sleep(MACHINE.rttMs);+ for await (const piece of stream) {+ const bytes = piece as Buffer;+ if (paced && linkKbps > 0) await sleep((bytes.length * 8) / linkKbps);+ if (res.destroyed) break;+ if (!res.write(bytes)) await once(res, "drain");+ }+ } catch {+ // The page let go of the read; nothing to finish.+ } finally {+ stream.destroy();+ res.end();+ }+ }+}++/** The server's address for one fixture, as the route sends the page there. */+function fixtureUrl(servers: FixtureServers, kind: "disk" | "link", requestUrl: string): string {+ const server = servers[kind];+ const addr = server.address();+ const port = typeof addr === "object" && addr ? addr.port : 0;+ const scheme = kind === "link" ? "https" : "http";+ return `${scheme}://127.0.0.1:${port}/${kind}/${path.basename(new URL(requestUrl).pathname)}`;}/** A browser with the fixture media served from disk and the session answered. */@@ -2167,6 +2299,8 @@ async function launch(): Promise<{ browser: Browser; page: Page }> {const context = await browser.newContext({extraHTTPHeaders: { "x-donkey-dev-auth-bypass": "1" },viewport: { width: 1600, height: 1000 },+ // The fixture server's certificate is its own.+ ignoreHTTPSErrors: true,});// The shim is what enforces the slot limit as well as the per-frame cost, so// asking for slots on a profile that pays no CPU cost has to install it too.@@ -2284,15 +2418,20 @@ async function launch(): Promise<{ browser: Browser; page: Page }> {// every seek to zero, which the filmstrip case's ground-truth probe relies// on being wrong about.//- // Cloud media's link is charged in the route rather than through CDP's- // network emulation, which throttles the whole context: the page, the dev- // build's module graph and the API all have to arrive at their usual speed,- // or every case measures Next's first load.- await context.route("**/__cutperf/*", (route, request) => serveFixture(route, request, false));+ // Cloud media's link is charged by the fixture server rather than through+ // CDP's network emulation, which throttles the whole context: the page, the+ // dev build's module graph and the API all have to arrive at their usual+ // speed, or every case measures Next's first load.+ const fixtures = await startFixtureServers();+ await context.route("**/__cutperf/*", (route, request) =>+ route.continue({ url: fixtureUrl(fixtures, "disk", request.url()) })+ );// The same files addressed as cloud media, which is the path a cloud project// actually plays: `chunkIdentity` claims this origin, so reads arrive as 2MB// ranged chunks through the chunk cache and land in OPFS as they do.- await context.route(`${CUT_MEDIA_ORIGIN}/**`, (route, request) => serveFixture(route, request, true));+ await context.route(`${CUT_MEDIA_ORIGIN}/**`, (route, request) =>+ route.continue({ url: fixtureUrl(fixtures, "link", request.url()) })+ );const page = await context.newPage();if (MACHINE.cpu > 1 || MACHINE.softwareMs > 0 || arg("--slots")) {if (MACHINE.cpu > 1) {@@ -2405,8 +2544,8 @@ async function fanOut(names: string[]): Promise<CaseResult[]> {}const detailOf = (r: CaseResult) =>- r.scrub- ? `p50=${r.scrub.p50}ms p95=${r.scrub.p95}ms max=${r.scrub.max}ms n=${r.scrub.n}`+ r.late+ ? `late p50=${r.late.p50} p95=${r.late.p95} max=${r.late.max} frames @${r.frameMs}ms (${r.scrub?.p50}/${r.scrub?.p95}ms) n=${r.late.n}`: r.tileErr? `errP50=${r.tileErr.p50}s errMax=${r.tileErr.max}s n=${r.tileErr.n}`: r.idleTicks !== undefineddiff --git a/site/src/cut/lib/perfTrace.test.ts b/site/src/cut/lib/perfTrace.test.tsnew file mode 100644index 00000000..7242433d--- /dev/null+++ b/site/src/cut/lib/perfTrace.test.ts@@ -0,0 +1,47 @@+import { describe, expect, test } from "bun:test";+import { markPresent, markSeek, markTick, startTrace, stopTrace } from "./perfTrace";++const present = (t: number, exact = true) =>+ markPresent({ t, srcTs: t, wantSrc: t, clipId: "c", exact, stale: false });++describe("seek latency in frames", () => {+ test("the first tick after the ask is not late", () => {+ startTrace();+ markTick();+ markSeek(1);+ markTick();+ present(1);+ const trace = stopTrace()!;+ expect(trace.seeks[0].lateFrames).toBe(0);+ expect(trace.seeks[0].latencyMs).not.toBeNull();+ });++ test("a held frame on the first tick makes the answer one frame late", () => {+ startTrace();+ markSeek(2);+ markTick();+ present(2, false);+ markTick();+ present(2);+ expect(stopTrace()!.seeks[0].lateFrames).toBe(1);+ });++ test("an ask made inside the tick that answers it is not late", () => {+ startTrace();+ markTick();+ markSeek(3);+ present(3);+ expect(stopTrace()!.seeks[0].lateFrames).toBe(0);+ });++ test("a superseded ask never resolves", () => {+ startTrace();+ markSeek(4);+ markSeek(5);+ markTick();+ present(5);+ const trace = stopTrace()!;+ expect(trace.seeks[0].lateFrames).toBeNull();+ expect(trace.seeks[1].lateFrames).toBe(0);+ });+});diff --git a/site/src/cut/lib/perfTrace.ts b/site/src/cut/lib/perfTrace.tsindex 690ffd20..bc59c8ed 100644--- a/site/src/cut/lib/perfTrace.ts+++ b/site/src/cut/lib/perfTrace.ts@@ -67,9 +67,23 @@ export interface PresentRecord {export interface SeekRecord {t: number;at: number;+ /** Engine ticks that had run when the time was asked for. The first tick+ * after it is the first chance to paint the answer. */+ tick: number;/** How long until a frame for this time was painted. Null while unresolved —* either still pending, or superseded by a later seek. */latencyMs: number | null;+ /**+ * Frames the answer missed: zero when the first paint after the ask showed+ * it, one when it took the next.+ *+ * This is the measure a drag is judged on. The milliseconds above also+ * count the wait for the next frame, and that wait is the display's — half+ * a frame on average, eight milliseconds at a hundred and twenty hertz and+ * sixteen at sixty — so the same engine reads twice as slow on one monitor+ * as on another. Frames missed reads the same on both.+ */+ lateFrames: number | null;}export interface LongTaskRecord {@@ -187,7 +201,7 @@ export function stopTrace(): Trace | null {/** A time was asked for — a scrub, a skim, a click on the ruler. */export function markSeek(t: number): void {if (!trace) return;- const rec: SeekRecord = { t, at: performance.now(), latencyMs: null };+ const rec: SeekRecord = { t, at: performance.now(), tick: trace.ticks, latencyMs: null, lateFrames: null };keep(trace.seeks, rec);pendingSeek = rec;}@@ -202,6 +216,9 @@ export function markPresent(rec: Omit<PresentRecord, "at">): void {// clock keeps running until the frame that belongs at that time is on screen.if (pendingSeek && rec.exact && !rec.stale && Math.abs(rec.t - pendingSeek.t) <= SAME_TIME) {pendingSeek.latencyMs = at - pendingSeek.at;+ // A present runs inside a tick, after the tick has counted itself; the+ // first tick after the ask is the one that could have answered it.+ pendingSeek.lateFrames = Math.max(0, trace.ticks - pendingSeek.tick - 1);pendingSeek = null;}}