Skip to content

Open source CapCut alternative.

TypeScript

test(site): the preview perf eval runs on machine profiles

David Le0d9963a0a91d
diff --git a/site/scripts/eval-cut-perf.ts b/site/scripts/eval-cut-perf.ts
index 79cbccef..33b0fe3f 100644
--- a/site/scripts/eval-cut-perf.ts
+++ b/site/scripts/eval-cut-perf.ts
@@ -35,6 +35,7 @@
*
* npm run eval:cut-perf [--only <case>] [--runs N] [--out path]
* [--enforce-budgets] [--headed]
+ * [--machine desktop|laptop|ryzen-5500u|potato] [--cpu N]
*/
import { chromium, type Browser, type Page } from "playwright";
@@ -59,6 +60,67 @@ const RUNS = Number(arg("--runs") ?? 1);
const ONLY = arg("--only");
const BUCKET = arg("--bucket");
const ENFORCE = has("--enforce-budgets");
+
+/**
+ * The machine this run pretends to be.
+ *
+ * A preview that holds together on a developer's desktop says very little
+ * about the laptop most people edit on. The desktop has decode slots to spare
+ * and never falls behind, so the pool's caps never bind, the budget controller
+ * never has to do anything, and a regression in either is invisible — every
+ * pass composes at the same rate and reports the same numbers. These profiles
+ * put the page on a smaller machine and hold the picture to what a viewer
+ * would accept there.
+ *
+ * Two kinds of pressure, because the failure needs both. A slow main thread
+ * makes the compositor miss frames. Few decode slots make the decoders behind
+ * those frames deliver late, which is the part that actually drifts the
+ * picture away from the sound on a real laptop.
+ */
+interface Machine {
+ name: string;
+ /** Main-thread slowdown as a multiplier: 6 gives the page a sixth of this
+ * machine, applied through the same CDP knob DevTools uses. */
+ cpu: number;
+ /** Hardware decode slots. A decoder past this still works; it just delivers
+ * its frames late, which is what falling back to software costs. Zero leaves
+ * decoding alone. */
+ slots: number;
+ /** What a frame costs once it is decoding in software. */
+ softwareMs: number;
+ /** How far the picture may sit behind the sound here before a viewer would
+ * call it broken. */
+ lagP95S: number;
+ lagMaxS: number;
+ /** Share of played frames that may show a picture which does not belong at
+ * that instant. This is the plainest reading of "does the video play": a
+ * frame the decoder failed to deliver on time is a frame the viewer sees
+ * repeated or missing. */
+ lateShare: number;
+}
+
+const MACHINES: Machine[] = [
+ // The machine this is being written on: nothing binds, and the numbers are
+ // held to the tightest budget in the file.
+ { name: "desktop", cpu: 1, slots: 0, softwareMs: 0, lagP95S: 0.05, lagMaxS: 0.25, lateShare: 0.02 },
+ // A mainstream laptop with an integrated GPU.
+ { name: "laptop", cpu: 4, slots: 6, softwareMs: 6, lagP95S: 0.08, lagMaxS: 0.3, lateShare: 0.08 },
+ // The one people report from: a 15W six-core with Vega graphics, running a
+ // browser that is also drawing the editor.
+ { name: "ryzen-5500u", cpu: 10, slots: 4, softwareMs: 8, lagP95S: 0.1, lagMaxS: 0.4, lateShare: 0.12 },
+ // Worse than anything anyone has reported, which is the point: what holds
+ // here holds on the machines that have not written in yet.
+ { name: "potato", cpu: 20, slots: 2, softwareMs: 12, lagP95S: 0.15, lagMaxS: 0.6, lateShare: 0.2 },
+];
+
+const MACHINE_NAME = arg("--machine") ?? "desktop";
+const MACHINE = ((): Machine => {
+ const m = MACHINES.find((x) => x.name === MACHINE_NAME);
+ if (!m) throw new Error(`unknown --machine ${MACHINE_NAME}; try ${MACHINES.map((x) => x.name).join(", ")}`);
+ // `--cpu` overrides the profile's throttle, for finding where a machine
+ // actually breaks.
+ return { ...m, cpu: Number(arg("--cpu") ?? m.cpu) };
+})();
/** The dev-only account the API bypass authenticates as. */
const DEV_USER = "donkey-dev-auth-bypass";
@@ -73,7 +135,7 @@ const DEV_USER = "donkey-dev-auth-bypass";
// These are measured against `next dev`, so they carry the development build's
// overhead. A production build has less of it; the gate is set where the dev
// build can hold it under ordinary machine load.
-const GATE = {
+const BASE_GATE = {
// Every step of a drag lands inside one frame at 60Hz, tail included. That is
// the whole claim.
scrubP50Ms: 10,
@@ -107,6 +169,15 @@ const GATE = {
warmMb: 400,
};
+/** The budget this run is judged against: the tight one, with the picture-lag
+ * limits relaxed to what the chosen machine can honestly hold. */
+const GATE = {
+ ...BASE_GATE,
+ lagP95S: MACHINE.lagP95S,
+ lagMaxS: MACHINE.lagMaxS,
+ lateShare: MACHINE.lateShare,
+};
+
// ── Types shared with perfTrace.ts ──────────────────────────────────────────
interface PresentRecord {
t: number;
@@ -464,6 +535,34 @@ const setPlaying = (page: Page, playing: boolean) =>
dev.useEditor.getState().setPlaying(p);
}, playing);
+/**
+ * Play from where the playhead is to the end of the cut, and wait for it to
+ * get there.
+ *
+ * Waiting on the playhead rather than on a stopwatch is what makes a slow
+ * machine measurable. A fixed timeout cuts the pass off wherever the throttled
+ * page happened to have reached, so the run ends early, the trace is a
+ * fraction of the length the fast machine's was, and the two are not
+ * comparable — one of them simply saw less of the cut. Here every profile
+ * plays the whole thing; the slow one just takes longer over it.
+ */
+async function playThrough(page: Page, duration: number): Promise<void> {
+ await setPlaying(page, true);
+ await page.waitForFunction(
+ (end: number) => {
+ const dev = (window as unknown as {
+ __cutDev: { playheadAt(): number; useEditor: { getState(): { playing: boolean } } };
+ }).__cutDev;
+ return dev.playheadAt() >= end || !dev.useEditor.getState().playing;
+ },
+ duration - 0.05,
+ // Generous: a machine given a twelfth of a CPU still has to get to the end,
+ // and the point of the wait is that it does.
+ { timeout: Math.max(30_000, duration * 6000), polling: 100 }
+ );
+ await setPlaying(page, false);
+}
+
/** Put the playhead back to the start and wait until it is actually there. */
async function rewind(page: Page): Promise<void> {
await setPlaying(page, false);
@@ -521,6 +620,11 @@ interface CaseResult {
/** Seconds between a strip tile's own moment and the moment its picture
* was captured at. */
tileErr?: Agg;
+ /** Concurrent decoders at their peak, and how many of them had to run in
+ * software because this machine had no hardware slot left for them. */
+ decoders?: { peak: number; software: number; softwarePeak: number };
+ /** Which machine profile the run was judged as. */
+ machine?: string;
}
interface EvalCase {
@@ -599,6 +703,7 @@ const scrubCase = (shape: ScrubShape): EvalCase => ({
scrub: a,
longTasks: lt,
liveSamples: Math.max(0, ...trace.liveSamples),
+ machine: MACHINE.name,
};
},
});
@@ -621,13 +726,18 @@ const playbackCase = (name: string, transitions: boolean): EvalCase => ({
await seek(page, 0);
await settle(page);
await startTrace(page);
- await setPlaying(page, true);
- // Play the whole fixture through, plus a moment to land the last frame.
- await page.waitForTimeout((fx.duration + 1) * 1000);
- await setPlaying(page, false);
+ await playThrough(page, fx.duration);
const trace = await stopTrace(page);
const notes: string[] = [];
if (!trace) return { name, bucket: "playback", pass: false, notes: ["no trace"] };
+ const decoders = await page.evaluate(
+ () =>
+ (
+ window as unknown as {
+ __cutDecoders?: { peak: number; software: number; softwarePeak: number };
+ }
+ ).__cutDecoders ?? null
+ );
// A frame the decoder failed to deliver: the engine painted, but with a
// picture that does not belong at that instant. Repeating a source frame is
@@ -662,6 +772,8 @@ const playbackCase = (name: string, transitions: boolean): EvalCase => ({
boundaryDrops,
longTasks: lt,
liveSamples: Math.max(0, ...trace.liveSamples),
+ decoders: decoders ?? undefined,
+ machine: MACHINE.name,
};
},
});
@@ -690,8 +802,7 @@ const montageCase = (name: string, passes: number): EvalCase => ({
await startTrace(page);
for (let i = 0; i < passes; i++) {
await seek(page, 0);
- await setPlaying(page, true);
- await page.waitForTimeout((fx.duration + 0.6) * 1000);
+ await playThrough(page, fx.duration);
if (has("--detail")) {
const state = await page.evaluate(() => {
const w = window as unknown as {
@@ -721,6 +832,14 @@ const montageCase = (name: string, passes: number): EvalCase => ({
const trace = await stopTrace(page);
const notes: string[] = [];
if (!trace) return { name, bucket: "playback", pass: false, notes: ["no trace"] };
+ const decoders = await page.evaluate(
+ () =>
+ (
+ window as unknown as {
+ __cutDecoders?: { peak: number; software: number; softwarePeak: number };
+ }
+ ).__cutDecoders ?? null
+ );
// The moments a clip's decoder is being opened for the first time in a pass
// belong to the open, so the measure starts a beat inside every clip and
@@ -788,6 +907,15 @@ const montageCase = (name: string, passes: number): EvalCase => ({
}
}
+ // The plainest question this case asks, and the one a viewer would ask:
+ // did the frames arrive. A run where nothing decoded at all used to score
+ // a clean sheet here — every frame stale, no lag to measure, because there
+ // was never a picture to be behind.
+ const lateShare = +(late.length / played.length).toFixed(3);
+ if (lateShare > GATE.lateShare) {
+ notes.push(`${Math.round(lateShare * 100)}% of frames did not arrive on time`);
+ }
+ if (decoders && decoders.peak === 0) notes.push("nothing decoded at all");
if (lag.p95 > GATE.lagP95S) notes.push(`picture ${lag.p95}s behind the clock at p95`);
if (lag.max > GATE.lagMaxS) notes.push(`picture ${lag.max}s behind the clock at worst`);
if (passes > 1 && decay.last > Math.max(0.02, decay.first * GATE.decayRatio)) {
@@ -812,6 +940,8 @@ const montageCase = (name: string, passes: number): EvalCase => ({
liveSources: Math.max(0, ...trace.liveSources),
keptSources: Math.max(0, ...trace.keptSources),
warmMb,
+ decoders: decoders ?? undefined,
+ machine: MACHINE.name,
};
},
});
@@ -1168,6 +1298,71 @@ const CASES: EvalCase[] = [
// ── Run ─────────────────────────────────────────────────────────────────────
/** A browser with the fixture media served from disk and the session answered. */
+/**
+ * Give the page a fixed number of hardware decode slots.
+ *
+ * Chrome hands a tab as many as the GPU and driver happen to allow and offers
+ * no way to ask for fewer, so this stands in for that limit: a decoder built
+ * once the slots are gone has its output deferred, which is what falling back
+ * to software costs the picture — the frames still arrive, they just arrive
+ * late. Everything else about the decoder is the real one.
+ *
+ * The delay scales with two things, because those are what a CPU decoding
+ * video actually charges for. How many decoders are in software at that
+ * moment: one stream on the CPU is nothing, eight of them are eight streams
+ * sharing one CPU, each slower for every other one open. And how big the
+ * frames are: decode cost runs with pixel count, so a stream decoded at half
+ * height costs a quarter as much. Both are levers the preview can pull —
+ * closing decoders, or asking for smaller frames — and the point of modelling
+ * them is that the eval can tell which one is worth pulling.
+ *
+ * Browser source, injected before any page script, so `window.VideoDecoder` is
+ * already wrapped by the time the preview reaches for it.
+ */
+const decoderShim = ({ slots, softwareMs }: Machine) => `
+(() => {
+ const Native = window.VideoDecoder;
+ if (!Native) return;
+ const stats = (window.__cutDecoders = { peak: 0, software: 0, softwarePeak: 0 });
+ let live = 0;
+ let onCpu = 0;
+ const freed = new WeakSet();
+ const slow = new WeakSet();
+ window.VideoDecoder = class extends Native {
+ constructor(init) {
+ const overflow = live >= ${slots};
+ super(
+ overflow
+ ? Object.assign({}, init, {
+ output: (frame) => {
+ // Cost runs with area, against 720p as the unit.
+ const area = ((frame.codedWidth || 1280) * (frame.codedHeight || 720)) / (1280 * 720);
+ setTimeout(() => init.output(frame), ${softwareMs} * Math.max(1, onCpu) * area);
+ },
+ })
+ : init
+ );
+ live++;
+ stats.peak = Math.max(stats.peak, live);
+ if (overflow) {
+ slow.add(this);
+ onCpu++;
+ stats.software++;
+ stats.softwarePeak = Math.max(stats.softwarePeak, onCpu);
+ }
+ }
+ close() {
+ if (!freed.has(this)) {
+ freed.add(this);
+ live--;
+ if (slow.has(this)) onCpu--;
+ }
+ return super.close();
+ }
+ };
+})();
+`;
+
async function launch(): Promise<{ browser: Browser; page: Page }> {
const browser = await chromium.launch({
channel: "chrome",
@@ -1180,6 +1375,7 @@ async function launch(): Promise<{ browser: Browser; page: Page }> {
extraHTTPHeaders: { "x-donkey-dev-auth-bypass": "1" },
viewport: { width: 1600, height: 1000 },
});
+ if (MACHINE.slots > 0) await context.addInitScript({ content: decoderShim(MACHINE) });
// The editor's session gate is a client-side cookie read, and the only sign-in
// this build offers is Google's. Answering that one request for the dev-bypass
// user is the whole of the fake: every other request the page makes is real,
@@ -1313,6 +1509,16 @@ async function launch(): Promise<{ browser: Browser; page: Page }> {
});
});
const page = await context.newPage();
+ if (MACHINE.cpu > 1 || MACHINE.slots > 0) {
+ if (MACHINE.cpu > 1) {
+ const cdp = await context.newCDPSession(page);
+ await cdp.send("Emulation.setCPUThrottlingRate", { rate: MACHINE.cpu });
+ }
+ console.log(
+ `[machine] ${MACHINE.name}: cpu 1/${MACHINE.cpu}` +
+ (MACHINE.slots ? `, ${MACHINE.slots} decode slots, +${MACHINE.softwareMs}ms in software` : "")
+ );
+ }
page.on("pageerror", (e) => console.log(`[pageerror] ${String(e).slice(0, 200)}`));
page.on("crash", () => console.log("[crash] the page went down"));
return { browser, page };
@@ -1343,11 +1549,31 @@ async function open(page: Page, projectId: string): Promise<void> {
* what is being measured is the engine from a cold start, the isolation has to
* be real, and a process is the only boundary that has proved to be.
*/
+/** A fresh project on the dev account for one run to seed into. */
+async function newProject(): Promise<string> {
+ const res = await fetch(`${BASE}/api/cut/projects?u=${DEV_USER}`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ name: "cut-perf eval" }),
+ });
+ if (!res.ok) throw new Error(`create project failed: ${res.status} (is next dev running?)`);
+ return ((await res.json()) as { id: string }).id;
+}
+
async function fanOut(names: string[]): Promise<CaseResult[]> {
const results: CaseResult[] = [];
for (const name of names) {
const out = path.join(OUT, `case-${name}.json`);
- const passthrough = ["--base", BASE, ...(has("--headed") ? ["--headed"] : [])];
+ const passthrough = [
+ "--base",
+ BASE,
+ "--machine",
+ MACHINE_NAME,
+ "--runs",
+ String(RUNS),
+ ...(has("--headed") ? ["--headed"] : []),
+ ...(arg("--cpu") ? ["--cpu", String(MACHINE.cpu)] : []),
+ ];
await run(process.execPath, [
import.meta.path,
"--only",
@@ -1374,7 +1600,7 @@ const detailOf = (r: CaseResult) =>
: r.idleTicks !== undefined
? `ticks=${r.idleTicks}`
: r.lag
- ? `late=${r.drops}/${r.presented} lagP50=${r.lag.p50}s lagP95=${r.lag.p95}s lagMax=${r.lag.max}s decoders=${r.liveSources} kept=${r.keptSources} warm=${r.warmMb}MB decay=${r.decay?.first}→${r.decay?.last}`
+ ? `late=${r.drops}/${r.presented} lagP95=${r.lag.p95}s lagMax=${r.lag.max}s sources=${r.liveSources} sw=${r.decoders?.softwarePeak ?? "-"}/${r.decoders?.software ?? "-"} warm=${r.warmMb}MB decay=${r.decay?.first}→${r.decay?.last}`
: `late=${r.drops}/${r.presented} atCut=${r.boundaryDrops}`;
async function main(): Promise<void> {
@@ -1398,18 +1624,16 @@ async function main(): Promise<void> {
}
const { browser, page } = await launch();
- const res = await fetch(`${BASE}/api/cut/projects?u=${DEV_USER}`, {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ name: "cut-perf eval" }),
- });
- if (!res.ok) throw new Error(`create project failed: ${res.status} (is next dev running?)`);
- const projectId = ((await res.json()) as { id: string }).id;
- console.log(`[ready] ${BASE}/app/p/${projectId}`);
const results: CaseResult[] = [];
for (const c of cases) {
for (let i = 0; i < RUNS; i++) {
+ // A project per run. Seeding a second time into one that already holds
+ // the first run's clips leaves the editor with media it cannot decode,
+ // and every frame after that is a frame the picture never arrived for —
+ // which read as a clean sheet for as long as the only gates here were
+ // about lag, since a picture that never comes is never behind.
+ const projectId = await newProject();
await open(page, projectId);
const fx = await (c.seed ? c.seed(page) : seed(page, files, c.transitions));
await settle(page);
@@ -1428,6 +1652,7 @@ async function main(): Promise<void> {
async function writeReport(results: CaseResult[]): Promise<void> {
const report = {
schema: "cut-perf-eval/v1",
+ machine: MACHINE,
gate: GATE,
fixture: { clips: CLIPS, clipSeconds: CLIP_S },
results,
@@ -1439,12 +1664,24 @@ async function writeReport(results: CaseResult[]): Promise<void> {
console.log(`[report] ${outPath}`);
}
- const failed = results.filter((r) => !r.pass);
- if (failed.length && ENFORCE) {
- console.error(`\n${failed.length} case(s) breached the gate.`);
- process.exit(1);
+ // A case is judged on its runs together. On a machine deliberately starved
+ // of CPU one bad run in three is the noise floor, and a gate that trips on
+ // it is a gate people learn to re-run until it goes green.
+ const byCase = new Map<string, CaseResult[]>();
+ for (const r of results) {
+ const runs = byCase.get(r.name) ?? [];
+ runs.push(r);
+ byCase.set(r.name, runs);
+ }
+ const failed = [...byCase].filter(([, runs]) => runs.filter((r) => !r.pass).length * 2 > runs.length);
+ if (failed.length) {
+ const how = failed
+ .map(([name, runs]) => `${name} (${runs.filter((r) => !r.pass).length}/${runs.length})`)
+ .join(", ");
+ console.error(`\n${failed.length} case(s) breached the ${MACHINE.name} gate: ${how}`);
+ if (ENFORCE) process.exit(1);
+ console.error("(report only)");
}
- if (failed.length) console.error(`\n${failed.length} case(s) breached the gate (report only).`);
}
main().catch((e) => {