Open source CapCut alternative.
TypeScript
test(site): a breached perf case runs again and the majority decides
David Le3a3d0e1382a6
diff --git a/.github/workflows/nightly-cut-perf.yml b/.github/workflows/nightly-cut-perf.ymlindex 83df520c..76f14737 100644--- a/.github/workflows/nightly-cut-perf.yml+++ b/.github/workflows/nightly-cut-perf.yml@@ -4,9 +4,11 @@ name: Nightly Cut Perf# in a real Chrome against a dev server, on a machine whose display and load# never change. The scrub cases are held to the `ci` machine profile's# budgets: the runner has no hardware video decoder, and that profile names-# what a scrub costs on one. The playback cases run for the record — their-# lag budgets describe a machine with a decoder — and the full report, with-# every scrub case's trace, is kept as an artifact.+# what a scrub costs on one. A case that breaches runs again, to three runs,+# and the majority decides, so one hiccup on the shared host is one run. The+# playback cases run for the record on the same profile — their lag budgets+# describe a machine with a decoder — and the full report, with every scrub+# case's trace, is kept as an artifact.on:workflow_dispatch:@@ -81,7 +83,7 @@ jobs:- name: Full eval, for the recordif: always()- run: npm run eval:cut-perf -- --out /tmp/cut-perf.json | tee /tmp/cut-perf.log+ run: npm run eval:cut-perf -- --machine ci --out /tmp/cut-perf.json | tee /tmp/cut-perf.log- name: Keys after a dropif: always()@@ -91,7 +93,11 @@ jobs:if: always()run: |{- echo '## Cut perf'+ echo '## Scrub gate, ci profile'+ echo '```'+ grep -E '^\[(ok|FAIL)|breached' /tmp/cut-perf-scrub.log || true+ echo '```'+ echo '## Full eval, for the record'echo '```'grep -E '^\[(ok|FAIL)' /tmp/cut-perf.log || trueecho '```'diff --git a/site/scripts/eval-cut-perf.ts b/site/scripts/eval-cut-perf.tsindex 80163e57..23ab71a6 100644--- a/site/scripts/eval-cut-perf.ts+++ b/site/scripts/eval-cut-perf.ts@@ -45,6 +45,10 @@* [--enforce-budgets] [--headed] [--dump-trace] [--software-decode]* [--machine desktop|laptop|ryzen-5500u|potato|software-decode|ci] [--cpu N]* [--net <kbps>] [--rtt <ms>] [--big-file <path>]+ *+ * `--enforce-budgets` holds the run to the machine's gate: a case that+ * breaches is run again, to three runs, and the majority decides. A+ * regression breaches every run; a hiccup on a shared host breaches one.*/import { chromium, type Browser, type Page } from "playwright";@@ -71,6 +75,11 @@ const arg = (name: string) => {const has = (name: string) => args.includes(name);const BASE = arg("--base") ?? "http://localhost:3000";const RUNS = Number(arg("--runs") ?? 1);+/** Set on the process the coordinator spawns per case. */+const CHILD = has("--child");+/** Runs a case that breached is taken to before the majority rule judges it.+ * Three is the fewest with a majority that one bad run cannot make. */+const RUNS_ON_BREACH = 3;const ONLY = arg("--only");const BUCKET = arg("--bucket");const ENFORCE = has("--enforce-budgets");@@ -2752,7 +2761,7 @@ async function fanOut(names: string[]): Promise<CaseResult[]> {MACHINE_NAME,"--runs",String(RUNS),- ...["--headed", "--dump-trace", "--detail", "--software-decode"].filter((f) => has(f)),+ ...["--headed", "--dump-trace", "--detail", "--software-decode", "--enforce-budgets"].filter((f) => has(f)),...(arg("--cpu") ? ["--cpu", String(MACHINE.cpu)] : []),...(arg("--slots") ? ["--slots", String(MACHINE.slots)] : []),...(arg("--sw") ? ["--sw", String(MACHINE.softwareMs)] : []),@@ -2763,6 +2772,7 @@ async function fanOut(names: string[]): Promise<CaseResult[]> {import.meta.path,"--only",name,+ "--child","--out",out,...passthrough,@@ -2819,7 +2829,12 @@ async function main(): Promise<void> {const results: CaseResult[] = [];for (const c of cases) {- for (let i = 0; i < RUNS; i++) {+ // Under the gate a breach earns the case more runs, up to the count the+ // majority rule needs; the runs a case is asked for it always gets.+ const runsFor = (done: CaseResult[]) =>+ ENFORCE && done.some((r) => !r.pass) ? Math.max(RUNS, RUNS_ON_BREACH) : RUNS;+ const mine: CaseResult[] = [];+ for (let i = 0; i < runsFor(mine); i++) {// A project per run. Seeding a second time into one that already holds// the first run's clips leaves the editor with media it cannot decode,// and every frame after that is a frame the picture never arrived for —@@ -2849,6 +2864,7 @@ async function main(): Promise<void> {if (shelfId) await unshelve(shelfId);}results.push(r);+ mine.push(r);const mark = r.pass ? "ok " : "FAIL";console.log(`[${mark}] ${c.name.padEnd(24)} ${detailOf(r)}${r.notes.length ? ` — ${r.notes.join("; ")}` : ""}`);}@@ -2889,8 +2905,9 @@ async function writeReport(results: CaseResult[]): Promise<void> {.map(([name, runs]) => `${name} (${runs.filter((r) => !r.pass).length}/${runs.length})`).join(", ");console.error(`\n${failed.length} case(s) breached the ${MACHINE.name} gate: ${how}`);- if (ENFORCE) process.exit(1);- console.error("(report only)");+ // A child reports to the coordinator, which judges the whole run.+ if (ENFORCE && !CHILD) process.exit(1);+ if (!ENFORCE) console.error("(report only)");}}