A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
fix(tooling): the perf gate reads the fastest sample, not the average
esengine5c8af4d545e4
diff --git a/sdk/etc/perf.snapshot.json b/sdk/etc/perf.snapshot.jsonindex 2a8c3caa0..f48e3bf1f 100644--- a/sdk/etc/perf.snapshot.json+++ b/sdk/etc/perf.snapshot.json@@ -1,13 +1,12 @@{"note": "Generated by tools/perf-guard.mjs — run --update to accept changes.",- "tolerance": 0.3,+ "tolerance": 0.5,"cases": 230,"ratios": {- "ui-layout: editing a field vs adding a node": 0.227,- "ui-layout: editing a field vs an idle frame": 10.9,- "query: count vs iterating": 0.00609,+ "ui-layout: editing a field vs adding a node": 0.195,+ "ui-layout: editing a field vs an idle frame": 9.33,+ "query: count vs iterating": 0.0046,"change-tracking: tracked vs untracked write-back": 1.05,- "uiMath: reusing a result vs allocating one": 0.288,- "system: three params vs none": 3.29+ "uiMath: reusing a result vs allocating one": 0.289}}diff --git a/tools/perf-guard.mjs b/tools/perf-guard.mjsindex 422304542..73d769ea4 100644--- a/tools/perf-guard.mjs+++ b/tools/perf-guard.mjs@@ -29,11 +29,18 @@// is the failure this guard was written after, and it is invisible in every// other signal.//-// TOLERANCE is deliberately loose. Measured run-to-run noise on one machine is-// ~1% median but reaches 14% on the smallest cases, and a ratio compounds two of-// those. At 30% this cannot see a 5% slowdown — it is not meant to. It is meant-// to catch the 2x-and-up kind, which is what an architectural property looks-// like when it breaks.+// THE STATISTIC IS `min`, not the mean. Noise on a shared CI runner is one-sided+// — scheduling can only ever make a sample slower — so the fastest sample is the+// one closest to what the code actually costs, and the mean is the one that+// moves when the machine is busy. Read as means, this gate's first CI run+// reported four regressions of +33% to +49% in the same direction on a build+// that had changed none of them.+//+// TOLERANCE is deliberately loose on top of that. Run-to-run noise on one+// machine is ~1% median but reaches 14% on the smallest cases, and a ratio+// compounds two of those. At 50% this cannot see a 5% slowdown — it is not meant+// to. It is meant to catch the 2x-and-up kind, which is what an architectural+// property looks like when it breaks.//// Run: node tools/perf-guard.mjs --check (CI: exit 1 on regression)// node tools/perf-guard.mjs --update (accept the new numbers)@@ -86,16 +93,14 @@ const METRICS = [of: 'invertMatrix4 x100 (result reuse)',over: 'invertMatrix4 x100 (no result reuse)',},- {- name: 'system: three params vs none',- why: 'Param resolution per dispatch — bounded, not proportional to work.',- group: 'System - Dispatch overhead (empty system)',- of: '3 params (Query + Res + Commands)',- over: '0 params',- },];-const DEFAULT_TOLERANCE = 0.30;+// A metric whose denominator is tens of nanoseconds cannot be guarded: the+// fastest sample of a system dispatch with no params rounds to zero, and a ratio+// between two quantities at the timer's resolution is measuring the timer. The+// dispatch benchmarks stay — they are worth reading — but nothing gates on them.++const DEFAULT_TOLERANCE = 0.50;const mode = process.argv[2] ?? '--check';if (mode !== '--check' && mode !== '--update') {@@ -124,17 +129,17 @@ try {}function main(report) {- // group + case -> mean ms. A case with no `mean` never ran; it is counted- // as missing rather than read as a zero.- const means = new Map();+ // group + case -> fastest sample, in ms. A case with no timing never ran; it+ // is counted as missing rather than read as a zero.+ const times = new Map();let cases = 0;for (const file of report.files ?? []) {for (const group of file.groups ?? []) {const groupName = group.fullName.replace(/^.*bench\.ts > /, '');for (const b of group.benchmarks ?? []) {- if (b.mean === undefined) continue;+ if (b.min === undefined || b.mean === undefined) continue;cases++;- means.set(`${groupName} :: ${b.name}`, b.mean);+ times.set(`${groupName} :: ${b.name}`, b.min);}}}@@ -142,10 +147,16 @@ function main(report) {const measured = { cases, ratios: {} };const missing = [];for (const m of METRICS) {- const num = means.get(`${m.group} :: ${m.of}`);- const den = means.get(`${m.group} :: ${m.over}`);- if (num === undefined || den === undefined || den === 0) {- missing.push(m.name);+ const num = times.get(`${m.group} :: ${m.of}`);+ const den = times.get(`${m.group} :: ${m.over}`);+ if (num === undefined || den === undefined) {+ missing.push(`${m.name} (a benchmark it names did not run)`);+ continue;+ }+ // Below the timer's resolution the fastest sample is 0, and a ratio+ // against it says nothing. Pick a denominator with real duration.+ if (den === 0) {+ missing.push(`${m.name} (its denominator is too fast to time)`);continue;}measured.ratios[m.name] = round(num / den);