A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
test(native): measure what a launch cost, and let the frame go unjudged
esenginec35738fa9b5a
diff --git a/tools/verify-native-boot.mjs b/tools/verify-native-boot.mjsindex 73c1f110e..3fad52a58 100644--- a/tools/verify-native-boot.mjs+++ b/tools/verify-native-boot.mjs@@ -24,6 +24,14 @@* node tools/verify-native-boot.mjs --platform android --examples all* node tools/verify-native-boot.mjs --platform android --examples all --shard 1/3*+ * For a compatibility run — the same APK on one emulator per Android version —+ * `--no-frame-judge` drops the pixel question entirely and `--metrics-out` files+ * what the launch cost, so the versions can be compared and the frames reviewed+ * by someone. There, a dark frame is a thing to look at, not a thing to fail on:+ *+ * node tools/verify-native-boot.mjs --platform android --apk game.apk \+ * --label api29 --no-frame-judge --metrics-out build/compat/api29.json+ ** `--examples` packages each project the way the editor's Package dialog does and* runs the same two questions against every one, on ONE booted device: booting is* minutes and each app is seconds, so a device per example would spend the whole@@ -78,10 +86,15 @@ function parseArgs(argv) {minColors: 2,out: path.join(ROOT, 'build', 'native-boot'),template: path.join(ROOT, 'template'),+ // Names the artifacts. One compatibility job per Android version writes+ // into a directory the PR comment reads as a whole, so the version has to+ // be in the filename or eight runs land on top of each other.+ label: 'app',};for (let i = 0; i < argv.length; i++) {const key = argv[i].replace(/^--/, '');if (key === 'allow-skip') { opts.allowSkip = true; continue; }+ if (key === 'no-frame-judge') { opts.frameJudge = false; continue; }opts[key.replace(/-([a-z])/g, (_, c) => c.toUpperCase())] = argv[++i];}opts.timeout = Number(opts.timeout);@@ -115,6 +128,34 @@ function quietly(what, cmd, args) {// Android — adb against whatever emulator or phone is attached// =============================================================================+/** `+1s234ms`, `+834ms` — how the platform writes a launch duration. */+function launchDuration(text) {+ const m = /\+(?:(\d+)m)?(?:(\d+)s)?(\d+)ms/.exec(text ?? '');+ if (!m) return null;+ return Number(m[1] ?? 0) * 60_000 + Number(m[2] ?? 0) * 1000 + Number(m[3]);+}++const firstNumber = (re, text) => {+ const m = re.exec(text ?? '');+ return m ? Number(m[1]) : null;+};++/**+ * utime+stime for a pid, in clock ticks.+ *+ * `comm` is parenthesised and may itself contain spaces and parentheses, so the+ * positional fields start after the LAST `)` — splitting the whole line on+ * whitespace misreads any process whose name has a space in it.+ */+function cpuTicks(adb, pid) {+ const stat = trySh(adb, ['shell', 'cat', `/proc/${pid}/stat`]).stdout ?? '';+ const fields = stat.slice(stat.lastIndexOf(')') + 1).trim().split(/\s+/);+ if (fields.length < 13) return null;+ const utime = Number(fields[11]);+ const stime = Number(fields[12]);+ return Number.isFinite(utime + stime) ? utime + stime : null;+}+function androidDriver(opts) {const adb = process.env.ANDROID_HOME? path.join(process.env.ANDROID_HOME, 'platform-tools', 'adb') : 'adb';@@ -152,8 +193,11 @@ function androidDriver(opts) {trySh(adb, ['shell', 'rm', '-f', logFile]);trySh(adb, ['logcat', '-c']);},+ // `-W` makes this wait for the launch to finish and print what it cost, so+ // the timing comes from the platform's own measurement rather than from a+ // stopwatch around adb — which would also be timing adb.launch() {- sh(adb, ['shell', 'am', 'start', '-W', '-n', `${APP_ID}/android.app.NativeActivity`]);+ return sh(adb, ['shell', 'am', 'start', '-W', '-n', `${APP_ID}/android.app.NativeActivity`]);},stop() {trySh(adb, ['shell', 'am', 'force-stop', APP_ID]);@@ -177,6 +221,115 @@ function androidDriver(opts) {screenshot() {return execFileSync(adb, ['exec-out', 'screencap', '-p'], { maxBuffer: 64 * 1024 * 1024 });},+ /**+ * What the run cost on THIS platform version, read while the app is still+ * up — every number here is gone the moment the process is.+ *+ * A field this version cannot answer for is `null`, never 0: the whole+ * point is comparing one Android release against another, and a zero+ * would read as "free" rather than "not measurable here". `notes` says+ * which ones those were, so a gap in the table is explained rather than+ * looking like a regression.+ */+ async metrics(launchOutput) {+ const notes = [];+ const pid = trySh(adb, ['shell', 'pidof', APP_ID]).stdout.trim().split(/\s+/)[0] || null;++ const startup = {+ // TotalTime is the activity reaching drawable; WaitTime adds the+ // system's own work before it got there.+ totalMs: firstNumber(/^TotalTime:\s*(\d+)/m, launchOutput),+ waitMs: firstNumber(/^WaitTime:\s*(\d+)/m, launchOutput),+ // The tag is ActivityTaskManager from API 29 and ActivityManager+ // below it, so both are asked for rather than branching on version.+ displayedMs: launchDuration(+ (trySh(adb, ['logcat', '-d', '-s', 'ActivityTaskManager:I', 'ActivityManager:I']).stdout ?? '')+ .split('\n').filter((l) => l.includes('Displayed') && l.includes(APP_ID)).pop()),+ readyMs: null, // the caller fills this from the boot record+ };++ if (!pid) {+ notes.push('the process was gone before anything could be sampled');+ return { startup, memory: null, cpu: null, frames: null, notes };+ }++ const meminfo = trySh(adb, ['shell', 'dumpsys', 'meminfo', APP_ID]).stdout ?? '';+ const memRow = (label) => firstNumber(new RegExp(`^\\s*${label}\\s+(\\d+)`, 'm'), meminfo);+ const memory = {+ // "TOTAL PSS:" is the App Summary line on API 29+; older releases+ // label the same figure "TOTAL" in the table above it.+ totalPssKb: firstNumber(/^\s*TOTAL PSS:\s*(\d+)/m, meminfo) ?? memRow('TOTAL'),+ nativeHeapKb: memRow('Native Heap'),+ // Dawn's Vulkan allocations land here, not in Native Heap — which is+ // why a renderer regression is invisible in the heap figure alone.+ graphicsKb: memRow('Graphics'),+ vmHwmKb: firstNumber(/^VmHWM:\s*(\d+)/m,+ trySh(adb, ['shell', 'cat', `/proc/${pid}/status`]).stdout),+ };+ if (memory.totalPssKb === null) notes.push('dumpsys meminfo named no total this version parses');++ const ticksPerSec = Number(trySh(adb, ['shell', 'getconf', 'CLK_TCK']).stdout.trim()) || 100;+ const before = cpuTicks(adb, pid);+ const threads = firstNumber(/^Threads:\s*(\d+)/m,+ trySh(adb, ['shell', 'cat', `/proc/${pid}/status`]).stdout);+ let cpu = null;+ if (before === null) {+ notes.push('/proc is not readable for another uid on this version — no CPU figure');+ } else {+ const windowMs = 2000;+ await sleep(windowMs);+ const after = cpuTicks(adb, pid);+ cpu = after === null ? null : {+ percent: Math.round(((after - before) / ticksPerSec) / (windowMs / 1000) * 1000) / 10,+ threads,+ windowMs,+ };+ }++ // SurfaceFlinger, not `dumpsys gfxinfo`: gfxinfo reports HWUI, and a+ // NativeActivity drawing to its own ANativeWindow through Vulkan never+ // touches HWUI, so gfxinfo's framestats for this app are empty.+ let frames = null;+ const layer = (trySh(adb, ['shell', 'dumpsys', 'SurfaceFlinger', '--list']).stdout ?? '')+ .split('\n').map((l) => l.trim()).filter((l) => l.includes(APP_ID)).pop();+ if (!layer) {+ notes.push('SurfaceFlinger listed no layer for the app — no frame timing');+ } else {+ const raw = trySh(adb, ['shell', 'dumpsys', 'SurfaceFlinger', '--latency', `'${layer}'`]).stdout ?? '';+ const lines = raw.trim().split('\n');+ const refreshNs = Number(lines[0]);+ // Three timestamps a row; the middle one is actual present. A pending+ // frame is reported as INT64_MAX and a dropped one as 0 — both are+ // "no measurement", not a zero-length frame.+ const present = lines.slice(1)+ .map((l) => Number(l.trim().split(/\s+/)[1]))+ .filter((n) => Number.isFinite(n) && n > 0 && n < 9.2e18);+ const gaps = present.slice(1).map((n, i) => (n - present[i]) / 1e6).filter((ms) => ms > 0);+ gaps.sort((a, b) => a - b);+ frames = gaps.length < 2 ? null : {+ count: gaps.length + 1,+ medianMs: Math.round(gaps[Math.floor(gaps.length / 2)] * 100) / 100,+ p95Ms: Math.round(gaps[Math.floor(gaps.length * 0.95)] * 100) / 100,+ refreshHz: Number.isFinite(refreshNs) && refreshNs > 0+ ? Math.round(1e9 / refreshNs) : null,+ };+ if (!frames) notes.push('SurfaceFlinger had too few presented frames to time');+ }++ // Recorded rather than judged. On a runner this is SwiftShader — a CPU+ // rasteriser — so the frame times above are a software renderer's, and+ // GPU utilisation and power do not exist to be read at all. Comparing+ // them across Android versions is still meaningful; reading them as+ // device performance is not.+ const renderer = /GLES:\s*(.+)/.exec(+ trySh(adb, ['shell', 'dumpsys', 'SurfaceFlinger']).stdout ?? '')?.[1]?.trim() ?? null;++ return {+ api: Number(trySh(adb, ['shell', 'getprop', 'ro.build.version.sdk']).stdout.trim()) || null,+ release: trySh(adb, ['shell', 'getprop', 'ro.build.version.release']).stdout.trim() || null,+ startup, memory, cpu, frames, renderer, notes,+ };+ },diagnostics() {return [['logcat (EstellaSDK)', trySh(adb, ['logcat', '-d', '-s', 'EstellaSDK:*']).stdout],@@ -294,11 +447,12 @@ async function verifyApp(driver, artifact, label, opts, judgeFrame = true) {let log = '';let offScreen = null;+ let launchOutput = '';// One retry, because a system dialog stealing focus is the emulator having a// bad minute, not the game being broken — and a gate that reds on that gets// ignored within a week.for (let attempt = 0; attempt < 2; attempt++) {- driver.launch();+ launchOutput = driver.launch() ?? '';const deadline = Date.now() + opts.timeout * 1000;while (Date.now() < deadline) {log = driver.readLog();@@ -318,24 +472,33 @@ async function verifyApp(driver, artifact, label, opts, judgeFrame = true) {const shot = path.join(opts.out, `${driver.name}-${label}.png`);writeFileSync(shot, frame);writeFileSync(path.join(opts.out, `${driver.name}-${label}.log`), log);++ // Before stop(): a dead process has no memory, no threads and no layer.+ const metrics = driver.metrics ? await driver.metrics(launchOutput) : null;driver.stop();const image = decodePng(frame);const colors = distinctColors(image);const ready = log.includes(READY);const readyLine = log.split('\n').find((l) => l.includes(READY))?.trim() ?? '';+ if (metrics) metrics.startup.readyMs = firstNumber(/ready in (\d+) ms/, readyLine);// The record already carries structured errors, and they say far more than a// pixel count can: drawing-demo drew a black screen because a resize sent// es_onNativeVisibility into infinite recursion, and the record said so.const errors = log.split('\n').filter((l) => l.startsWith('ERROR ['));+ // `--no-frame-judge` turns the pixel question off entirely: a compatibility+ // run compares one Android version against another, and the frame is there to+ // be LOOKED AT, not counted. Counting colors would red the run on a scene that+ // is legitimately dark and still say nothing about what a reviewer can see.+ const countColors = judgeFrame && opts.frameJudge !== false;const why = !ready ? 'never reported ready': offScreen ? `the game was not on screen — ${offScreen}`: errors.length ? errors[0].trim()- : (judgeFrame && colors < opts.minColors) ? `the frame is ${colors} flat color` : '';+ : (countColors && colors < opts.minColors) ? `the frame is ${colors} flat color` : '';return {- ok: !why, ready, colors, readyLine, errors, offScreen,+ ok: !why, ready, colors, readyLine, errors, offScreen, metrics,size: `${image.width}x${image.height}`, log, shot, why,};}@@ -421,10 +584,25 @@ if (!opts.examples) {console.error(`✗ no ${driver.name} app to verify (pass --${driver.artifactFlag} <path>)`);process.exit(2);}- const r = await verifyApp(driver, artifact, 'app', opts);+ const r = await verifyApp(driver, artifact, opts.label, opts);console.log(`boot record: ${r.ready ? r.readyLine : 'never reached "ready in"'}`);- console.log(`frame: ${r.size}, ${r.colors} distinct colors (need ${opts.minColors})`);+ console.log(`frame: ${r.size}${opts.frameJudge === false ? '' : `, ${r.colors} distinct colors (need ${opts.minColors})`}`);console.log(`written: ${r.shot}`);+ if (r.metrics) console.log(`metrics:\n${JSON.stringify(r.metrics, null, 2)}`);++ // Written whether or not the launch worked. A version that crashes is the+ // result a compatibility matrix most needs to report, and a run that files+ // nothing leaves a hole in the table that reads like the job never ran.+ if (opts.metricsOut) {+ mkdirSync(path.dirname(opts.metricsOut), { recursive: true });+ writeFileSync(opts.metricsOut, `${JSON.stringify({+ label: opts.label, device, ok: r.ok, why: r.why, ready: r.ready,+ readyLine: r.readyLine, errors: r.errors, shot: path.basename(r.shot),+ ...r.metrics,+ }, null, 2)}\n`);+ console.log(`metrics written: ${opts.metricsOut}`);+ }+if (r.ok) {console.log(`\n✓ ${driver.name}: the packaged game started and drew a frame`);process.exit(0);