A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
* @file verify-render.mjs — run the pixel gates a tier pays for.
*
* One runner over {@link renderScenes.mjs}, so CI and a developer run the same
* list by the same names. Reports the whole matrix rather than stopping at the
* first failure: which scenes broke is the useful answer, not which broke first.
*
* node tools/verify-render.mjs --tier pr
* node tools/verify-render.mjs --tier pr --backend webgpu --budget 45
* node tools/verify-render.mjs --tier nightly --only spine,video
*/
import path from 'node:path';
import { existsSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { TIERS, SCENES, scenesAtTier, sceneWatchdogMs } from './renderScenes.mjs';
import { retryOnDeadGpu, deadGpuVerdict, launchNeverHappenedVerdict, resultMeasured } from './lib/deadGpu.mjs';
import { runTool } from './lib/runTool.mjs';
import { runElectron } from './lib/electronRun.mjs';
import { requireCurrentEngine } from './lib/engineBuild.mjs';
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const DESKTOP = path.join(ROOT, 'desktop');
const argv = process.argv.slice(2);
const flag = (n, d) => {
const i = argv.indexOf(`--${n}`);
return i >= 0 && argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[i + 1] : d;
};
const TIER = flag('tier', 'pr');
// Which host's scenes to run. The engine's gates stand on their own, so a
// checkout without the editor submodule runs `--host engine` and is not quietly
// short a hundred checks — it says which list it took.
const HOST_FILTER = flag('host', '');
const ONLY = flag('only', '');
const BACKEND = flag('backend', 'webgl2');
const BUDGET = flag('budget', '');
const NO_BUILD = argv.includes('--no-build');
if (argv.includes('--no-xvfb')) process.env.ESTELLA_NO_XVFB = '1';
const only = ONLY ? new Set(ONLY.split(',').map((s) => s.trim())) : null;
if (only) {
const known = new Set(SCENES.map((s) => s.id));
const bogus = [...only].filter((id) => !known.has(id));
if (bogus.length) {
console.error(`verify-render: no such scene(s): ${bogus.join(', ')}`);
process.exit(2);
}
}
if (!TIERS.includes(TIER)) {
console.error(`verify-render: unknown tier "${TIER}" (have: ${TIERS.join(', ')})`);
process.exit(2);
}
if (HOST_FILTER && !['engine', 'editor'].includes(HOST_FILTER)) {
console.error(`verify-render: unknown host "${HOST_FILTER}" (have: engine, editor)`);
process.exit(2);
}
if (!['webgl2', 'webgpu'].includes(BACKEND)) {
console.error(`verify-render: unknown backend "${BACKEND}" (have: webgl2, webgpu)`);
process.exit(2);
}
if (BUDGET && !(Number(BUDGET) > 0)) {
console.error(`verify-render: --budget takes minutes, not "${BUDGET}"`);
process.exit(2);
}
// Minutes the whole run may spend before it stops and says what it never
// measured. A tier that outgrows its runner is a report either way; without one
// the report is a killed process, which names no scene at all.
const BUDGET_MS = Number(BUDGET || 0) * 60_000;
const watchdogFor = (host) => sceneWatchdogMs(BACKEND, host);
// The scene's own watchdog produces a verdict, so this kill is for the launch
// that never reaches it — a whole electron start-up beyond.
const LAUNCH_MARGIN_MS = 30_000;
// A scene runs on the second backend only where it says so: the two backends
// agree on most frames and deliberately differ on a couple of tolerances, and
// guessing that per scene is how the WebGPU list became a third copy.
// A tier run on the second backend takes the scenes that DECLARE it; naming
// scenes explicitly is a decision already made, so `--only` is not filtered —
// that is how a scene earns the declaration in the first place.
// A Linux runner has no GPU, so WebGL2 there is a software rasteriser, and a scene that says `softwareGpu: false` takes its GPU process
// down before it draws, every launch. That says nothing about the scene and spends a budget the other scenes need. Named below, and
// still run wherever there is a GPU, the second backend's Metal runner included.
const softwareGpu = process.platform === 'linux' && Boolean(process.env.CI);
const tierScenes = scenesAtTier(TIER);
const needsGpu = BACKEND === 'webgl2' && softwareGpu ? tierScenes.filter((s) => s.softwareGpu === false) : [];
const selected = only
? SCENES.filter((s) => only.has(s.id))
: tierScenes.filter((s) => (BACKEND === 'webgpu' ? Boolean(s.webgpu) : !needsGpu.includes(s)));
if (!only && needsGpu.length > 0) {
console.log(`not run on this machine's software rasteriser: ${needsGpu.map((s) => s.id).join(', ')}`);
}
const scenes = selected.map((s) => ({
id: s.id,
// Which host answers this scene's driving surface. The engine's is the
// default; a scene names the editor's only when its SUBJECT is an editor door.
host: s.host ?? 'engine',
env: BACKEND === 'webgpu'
? {
...s.env,
...(typeof s.webgpu === 'object' ? s.webgpu : {}),
ESTELLA_VERIFY_BACKEND: 'webgpu',
// The adapter Dawn picks for itself: a real driver is the only kind that
// answers the portability question. `swiftshader` names the bundled software
// one, for a machine with no GPU. The boot log says which it got.
ESTELLA_VERIFY_WEBGPU_ADAPTER: process.env.ESTELLA_VERIFY_WEBGPU_ADAPTER ?? '',
}
: s.env,
}));
const hosted = HOST_FILTER ? scenes.filter((s) => s.host === HOST_FILTER) : scenes;
if (HOST_FILTER) {
const dropped = scenes.length - hosted.length;
if (dropped) console.log(` not on this host: ${dropped} scene(s)`);
scenes.length = 0;
scenes.push(...hosted);
}
const hostsUsed = new Set(scenes.map((s) => s.host));
console.log(`render ${only ? `${scenes.length} named scene(s)` : `${TIER}: ${scenes.length} scene(s)`} on ${BACKEND}`
+ ` (host: ${[...hostsUsed].sort().join(' + ')})`);
/**
* A host with no page serves 404s, the driving surface never appears, and every
* scene fails with "cannot read properties of undefined". Say which build is
* missing instead. `git submodule deinit` empties the editor's dist among other
* ignored output, so this is reachable without anyone deleting anything.
*/
function requirePage(page, host, build) {
if (existsSync(page)) return;
console.error(`verify-render: the ${host} host has no page at ${path.relative(ROOT, page)}.`);
console.error(` build it: ${build}`);
process.exit(2);
}
// Build each host this run needs, then check the wasm it will actually SERVE:
// --no-build leaves a host's output as it stands, and a freshly built public/
// copy it never staged is evidence about a binary no scene below is judging.
if (hostsUsed.has('engine')) {
if (!NO_BUILD) {
const built = runTool('node', ['tools/render-host/build.mjs'], { cwd: ROOT, encoding: 'utf8' });
if (built.status !== 0) {
console.error(`verify-render: render-host build failed\n${(built.stderr || built.stdout || '').slice(-800)}`);
process.exit(1);
}
}
requireCurrentEngine(ROOT, path.join(ROOT, 'build', 'render-host', 'wasm'));
requirePage(path.join(ROOT, 'build', 'render-host', 'index.html'), 'engine',
'node tools/render-host/build.mjs');
}
if (hostsUsed.has('editor')) {
if (!existsSync(DESKTOP)) {
console.error('verify-render: these scenes name the editor host, which is not checked out'
+ ' (desktop/ is an optional submodule).');
process.exit(2);
}
if (!NO_BUILD) {
const built = runTool('pnpm', ['exec', 'vite', 'build'], { cwd: DESKTOP, encoding: 'utf8' });
if (built.status !== 0) {
console.error(`verify-render: vite build failed\n${(built.stderr || built.stdout || '').slice(-800)}`);
process.exit(1);
}
}
requireCurrentEngine(ROOT, path.join(DESKTOP, NO_BUILD ? 'dist' : 'public', 'wasm'));
requirePage(path.join(DESKTOP, 'dist', 'headless.html'), 'editor',
'pnpm --filter @estella/editor exec vite build');
}
const seconds = (ms) => `${(ms / 1000).toFixed(1)}s`;
function runScene(scene) {
const RUNNER = 'tools/render-host/run.mjs';
const watchdog = watchdogFor(scene.host);
const at = Date.now();
const r = runElectron([RUNNER], {
cwd: ROOT,
encoding: 'utf8',
timeout: watchdog + LAUNCH_MARGIN_MS,
env: {
ESTELLA_VERIFY_TIMEOUT_MS: String(watchdog),
ESTELLA_VERIFY_HOST: scene.host,
...scene.env,
},
});
const out = `${r.stdout || ''}${r.stderr || ''}`;
// DRIVE_RESULT is the run's own verdict: printed once the page loaded, the
// engine booted and the driver finished. Whether it is there answers "did a
// measurement happen" far better than anything in the log text can.
const drive = /^DRIVE_RESULT (\{.*\})$/m.exec(out);
let result = null;
try {
result = drive ? JSON.parse(drive[1]) : null;
} catch {
result = null; // truncated line — treat as no verdict
}
return {
ok: r.status === 0,
out,
ms: Date.now() - at,
measured: resultMeasured(result),
drew: result?.capture?.rendered,
verdict: r.timedOut
? 'the run never reached its own watchdog and was killed'
: ((r.stdout || '').match(/\[verify:render\][^\n]*/) ?? [''])[0].replace('[verify:render] ', ''),
};
}
// Published assets a scene draws (tools/corpora.mjs), fetched and imported before
// any scene runs: a download inside a scene's watchdog would be timed as rendering.
const corporaNeeded = [...new Set(selected.flatMap((s) => s.corpora ?? []))];
if (corporaNeeded.length) {
const { importedCorpus } = await import('./corpora.mjs');
for (const id of corporaNeeded) await importedCorpus(id);
}
const failed = [];
const unmeasured = [];
const startedAt = Date.now();
let retried = 0;
// A runner with no GPU for this backend fails every scene the same way, six
// attempts apiece — an hour to say nothing, with the rest blamed on the budget.
// Once this many have died with it down and none has drawn, no scene can change it.
const DEAD_GPU_GIVE_UP = 3;
let deadInARow = 0;
let drewSomething = false;
let noGpu = false;
for (const [index, scene] of scenes.entries()) {
const outOfBudget = BUDGET_MS && Date.now() - startedAt >= BUDGET_MS;
noGpu = !drewSomething && deadInARow >= DEAD_GPU_GIVE_UP;
if (outOfBudget || noGpu) {
for (const rest of scenes.slice(index)) unmeasured.push(rest.id);
break;
}
// A scene that TAKES the GPU away on purpose owns its own device-loss result;
// everything else shares one retry policy with the golden runner, or the same
// dead runner is a retry in one and a broken engine in the other.
let run;
if (scene.env.ESTELLA_VERIFY_DEVICE_LOSS) {
run = runScene(scene);
} else {
// `gpuNeverCameUp` matches a message this runner prints on success too, so
// every failure reads as "the GPU died" and the real reason never reaches the
// log — a host failing half its launches looks exactly like a flaky runner.
let firstFailure = null;
const attempt = retryOnDeadGpu(
() => { const r = runScene(scene); return { ...r, output: r.out }; },
(died) => {
retried++;
console.log(`↻ ${scene.id.padEnd(28)} ${died
? 'the GPU process died before it drew'
: 'a blank frame on the run after a GPU death'}; measuring again`);
},
);
if (attempt.retried && !firstFailure) {
firstFailure = (attempt.out || '').split('\n')
.filter((l) => /error|fail|refus|throw|undefined|cannot|timeout|watchdog/i.test(l))
.slice(-3);
if (firstFailure.length) {
console.log(` └ what the retried run(s) said: ${firstFailure.join(' | ').slice(0, 400)}`);
}
}
run = attempt;
}
if (run.ok) drewSomething = true; else failed.push(scene.id);
deadInARow = run.gpuDied ? deadInARow + 1 : 0;
const verdict = run.launchFailed ? launchNeverHappenedVerdict('the scene')
: run.gpuDied ? deadGpuVerdict('the scene')
: run.verdict;
console.log(`${run.ok ? '✓' : '✗'} ${scene.id.padEnd(28)} ${seconds(run.ms).padStart(7)} ${verdict}`);
if (!run.ok) {
for (const l of run.out.split('\n').slice(-8)) if (l.trim()) console.log(` ${l}`);
}
}
const measured = scenes.length - unmeasured.length;
console.log(`\nrender ${TIER}: ${measured - failed.length}/${measured} scene(s) drew what they claim`
+ ` in ${seconds(Date.now() - startedAt)}`
+ (retried ? `, ${retried} measured twice (the GPU had died)` : ''));
if (failed.length) for (const id of failed) console.log(` ✗ ${id}`);
if (unmeasured.length) {
console.log(noGpu
? `\nthe runner never gave ${BACKEND} a GPU: ${measured} scene(s) died with it down and `
+ `${unmeasured.length} were not run — this is a verdict about the RUNNER, not the engine`
: `\nthe ${BUDGET} minute budget ran out: ${unmeasured.length} scene(s) never ran`);
for (const id of unmeasured) console.log(` · ${id}`);
}
if (failed.length || unmeasured.length) process.exit(1);