Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

fix(ci): one answer to "the GPU died", and a driven launch that says why

esengine3b014e197971
diff --git a/tools/lib/deadGpu.mjs b/tools/lib/deadGpu.mjs
new file mode 100644
index 000000000..4a7fb72ea
--- /dev/null
+++ b/tools/lib/deadGpu.mjs
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file deadGpu.mjs — telling "the GPU never came up" from "the game is wrong".
+ *
+ * A runner whose GPU process dies during init leaves a context-lost renderer
+ * that cannot draw anything. That is not a verdict about the game: reporting it
+ * as one says the frame was wrong when there was no frame. Observed on the FIRST
+ * electron launch of a job, in run after run.
+ *
+ * One definition, because both pixel runners launch electron on the same runners
+ * and a second copy would drift — verify-render had this and verify-golden did
+ * not, so the same minute-old GPU failure was a retry in one and a red game in
+ * the other.
+ */
+
+/** Did the renderer never get a GPU, whatever it then said about pixels? */
+export function gpuNeverCameUp(output) {
+ return /Exiting GPU process due to errors during initialization/.test(output)
+ || (/GPU device lost/.test(output) && /Failed to create framebuffer/.test(output))
+ || /WebGL2 is not available/.test(output);
+}
+
+/**
+ * Run `attempt` and, if the GPU never came up, run it once more.
+ *
+ * `attempt` returns `{ ok, output }`. The retry is announced through `note` —
+ * a measurement taken twice has to say so, or a flaky runner reads as a stable
+ * one.
+ */
+export function retryOnDeadGpu(attempt, note) {
+ const first = attempt();
+ if (first.ok || !gpuNeverCameUp(first.output)) return { ...first, retried: false };
+ note();
+ return { ...attempt(), retried: true };
+}
diff --git a/tools/verify-golden.mjs b/tools/verify-golden.mjs
index 5d363e64f..572e81b65 100644
--- a/tools/verify-golden.mjs
+++ b/tools/verify-golden.mjs
@@ -21,6 +21,7 @@ import { mkdirSync, rmSync, readFileSync, existsSync } from 'node:fs';
import path from 'node:path';
import { atTier, projectDir, parityFor, interactFor, suspendFor, safeAreaFor, atlasFor, ROOT } from './goldenProjects.mjs';
import { frameDistance, frameCellMax, readPNG } from './frameCompare.mjs';
+import { retryOnDeadGpu } from './lib/deadGpu.mjs';
const argv = process.argv.slice(2);
const flag = (n, d) => {
@@ -108,11 +109,10 @@ function captureEditorFrame(id, out) {
/** Where a packaged run says the named entities are, or null if it never said. */
function probePositions(target, dir, w, h, names, extra = []) {
- const r = spawnSync('npx', [
- 'electron', path.join('scripts', LAUNCHER(target)),
+ const r = launchPackage('probe', target, [
'--dir', dir, '--w', String(w), '--h', String(h),
'--probe', names.join(','), ...extra,
- ], { encoding: 'utf8', cwd: DESKTOP });
+ ]);
const line = (r.stdout || '').split('\n').find((l) => l.includes('probe:'));
if (!line) return null;
try {
@@ -120,6 +120,23 @@ function probePositions(target, dir, w, h, names, extra = []) {
} catch { return null; }
}
+/**
+ * Launch a package, once more if the GPU never came up. Every launch here goes
+ * through this: the runner kills the first electron of a job often enough that a
+ * per-call-site answer means some launches retry and others report a broken game.
+ */
+function launchPackage(id, target, args) {
+ const run = retryOnDeadGpu(
+ () => {
+ const r = spawnSync('npx', ['electron', path.join('scripts', LAUNCHER(target)), ...args],
+ { encoding: 'utf8', cwd: DESKTOP });
+ return { ok: r.status === 0, output: `${r.stdout ?? ''}${r.stderr ?? ''}`, r };
+ },
+ () => console.log(`↻ ${id} ${target} — the GPU process died before it drew; launching again`),
+ );
+ return run.r;
+}
+
const results = [];
for (const { id, target } of pairs) {
const out = path.join(WORK, `${id}-${target}`);
@@ -172,11 +189,10 @@ for (const { id, target } of pairs) {
}
const packagePng = SHOTS ? path.join(SHOTS, `${id}-${target}.png`) : path.join(WORK, `${id}-${target}.png`);
- const launch = spawnSync('npx', [
- 'electron', path.join('scripts', LAUNCHER(target)),
+ const launch = launchPackage(id, target, [
'--dir', out, '--out', packagePng,
...(editor ? ['--w', String(editor.w), '--h', String(editor.h)] : []),
- ], { encoding: 'utf8', cwd: DESKTOP });
+ ]);
const line = (launch.stdout || '').split('\n').find((l) => l.startsWith('✓') || l.startsWith('✗')) ?? '';
if (launch.status !== 0) {
@@ -247,16 +263,20 @@ for (const { id, target } of pairs) {
let allAnswered = true;
for (const gesture of gestures) {
const drivenPng = path.join(WORK, `${id}-${target}-driven-${gesture.what.replace(/\W+/g, '-')}.png`);
- const drive = spawnSync('npx', [
- 'electron', path.join('scripts', LAUNCHER(target)),
+ const drive = launchPackage(id, target, [
'--dir', out, '--out', drivenPng,
'--w', String(editor.w), '--h', String(editor.h),
...(gesture.touch ? ['--touch'] : []),
'--input', JSON.stringify(gesture.spec),
- ], { encoding: 'utf8', cwd: DESKTOP });
+ ]);
if (drive.status !== 0) {
results.push({ id, target, stage: 'interact', ok: false, why: `the driven launch failed (${gesture.what})` });
console.log(`✗ ${id} ${target} — the driven launch failed for ${gesture.what}`);
+ // The tail, as the undriven launch already prints: "it failed" with no
+ // reason is a red nobody can act on without re-running it by hand.
+ for (const l of `${drive.stdout ?? ''}${drive.stderr ?? ''}`.split('\n').slice(-8)) {
+ if (l.trim()) console.log(` ${l}`);
+ }
allAnswered = false;
continue;
}
diff --git a/tools/verify-render.mjs b/tools/verify-render.mjs
index 3420fd93a..520414ece 100644
--- a/tools/verify-render.mjs
+++ b/tools/verify-render.mjs
@@ -15,6 +15,7 @@ import { spawnSync } from 'node:child_process';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { TIERS, SCENES, scenesAtTier } from './renderScenes.mjs';
+import { gpuNeverCameUp } from './lib/deadGpu.mjs';
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const DESKTOP = path.join(ROOT, 'desktop');
@@ -77,19 +78,6 @@ if (!NO_BUILD) {
}
}
-/**
- * Did the GPU never come up? A runner whose GPU process dies during init leaves
- * a context-lost renderer that cannot draw anything, which is NOT a pixel
- * verdict — reporting it as one says the scene drew the wrong thing when it drew
- * nothing at all. Observed on the FIRST launch of a job, three runs out of three.
- */
-function gpuNeverCameUp(out, scene) {
- // A scene that TAKES the GPU away on purpose owns its own device-loss result.
- if (scene.env.ESTELLA_VERIFY_DEVICE_LOSS) return false;
- return /Exiting GPU process due to errors during initialization/.test(out)
- || (/GPU device lost/.test(out) && /Failed to create framebuffer/.test(out));
-}
-
function runScene(scene) {
const args = XVFB
? ['-a', 'pnpm', 'exec', 'electron', 'scripts/headless-verify.mjs']
@@ -111,7 +99,8 @@ const failed = [];
let retried = 0;
for (const scene of scenes) {
let run = runScene(scene);
- if (!run.ok && gpuNeverCameUp(run.out, scene)) {
+ // A scene that TAKES the GPU away on purpose owns its own device-loss result.
+ if (!run.ok && !scene.env.ESTELLA_VERIFY_DEVICE_LOSS && gpuNeverCameUp(run.out)) {
retried++;
console.log(`↻ ${scene.id.padEnd(28)} the GPU process died before it drew; measuring again`);
run = runScene(scene);