Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

fix(verify): a launch that never happened is not a GPU that died

esengine20e31b148a0d
diff --git a/tools/lib/deadGpu.mjs b/tools/lib/deadGpu.mjs
index 13c506155..18fe93fe8 100644
--- a/tools/lib/deadGpu.mjs
+++ b/tools/lib/deadGpu.mjs
@@ -36,6 +36,22 @@ export function gpuNeverCameUp(output) {
|| /WebGL2 is not available/.test(output);
}
+/**
+ * Electron never started: no display, a sandbox helper that is not setuid root,
+ * a binary that is not there. Not the GPU, and not worth six retries — a nightly
+ * spent 24 minutes on eleven projects that never launched, and called it a dead GPU.
+ */
+export function launchNeverHappened(output) {
+ return /did not start:|Missing X server or \$DISPLAY|chrome-sandbox is owned by root|signal SIGTRAP/
+ .test(output);
+}
+
+/** What to report when the process never started. Shared for the same reason the
+ * detection is: the runners each named this differently, and none named it. */
+export function launchNeverHappenedVerdict(subject) {
+ return `electron never started on this machine — this says nothing about ${subject}`;
+}
+
/**
* What to report when a measurement never happened because the GPU never came up.
* Shared for the same reason the detection is: the two runners described the same
@@ -116,6 +132,9 @@ export function retryOnDeadGpu(attempt, note, stepMs = STEP_MS) {
for (let i = 0; i < MAX_ATTEMPTS; i++) {
last = attempt();
if (last.ok) return { ...last, retried: i > 0 };
+ // A launch that never happened will not happen on the sixth try either,
+ // and every retry buries the reason further under identical output.
+ if (launchNeverHappened(last.output ?? '')) return { ...last, retried: i > 0, launchFailed: true };
const noVerdict = reachedNoVerdict(last);
if (noVerdict) sawOutage = true;
diff --git a/tools/tests/dead-gpu.test.ts b/tools/tests/dead-gpu.test.ts
index e73f86c2c..a320b6b28 100644
--- a/tools/tests/dead-gpu.test.ts
+++ b/tools/tests/dead-gpu.test.ts
@@ -10,7 +10,8 @@
*/
import { describe, it, expect } from 'vitest';
// @ts-expect-error — a .mjs tool module, typed by its own JSDoc
-import { retryOnDeadGpu, gpuNeverCameUp, deadGpuVerdict, engineCouldNotDraw, resultMeasured } from '../lib/deadGpu.mjs';
+import { retryOnDeadGpu, gpuNeverCameUp, deadGpuVerdict, engineCouldNotDraw, resultMeasured,
+ launchNeverHappened } from '../lib/deadGpu.mjs';
const DEAD = 'Exiting GPU process due to errors during initialization';
const BLANK = 'painted=true live=false errors=0';
@@ -228,4 +229,34 @@ describe('retryOnDeadGpu', () => {
expect(r.attempts).toBe(6);
expect(r.gpuDied).toBe(true);
});
+
+ // golden reports measured:false when a launch fails, which is exactly what a
+ // dead GPU looks like — so a nightly with no xvfb retried eleven projects
+ // six times each and spent 24 minutes calling it a GPU that died.
+ it('does not retry a launch that never happened', () => {
+ const r = runReporting([{ ok: false, measured: false, output: 'pnpm did not start: ENOENT' }]);
+ expect(r.attempts).toBe(1);
+ expect(r.launchFailed).toBe(true);
+ expect(r.gpuDied).toBeFalsy();
+ });
+
+ it('still retries a dead GPU, which is not a launch that never happened', () => {
+ const r = run([DEAD]);
+ expect(r.launchFailed).toBeFalsy();
+ expect(r.gpuDied).toBe(true);
+ });
+});
+
+describe('launchNeverHappened', () => {
+ it('recognises the ways electron does not start', () => {
+ expect(launchNeverHappened('npx did not start: ENOENT')).toBe(true);
+ expect(launchNeverHappened('Missing X server or $DISPLAY')).toBe(true);
+ expect(launchNeverHappened('chrome-sandbox is owned by root and has mode 4755')).toBe(true);
+ expect(launchNeverHappened('electron exited with signal SIGTRAP')).toBe(true);
+ });
+
+ it('is not what a dead GPU or a blank frame looks like', () => {
+ expect(launchNeverHappened(DEAD)).toBe(false);
+ expect(launchNeverHappened(BLANK)).toBe(false);
+ });
});
diff --git a/tools/verify-aot-parity.mjs b/tools/verify-aot-parity.mjs
index 18fc1cccf..a4850a598 100644
--- a/tools/verify-aot-parity.mjs
+++ b/tools/verify-aot-parity.mjs
@@ -25,7 +25,7 @@ import { mkdirSync, rmSync, readFileSync, existsSync, readdirSync, statSync } fr
import path from 'node:path';
import { GOLDEN, projectDir, parityFor, launchTimeoutFor, ROOT } from './goldenProjects.mjs';
import { frameDistance } from './frameCompare.mjs';
-import { retryOnDeadGpu, deadGpuVerdict } from './lib/deadGpu.mjs';
+import { retryOnDeadGpu, deadGpuVerdict, launchNeverHappenedVerdict } from './lib/deadGpu.mjs';
import { runElectron } from './lib/electronRun.mjs';
import { requireCurrentEngine } from './lib/engineBuild.mjs';
@@ -117,7 +117,7 @@ function launch(id, out, png, timeoutMs) {
},
() => console.log(`↻ ${id} — the GPU process died before it drew; launching again`),
);
- return { ...run.r, gpuDied: Boolean(run.gpuDied) };
+ return { ...run.r, gpuDied: Boolean(run.gpuDied), launchFailed: Boolean(run.launchFailed) };
}
const results = [];
@@ -163,9 +163,11 @@ for (const g of subjects) {
const r = launch(id, dirs[kind], png, timeoutMs);
const line = (r.stdout || '').split('\n').find((l) => l.startsWith('✓') || l.startsWith('✗')) ?? '';
if (r.status !== 0 || !existsSync(png)) {
- const why = r.gpuDied
- ? `${deadGpuVerdict(`the ${kind} package`)} (${line || 'no frame'})`
- : line || 'launch failed';
+ const why = r.launchFailed
+ ? `${launchNeverHappenedVerdict(`the ${kind} package`)} (${(r.stderr || '').trim().slice(-200)})`
+ : r.gpuDied
+ ? `${deadGpuVerdict(`the ${kind} package`)} (${line || 'no frame'})`
+ : line || 'launch failed';
results.push({ id, stage: `launch-${kind}`, ok: false, why });
console.log(`✗ ${id} ${kind} — ${why}`);
broke = true;
diff --git a/tools/verify-golden.mjs b/tools/verify-golden.mjs
index 459818e46..90f0543b8 100644
--- a/tools/verify-golden.mjs
+++ b/tools/verify-golden.mjs
@@ -21,7 +21,7 @@ import { mkdirSync, rmSync, readFileSync, existsSync } from 'node:fs';
import path from 'node:path';
import { atTier, projectDir, parityFor, interactFor, audioFor, suspendFor, safeAreaFor, atlasFor, webPixels, launchTimeoutFor, ROOT } from './goldenProjects.mjs';
import { frameDistance, frameCellMax, readPNG } from './frameCompare.mjs';
-import { retryOnDeadGpu, deadGpuVerdict } from './lib/deadGpu.mjs';
+import { retryOnDeadGpu, deadGpuVerdict, launchNeverHappenedVerdict } from './lib/deadGpu.mjs';
import { runElectron } from './lib/electronRun.mjs';
import { requireCurrentEngine } from './lib/engineBuild.mjs';
@@ -139,9 +139,11 @@ function captureEditorFrame(id, out) {
if (!run.ok) {
return {
ok: false,
- why: run.gpuDied
- ? `${deadGpuVerdict('the editor\'s frame')} (${run.output.trim().slice(-200)})`
- : run.output.trim().slice(-300),
+ why: run.launchFailed
+ ? `${launchNeverHappenedVerdict('the editor\'s frame')} (${run.output.trim().slice(-200)})`
+ : run.gpuDied
+ ? `${deadGpuVerdict('the editor\'s frame')} (${run.output.trim().slice(-200)})`
+ : run.output.trim().slice(-300),
};
}
return { ok: true, w: run.w, h: run.h };
@@ -210,7 +212,7 @@ function launchPackage(id, target, args) {
// gpuDied travels with the result: without it a runner whose GPU never came up
// is reported as a game that draws nothing, which is the one confusion this
// whole retry exists to prevent.
- return { ...run.r, gpuDied: Boolean(run.gpuDied) };
+ return { ...run.r, gpuDied: Boolean(run.gpuDied), launchFailed: Boolean(run.launchFailed) };
}
const fmtMB = (bytes) => `${(bytes / 1024 / 1024).toFixed(2)}MB`;
@@ -334,9 +336,11 @@ for (const { id, target } of pairs) {
const line = (launch.stdout || '').split('\n').find((l) => l.startsWith('✓') || l.startsWith('✗')) ?? '';
if (launch.status !== 0) {
- const why = launch.gpuDied
- ? `${deadGpuVerdict('the game')} (${line || 'no frame'})`
- : line || 'launch failed';
+ const why = launch.launchFailed
+ ? `${launchNeverHappenedVerdict('the game')} (${(launch.stderr || '').trim().slice(-200)})`
+ : launch.gpuDied
+ ? `${deadGpuVerdict('the game')} (${line || 'no frame'})`
+ : line || 'launch failed';
results.push({ id, target, stage: 'launch', ok: false, why });
console.log(`✗ ${id} ${target} — ${why}`);
// Deep enough to carry the launcher's own diagnosis of a package that never
diff --git a/tools/verify-render.mjs b/tools/verify-render.mjs
index 954b7dda8..45eaac622 100644
--- a/tools/verify-render.mjs
+++ b/tools/verify-render.mjs
@@ -15,7 +15,7 @@ import path from 'node:path';
import { existsSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { TIERS, SCENES, scenesAtTier, sceneWatchdogMs } from './renderScenes.mjs';
-import { retryOnDeadGpu, deadGpuVerdict, resultMeasured } from './lib/deadGpu.mjs';
+import { retryOnDeadGpu, deadGpuVerdict, launchNeverHappenedVerdict, resultMeasured } from './lib/deadGpu.mjs';
import { runTool } from './lib/runTool.mjs';
import { runElectron } from './lib/electronRun.mjs';
import { requireCurrentEngine } from './lib/engineBuild.mjs';
@@ -246,8 +246,10 @@ for (const [index, scene] of scenes.entries()) {
}
if (run.ok) drewSomething = true; else failed.push(scene.id);
deadInARow = run.gpuDied ? deadInARow + 1 : 0;
- console.log(`${run.ok ? '✓' : '✗'} ${scene.id.padEnd(28)} ${seconds(run.ms).padStart(7)} `
- + `${run.gpuDied ? deadGpuVerdict('the scene') : run.verdict}`);
+ const verdict = run.launchFailed ? launchNeverHappenedVerdict('the scene')
+ : run.gpuDied ? deadGpuVerdict('the scene')
+ : run.verdict;
+ console.log(`${run.ok ? '✓' : '✗'} ${scene.id.padEnd(28)} ${seconds(run.ms).padStart(7)} ${verdict}`);
if (!run.ok) {
for (const l of run.out.split('\n').slice(-8)) if (l.trim()) console.log(` ${l}`);
}