A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
fix(verify): a launch that never happened is not a GPU that died
esengine20e31b148a0d
diff --git a/tools/lib/deadGpu.mjs b/tools/lib/deadGpu.mjsindex 13c506155..18fe93fe8 100644--- a/tools/lib/deadGpu.mjs+++ b/tools/lib/deadGpu.mjs@@ -36,6 +36,22 @@ export function gpuNeverCameUp(output) {|| /WebGL2 is not available/.test(output);}+/**+ * Electron never started: no display, a sandbox helper that is not setuid root,+ * a binary that is not there. Not the GPU, and not worth six retries — a nightly+ * spent 24 minutes on eleven projects that never launched, and called it a dead GPU.+ */+export function launchNeverHappened(output) {+ return /did not start:|Missing X server or \$DISPLAY|chrome-sandbox is owned by root|signal SIGTRAP/+ .test(output);+}++/** What to report when the process never started. Shared for the same reason the+ * detection is: the runners each named this differently, and none named it. */+export function launchNeverHappenedVerdict(subject) {+ return `electron never started on this machine — this says nothing about ${subject}`;+}+/*** What to report when a measurement never happened because the GPU never came up.* Shared for the same reason the detection is: the two runners described the same@@ -116,6 +132,9 @@ export function retryOnDeadGpu(attempt, note, stepMs = STEP_MS) {for (let i = 0; i < MAX_ATTEMPTS; i++) {last = attempt();if (last.ok) return { ...last, retried: i > 0 };+ // A launch that never happened will not happen on the sixth try either,+ // and every retry buries the reason further under identical output.+ if (launchNeverHappened(last.output ?? '')) return { ...last, retried: i > 0, launchFailed: true };const noVerdict = reachedNoVerdict(last);if (noVerdict) sawOutage = true;diff --git a/tools/tests/dead-gpu.test.ts b/tools/tests/dead-gpu.test.tsindex e73f86c2c..a320b6b28 100644--- a/tools/tests/dead-gpu.test.ts+++ b/tools/tests/dead-gpu.test.ts@@ -10,7 +10,8 @@*/import { describe, it, expect } from 'vitest';// @ts-expect-error — a .mjs tool module, typed by its own JSDoc-import { retryOnDeadGpu, gpuNeverCameUp, deadGpuVerdict, engineCouldNotDraw, resultMeasured } from '../lib/deadGpu.mjs';+import { retryOnDeadGpu, gpuNeverCameUp, deadGpuVerdict, engineCouldNotDraw, resultMeasured,+ launchNeverHappened } from '../lib/deadGpu.mjs';const DEAD = 'Exiting GPU process due to errors during initialization';const BLANK = 'painted=true live=false errors=0';@@ -228,4 +229,34 @@ describe('retryOnDeadGpu', () => {expect(r.attempts).toBe(6);expect(r.gpuDied).toBe(true);});++ // golden reports measured:false when a launch fails, which is exactly what a+ // dead GPU looks like — so a nightly with no xvfb retried eleven projects+ // six times each and spent 24 minutes calling it a GPU that died.+ it('does not retry a launch that never happened', () => {+ const r = runReporting([{ ok: false, measured: false, output: 'pnpm did not start: ENOENT' }]);+ expect(r.attempts).toBe(1);+ expect(r.launchFailed).toBe(true);+ expect(r.gpuDied).toBeFalsy();+ });++ it('still retries a dead GPU, which is not a launch that never happened', () => {+ const r = run([DEAD]);+ expect(r.launchFailed).toBeFalsy();+ expect(r.gpuDied).toBe(true);+ });+});++describe('launchNeverHappened', () => {+ it('recognises the ways electron does not start', () => {+ expect(launchNeverHappened('npx did not start: ENOENT')).toBe(true);+ expect(launchNeverHappened('Missing X server or $DISPLAY')).toBe(true);+ expect(launchNeverHappened('chrome-sandbox is owned by root and has mode 4755')).toBe(true);+ expect(launchNeverHappened('electron exited with signal SIGTRAP')).toBe(true);+ });++ it('is not what a dead GPU or a blank frame looks like', () => {+ expect(launchNeverHappened(DEAD)).toBe(false);+ expect(launchNeverHappened(BLANK)).toBe(false);+ });});diff --git a/tools/verify-aot-parity.mjs b/tools/verify-aot-parity.mjsindex 18fc1cccf..a4850a598 100644--- a/tools/verify-aot-parity.mjs+++ b/tools/verify-aot-parity.mjs@@ -25,7 +25,7 @@ import { mkdirSync, rmSync, readFileSync, existsSync, readdirSync, statSync } frimport path from 'node:path';import { GOLDEN, projectDir, parityFor, launchTimeoutFor, ROOT } from './goldenProjects.mjs';import { frameDistance } from './frameCompare.mjs';-import { retryOnDeadGpu, deadGpuVerdict } from './lib/deadGpu.mjs';+import { retryOnDeadGpu, deadGpuVerdict, launchNeverHappenedVerdict } from './lib/deadGpu.mjs';import { runElectron } from './lib/electronRun.mjs';import { requireCurrentEngine } from './lib/engineBuild.mjs';@@ -117,7 +117,7 @@ function launch(id, out, png, timeoutMs) {},() => console.log(`↻ ${id} — the GPU process died before it drew; launching again`),);- return { ...run.r, gpuDied: Boolean(run.gpuDied) };+ return { ...run.r, gpuDied: Boolean(run.gpuDied), launchFailed: Boolean(run.launchFailed) };}const results = [];@@ -163,9 +163,11 @@ for (const g of subjects) {const r = launch(id, dirs[kind], png, timeoutMs);const line = (r.stdout || '').split('\n').find((l) => l.startsWith('✓') || l.startsWith('✗')) ?? '';if (r.status !== 0 || !existsSync(png)) {- const why = r.gpuDied- ? `${deadGpuVerdict(`the ${kind} package`)} (${line || 'no frame'})`- : line || 'launch failed';+ const why = r.launchFailed+ ? `${launchNeverHappenedVerdict(`the ${kind} package`)} (${(r.stderr || '').trim().slice(-200)})`+ : r.gpuDied+ ? `${deadGpuVerdict(`the ${kind} package`)} (${line || 'no frame'})`+ : line || 'launch failed';results.push({ id, stage: `launch-${kind}`, ok: false, why });console.log(`✗ ${id} ${kind} — ${why}`);broke = true;diff --git a/tools/verify-golden.mjs b/tools/verify-golden.mjsindex 459818e46..90f0543b8 100644--- a/tools/verify-golden.mjs+++ b/tools/verify-golden.mjs@@ -21,7 +21,7 @@ import { mkdirSync, rmSync, readFileSync, existsSync } from 'node:fs';import path from 'node:path';import { atTier, projectDir, parityFor, interactFor, audioFor, suspendFor, safeAreaFor, atlasFor, webPixels, launchTimeoutFor, ROOT } from './goldenProjects.mjs';import { frameDistance, frameCellMax, readPNG } from './frameCompare.mjs';-import { retryOnDeadGpu, deadGpuVerdict } from './lib/deadGpu.mjs';+import { retryOnDeadGpu, deadGpuVerdict, launchNeverHappenedVerdict } from './lib/deadGpu.mjs';import { runElectron } from './lib/electronRun.mjs';import { requireCurrentEngine } from './lib/engineBuild.mjs';@@ -139,9 +139,11 @@ function captureEditorFrame(id, out) {if (!run.ok) {return {ok: false,- why: run.gpuDied- ? `${deadGpuVerdict('the editor\'s frame')} (${run.output.trim().slice(-200)})`- : run.output.trim().slice(-300),+ why: run.launchFailed+ ? `${launchNeverHappenedVerdict('the editor\'s frame')} (${run.output.trim().slice(-200)})`+ : run.gpuDied+ ? `${deadGpuVerdict('the editor\'s frame')} (${run.output.trim().slice(-200)})`+ : run.output.trim().slice(-300),};}return { ok: true, w: run.w, h: run.h };@@ -210,7 +212,7 @@ function launchPackage(id, target, args) {// gpuDied travels with the result: without it a runner whose GPU never came up// is reported as a game that draws nothing, which is the one confusion this// whole retry exists to prevent.- return { ...run.r, gpuDied: Boolean(run.gpuDied) };+ return { ...run.r, gpuDied: Boolean(run.gpuDied), launchFailed: Boolean(run.launchFailed) };}const fmtMB = (bytes) => `${(bytes / 1024 / 1024).toFixed(2)}MB`;@@ -334,9 +336,11 @@ for (const { id, target } of pairs) {const line = (launch.stdout || '').split('\n').find((l) => l.startsWith('✓') || l.startsWith('✗')) ?? '';if (launch.status !== 0) {- const why = launch.gpuDied- ? `${deadGpuVerdict('the game')} (${line || 'no frame'})`- : line || 'launch failed';+ const why = launch.launchFailed+ ? `${launchNeverHappenedVerdict('the game')} (${(launch.stderr || '').trim().slice(-200)})`+ : launch.gpuDied+ ? `${deadGpuVerdict('the game')} (${line || 'no frame'})`+ : line || 'launch failed';results.push({ id, target, stage: 'launch', ok: false, why });console.log(`✗ ${id} ${target} — ${why}`);// Deep enough to carry the launcher's own diagnosis of a package that neverdiff --git a/tools/verify-render.mjs b/tools/verify-render.mjsindex 954b7dda8..45eaac622 100644--- a/tools/verify-render.mjs+++ b/tools/verify-render.mjs@@ -15,7 +15,7 @@ import path from 'node:path';import { existsSync } from 'node:fs';import { fileURLToPath } from 'node:url';import { TIERS, SCENES, scenesAtTier, sceneWatchdogMs } from './renderScenes.mjs';-import { retryOnDeadGpu, deadGpuVerdict, resultMeasured } from './lib/deadGpu.mjs';+import { retryOnDeadGpu, deadGpuVerdict, launchNeverHappenedVerdict, resultMeasured } from './lib/deadGpu.mjs';import { runTool } from './lib/runTool.mjs';import { runElectron } from './lib/electronRun.mjs';import { requireCurrentEngine } from './lib/engineBuild.mjs';@@ -246,8 +246,10 @@ for (const [index, scene] of scenes.entries()) {}if (run.ok) drewSomething = true; else failed.push(scene.id);deadInARow = run.gpuDied ? deadInARow + 1 : 0;- console.log(`${run.ok ? '✓' : '✗'} ${scene.id.padEnd(28)} ${seconds(run.ms).padStart(7)} `- + `${run.gpuDied ? deadGpuVerdict('the scene') : run.verdict}`);+ const verdict = run.launchFailed ? launchNeverHappenedVerdict('the scene')+ : run.gpuDied ? deadGpuVerdict('the scene')+ : run.verdict;+ console.log(`${run.ok ? '✓' : '✗'} ${scene.id.padEnd(28)} ${seconds(run.ms).padStart(7)} ${verdict}`);if (!run.ok) {for (const l of run.out.split('\n').slice(-8)) if (l.trim()) console.log(` ${l}`);}