Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

fix(ci): a blank frame right after a dead GPU is not a verdict on the game

esengine1cd0871bfc6d
diff --git a/desktop/tests/dead-gpu.test.ts b/desktop/tests/dead-gpu.test.ts
new file mode 100644
index 000000000..1ca2626e1
--- /dev/null
+++ b/desktop/tests/dead-gpu.test.ts
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file dead-gpu.test.ts — when a measurement counts and when it is retaken.
+ *
+ * The property that matters is the one a retry policy can destroy: a game that
+ * genuinely draws nothing must still fail, on the first attempt, with no retry to
+ * hide it. Everything else here is about not reporting a runner's dead GPU as a
+ * verdict on the game.
+ */
+import { describe, it, expect } from 'vitest';
+// @ts-expect-error — a .mjs tool module, typed by its own JSDoc
+import { retryOnDeadGpu, gpuNeverCameUp } from '../../tools/lib/deadGpu.mjs';
+
+const DEAD = 'Exiting GPU process due to errors during initialization';
+const BLANK = 'painted=true live=false errors=0';
+
+/** Drive the policy over a scripted sequence of attempt outputs; `null` = success. */
+function run(outputs: Array<string | null>) {
+ let attempts = 0;
+ const notes: boolean[] = [];
+ const result = retryOnDeadGpu(
+ () => {
+ const out = outputs[Math.min(attempts++, outputs.length - 1)];
+ return { ok: out === null, output: out ?? '' };
+ },
+ (died: boolean) => notes.push(died),
+ );
+ return { ...result, attempts, notes };
+}
+
+describe('gpuNeverCameUp', () => {
+ it('recognises the three ways a runner says it has no GPU', () => {
+ expect(gpuNeverCameUp(DEAD)).toBe(true);
+ expect(gpuNeverCameUp('GPU device lost\nFailed to create framebuffer')).toBe(true);
+ expect(gpuNeverCameUp('WebGL2 is not available')).toBe(true);
+ });
+
+ it('does not read a verdict about pixels as one about the GPU', () => {
+ expect(gpuNeverCameUp(BLANK)).toBe(false);
+ // Device loss alone is a scene some gates drive on purpose; only paired
+ // with a framebuffer that could not be made is it a runner with no GPU.
+ expect(gpuNeverCameUp('GPU device lost')).toBe(false);
+ });
+});
+
+describe('retryOnDeadGpu', () => {
+ it('a game that draws nothing fails on the first attempt', () => {
+ // The whole point: no retry may stand between a black frame and a red gate.
+ const r = run([BLANK]);
+ expect(r.ok).toBe(false);
+ expect(r.attempts).toBe(1);
+ expect(r.notes).toEqual([]);
+ });
+
+ it('takes one measurement when the GPU is up', () => {
+ const r = run([null]);
+ expect(r.ok).toBe(true);
+ expect(r.attempts).toBe(1);
+ expect(r.retried).toBe(false);
+ });
+
+ it('measures again when the GPU died before it drew', () => {
+ const r = run([DEAD, null]);
+ expect(r.ok).toBe(true);
+ expect(r.attempts).toBe(2);
+ expect(r.notes).toEqual([true]);
+ });
+
+ it('measures again for a blank frame that FOLLOWS a death in the chain', () => {
+ // The case that made this necessary: a GPU process that dies and restarts
+ // leaves the next launch on a context-lost renderer, which paints a blank
+ // frame and prints nothing about why.
+ const r = run([DEAD, BLANK, null]);
+ expect(r.ok).toBe(true);
+ expect(r.attempts).toBe(3);
+ expect(r.notes).toEqual([true, false]);
+ });
+
+ it('gives up at the cap and says the GPU is why', () => {
+ const r = run([DEAD]);
+ expect(r.ok).toBe(false);
+ expect(r.attempts).toBe(3);
+ expect(r.gpuDied).toBe(true);
+ });
+});
diff --git a/tools/lib/deadGpu.mjs b/tools/lib/deadGpu.mjs
index 4a7fb72ea..2dfeb2c59 100644
--- a/tools/lib/deadGpu.mjs
+++ b/tools/lib/deadGpu.mjs
@@ -21,16 +21,28 @@ export function gpuNeverCameUp(output) {
|| /WebGL2 is not available/.test(output);
}
+/** Attempts one measurement is allowed, counting the first. */
+const MAX_ATTEMPTS = 3;
+
/**
- * Run `attempt` and, if the GPU never came up, run it once more.
+ * Run `attempt` (→ `{ ok, output }`) until it succeeds or fails with the GPU up.
*
- * `attempt` returns `{ ok, output }`. The retry is announced through `note` —
- * a measurement taken twice has to say so, or a flaky runner reads as a stable
- * one.
+ * A failure ANYWHERE after a death in this chain is inconclusive: a restarted GPU
+ * process leaves the next launch context-lost, painting a blank frame and printing
+ * nothing about why. A game that truly draws nothing still fails on the first
+ * attempt, since no attempt reports a death. `note(died)` announces each retry.
*/
export function retryOnDeadGpu(attempt, note) {
- const first = attempt();
- if (first.ok || !gpuNeverCameUp(first.output)) return { ...first, retried: false };
- note();
- return { ...attempt(), retried: true };
+ let sawDeath = false;
+ let last;
+ for (let i = 0; i < MAX_ATTEMPTS; i++) {
+ last = attempt();
+ if (last.ok) return { ...last, retried: i > 0 };
+ const died = gpuNeverCameUp(last.output);
+ sawDeath = sawDeath || died;
+ // Nothing in this chain blamed the GPU, so the failure is the game's.
+ if (!sawDeath) return { ...last, retried: i > 0 };
+ if (i < MAX_ATTEMPTS - 1) note(died);
+ }
+ return { ...last, retried: true, gpuDied: true };
}
diff --git a/tools/verify-golden.mjs b/tools/verify-golden.mjs
index d33abbcad..e9b74d502 100644
--- a/tools/verify-golden.mjs
+++ b/tools/verify-golden.mjs
@@ -132,7 +132,9 @@ function launchPackage(id, target, args) {
{ encoding: 'utf8', cwd: DESKTOP });
return { ok: r.status === 0, output: `${r.stdout ?? ''}${r.stderr ?? ''}`, r };
},
- () => console.log(`↻ ${id} ${target} — the GPU process died before it drew; launching again`),
+ (died) => console.log(`↻ ${id} ${target} — ${died
+ ? 'the GPU process died before it drew'
+ : 'a blank frame on the launch after a GPU death'}; launching again`),
);
return run.r;
}
diff --git a/tools/verify-render.mjs b/tools/verify-render.mjs
index 520414ece..7f6d4d525 100644
--- a/tools/verify-render.mjs
+++ b/tools/verify-render.mjs
@@ -15,7 +15,7 @@ import { spawnSync } from 'node:child_process';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { TIERS, SCENES, scenesAtTier } from './renderScenes.mjs';
-import { gpuNeverCameUp } from './lib/deadGpu.mjs';
+import { retryOnDeadGpu } from './lib/deadGpu.mjs';
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const DESKTOP = path.join(ROOT, 'desktop');
@@ -98,12 +98,23 @@ function runScene(scene) {
const failed = [];
let retried = 0;
for (const scene of scenes) {
- let run = runScene(scene);
- // A scene that TAKES the GPU away on purpose owns its own device-loss result.
- if (!run.ok && !scene.env.ESTELLA_VERIFY_DEVICE_LOSS && gpuNeverCameUp(run.out)) {
- retried++;
- console.log(`↻ ${scene.id.padEnd(28)} the GPU process died before it drew; measuring again`);
+ // A scene that TAKES the GPU away on purpose owns its own device-loss result;
+ // everything else shares one retry policy with the golden runner, or the same
+ // dead runner is a retry in one and a broken engine in the other.
+ let run;
+ if (scene.env.ESTELLA_VERIFY_DEVICE_LOSS) {
run = runScene(scene);
+ } else {
+ const attempt = retryOnDeadGpu(
+ () => { const r = runScene(scene); return { ...r, output: r.out }; },
+ (died) => {
+ retried++;
+ console.log(`↻ ${scene.id.padEnd(28)} ${died
+ ? 'the GPU process died before it drew'
+ : 'a blank frame on the run after a GPU death'}; measuring again`);
+ },
+ );
+ run = attempt;
}
if (!run.ok) failed.push(scene.id);
console.log(`${run.ok ? '✓' : '✗'} ${scene.id.padEnd(28)} ${run.verdict}`);