A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
* @file dead-gpu.test.ts — when a measurement counts and when it is retaken.
*
* The property that matters is the one a retry policy can destroy: a game that
* genuinely draws nothing must still fail, on the first attempt, with no retry to
* hide it. Everything else here is about not reporting a runner's dead GPU as a
* verdict on the game.
*/
import { describe, it, expect } from 'vitest';
// @ts-expect-error — a .mjs tool module, typed by its own JSDoc
import { retryOnDeadGpu, gpuNeverCameUp, deadGpuVerdict, engineCouldNotDraw, resultMeasured, failureLines,
launchNeverHappened, gpuCameUp } from '../lib/deadGpu.mjs';
const DEAD = 'Exiting GPU process due to errors during initialization';
const BLANK = 'painted=true live=false errors=0';
/** What the runner prints when the device DID arrive, on the two lines above the
* failure. Verbatim from the run that exposed the frame-lifecycle regression. */
const GPU_UP = '[renderer] [engine] webgpu adapter: apple metal-3 (default)\n'
+ "[renderer] [wasm] [INFO ] WebGPU device injected for '#canvas' (640x360)";
/** The regression's own shape: a device that came up, and a readback that did not. */
const EMPTY_CAPTURE = `${GPU_UP}\ncaptureViewportPixels: no pixels — the engine refused the readback`;
/** Drive the policy over a scripted sequence of attempt outputs; `null` = success. */
function run(outputs: Array<string | null>) {
let attempts = 0;
const notes: boolean[] = [];
const result = retryOnDeadGpu(
() => {
const out = outputs[Math.min(attempts++, outputs.length - 1)];
return { ok: out === null, output: out ?? '' };
},
(died: boolean) => notes.push(died),
0, // no backoff: there is no GPU process to wait for here
);
return { ...result, attempts, notes };
}
/** The same, for runs that report whether they reached a verdict. */
type Attempt = { ok: boolean; output?: string; measured?: boolean; drew?: boolean };
function runReporting(attempts_: Attempt[]) {
let attempts = 0;
const notes: boolean[] = [];
const result = retryOnDeadGpu(
() => ({ output: '', ...attempts_[Math.min(attempts++, attempts_.length - 1)] }),
(noVerdict: boolean) => notes.push(noVerdict),
0,
);
return { ...result, attempts, notes };
}
describe('failureLines', () => {
it('keeps what happened and drops what every launch prints', () => {
const out = `${DEAD}\n[6065:ERROR:dbus/bus.cc:405] Failed to connect to the bus\n`
+ '[screenshot] waitFor timed out: play realm ready\nok frame 3\nelectron exited with signal SIGTRAP';
expect(failureLines(out)).toEqual([
'[screenshot] waitFor timed out: play realm ready',
'electron exited with signal SIGTRAP',
]);
});
it('says so when an attempt printed no error at all', () => {
expect(failureLines(`${DEAD}\nall quiet`)).toEqual(['(the attempt printed no error line)']);
});
});
describe('gpuNeverCameUp', () => {
it('recognises the three ways a runner says it has no GPU', () => {
expect(gpuNeverCameUp(DEAD)).toBe(true);
expect(gpuNeverCameUp('GPU device lost\nFailed to create framebuffer')).toBe(true);
expect(gpuNeverCameUp('WebGL2 is not available')).toBe(true);
});
it('does not read a verdict about pixels as one about the GPU', () => {
expect(gpuNeverCameUp(BLANK)).toBe(false);
// Device loss alone is a scene some gates drive on purpose; only paired
// with a framebuffer that could not be made is it a runner with no GPU.
expect(gpuNeverCameUp('GPU device lost')).toBe(false);
});
});
describe('resultMeasured', () => {
it('reads a result with a capture as a measurement, whatever its verdict', () => {
expect(resultMeasured({ ok: true, capture: { rendered: true } })).toBe(true);
// The property the whole policy rests on: a scene that drew the wrong
// pixels HAS measured, and must not be handed a retry to hide behind.
expect(resultMeasured({ ok: false, capture: { rendered: true }, expect: { ok: false } })).toBe(true);
});
it('reads an error with no capture as the harness failing to measure', () => {
// What a lost GL context leaves: the readback throws, so there is no frame
// to judge and nothing about the scene has been established.
expect(resultMeasured({ ok: false, error: 'captureViewportPixels: no pixels' })).toBe(false);
});
it('has nothing to read when the driver printed no result at all', () => {
expect(resultMeasured(null)).toBe(false);
expect(resultMeasured(undefined)).toBe(false);
});
it('takes a capture over an error when a result carries both', () => {
// A run that captured and then tripped over something else still measured.
expect(resultMeasured({ ok: false, error: 'late failure', capture: { rendered: true } })).toBe(true);
});
});
describe('gpuCameUp', () => {
it('reads the two lines that say a device arrived', () => {
expect(gpuCameUp(GPU_UP)).toBe(true);
expect(gpuCameUp(EMPTY_CAPTURE)).toBe(true);
});
it('an adapter alone is not a device', () => {
expect(gpuCameUp('[renderer] [engine] webgpu adapter: apple metal-3 (default)')).toBe(false);
});
it('says nothing about a runner that printed neither', () => {
expect(gpuCameUp(DEAD)).toBe(false);
expect(gpuCameUp(BLANK)).toBe(false);
});
});
describe('retryOnDeadGpu', () => {
it('an empty capture from a device that CAME UP is the subject\u2019s failure', () => {
// `measured: false` is the half that makes this invisible: an error with
// no capture is what resultMeasured calls "no measurement", and without
// the veto the policy retries six times and blames the runner.
const r = runReporting([{ ok: false, output: EMPTY_CAPTURE, measured: false }]);
expect(r.ok).toBe(false);
expect(r.attempts).toBe(1);
expect(r.notes).toEqual([]);
expect(r.gpuDied).toBeUndefined();
});
it('still retries when the device that came up was then LOST', () => {
// "Came up" is not "stayed up": a device that goes mid-run took the
// measurement with it, and that is the runner's story after all.
const r = runReporting([
{ ok: false, output: `${GPU_UP}\nGPU device lost`, measured: false },
{ ok: true },
]);
expect(r.ok).toBe(true);
expect(r.attempts).toBe(2);
expect(r.notes).toEqual([true]);
});
it('a game that draws nothing fails on the first attempt', () => {
// The whole point: no retry may stand between a black frame and a red gate.
const r = run([BLANK]);
expect(r.ok).toBe(false);
expect(r.attempts).toBe(1);
expect(r.notes).toEqual([]);
});
it('takes one measurement when the GPU is up', () => {
const r = run([null]);
expect(r.ok).toBe(true);
expect(r.attempts).toBe(1);
expect(r.retried).toBe(false);
});
it('measures again when the GPU died before it drew', () => {
const r = run([DEAD, null]);
expect(r.ok).toBe(true);
expect(r.attempts).toBe(2);
expect(r.notes).toEqual([true]);
});
it('measures again for a blank frame that FOLLOWS a death in the chain', () => {
// The case that made this necessary: a GPU process that dies and restarts
// leaves the next launch on a context-lost renderer, which paints a blank
// frame and prints nothing about why.
const r = run([DEAD, BLANK, null]);
expect(r.ok).toBe(true);
expect(r.attempts).toBe(3);
expect(r.notes).toEqual([true, false]);
});
it('reads a lost device that then fails to make TEXTURES', () => {
// Verbatim from the runner, the day the second clause was found to only
// know about framebuffers.
const real = [
'[ERROR] GPU device lost: context-lost [backend=WebGL2, vendor=Google Inc.]',
'[ERROR] Texture::initialize: createTexture failed for 70x70',
'[ERROR] Failed to create texture from spec',
].join('\n');
expect(gpuNeverCameUp(real)).toBe(true);
});
it('words the give-up once, for both runners', () => {
// The two of them each had their own sentence for it, and one of them
// blamed the game — the failure this file exists to keep apart.
expect(deadGpuVerdict('the game')).toContain('says nothing about the game');
expect(deadGpuVerdict('the scene')).toContain('never came up');
});
// The runner this policy runs on prints the dead-GPU line on runs that then
// succeed, so text alone made every failure look like an outage: one job
// spent 71 retries and never named the scene that broke.
it('a run that reached a verdict is not retried, whatever the log says', () => {
const r = runReporting([{ ok: false, output: DEAD, measured: true, drew: true }]);
expect(r.ok).toBe(false);
expect(r.attempts).toBe(1);
expect(r.notes).toEqual([]);
});
it('retries a run that reached no verdict, whatever the log says', () => {
// Nothing in this output mentions a GPU; the run simply never got far
// enough to judge anything, which is not a verdict about the subject.
const r = runReporting([
{ ok: false, output: 'launch timed out', measured: false },
{ ok: true },
]);
expect(r.ok).toBe(true);
expect(r.attempts).toBe(2);
expect(r.notes).toEqual([true]);
});
it('still retries a blank frame that follows a run with no verdict', () => {
const r = runReporting([
{ ok: false, measured: false },
{ ok: false, measured: true, drew: false },
{ ok: true },
]);
expect(r.ok).toBe(true);
expect(r.attempts).toBe(3);
expect(r.notes).toEqual([true, false]);
});
it('stops retrying once a run draws, even after an outage', () => {
// A drawn frame that fails its assertion is the subject's own failure;
// the outage is over and further attempts only hide it.
const r = runReporting([
{ ok: false, measured: false },
{ ok: false, measured: true, drew: true },
]);
expect(r.ok).toBe(false);
expect(r.attempts).toBe(2);
expect(r.gpuDied).toBeUndefined();
});
// The runner ran out of GPU resources and every scene came back empty. Each
// run still PRINTED its result, so 'it finished' called it a measurement and
// 46 of 70 scenes were reported as broken renderers in one job.
it('retries a blank frame within two minutes of an outage in ANOTHER chain', () => {
// golden captures the editor in one chain and launches the package in the
// next; a GPU that died under the first is still coming back under the
// second. A chain that waits (stepMs > 0) remembers; 1 ms keeps this quick.
retryOnDeadGpu(() => ({ ok: false, output: DEAD }), () => {}, 1);
let attempts = 0;
const r = retryOnDeadGpu(() => {
attempts++;
return attempts === 1 ? { ok: false, output: '', measured: true, drew: false } : { ok: true, output: '' };
}, () => {}, 1);
expect(r.ok).toBe(true);
expect(attempts).toBe(2);
});
it('a chain that waits for nothing keeps no memory of outages', () => {
// The tests above run in one process; a remembered outage would make a
// game that draws nothing pass through a retry it must never get.
run([DEAD]);
const r = runReporting([{ ok: false, measured: true, drew: false }]);
expect(r.attempts).toBe(1);
});
it('a lost device with nothing left to create is still no verdict', () => {
// Verbatim from the runner: the context went, the launcher judged the flat
// frame that followed as "drew nothing", and no texture failed because
// nothing was asked for after the loss.
const lost = '[ERROR] GPU device lost: context-lost [backend=WebGL2, vendor=Google Inc. (Google)]';
expect(engineCouldNotDraw(lost)).toBe(true);
const r = runReporting([{ ok: false, output: lost, measured: true, drew: false }, { ok: true }]);
expect(r.ok).toBe(true);
expect(r.attempts).toBe(2);
});
it('a run whose engine could not draw has measured nothing', () => {
const broke = 'Framebuffer is incomplete! (size: 640x480, GL error 0x37442)';
expect(engineCouldNotDraw(broke)).toBe(true);
expect(engineCouldNotDraw(BLANK)).toBe(false);
const r = runReporting([
{ ok: false, output: broke, measured: true, drew: false },
{ ok: true },
]);
expect(r.ok).toBe(true);
expect(r.attempts).toBe(2);
expect(r.notes).toEqual([true]);
});
it('retakes a measurement the harness could not take, and stops when it can', () => {
// The runner loses the first launch's GL context and holds it after that,
// which had every run's FIRST scene red while the hundred behind it passed.
let attempts = 0;
const r = retryOnDeadGpu(() => {
attempts++;
const result = attempts === 1
? { ok: false, error: 'captureViewportPixels: no pixels' }
: { ok: true, capture: { rendered: true } };
return { ok: result.ok, output: '', measured: resultMeasured(result), drew: result.capture?.rendered };
}, () => {}, 0);
expect(attempts).toBe(2);
expect(r.ok).toBe(true);
expect(r.retried).toBe(true);
});
it('does not retake a measurement that happened and disagreed', () => {
let attempts = 0;
const r = retryOnDeadGpu(() => {
attempts++;
const result = { ok: false, capture: { rendered: true }, expect: { ok: false } };
return { ok: false, output: '', measured: resultMeasured(result), drew: true };
}, () => {}, 0);
expect(attempts).toBe(1);
expect(r.gpuDied).toBeFalsy();
});
it('gives up at the cap without blaming the GPU on the line every launch prints', () => {
const r = run([DEAD]);
expect(r.ok).toBe(false);
expect(r.attempts).toBe(6);
expect(r.gpuDied).toBe(false);
});
it('blames the GPU at the cap when the engine says it could not draw', () => {
const r = run([`${DEAD}\n[ERROR] GPU device lost: context-lost [backend=WebGL2]\n[ERROR] Texture::initialize: createTexture failed for 70x70`]);
expect(r.attempts).toBe(6);
expect(r.gpuDied).toBe(true);
});
// golden reports measured:false when a launch fails, which is exactly what a
// dead GPU looks like — so a nightly with no xvfb retried eleven projects
// six times each and spent 24 minutes calling it a GPU that died.
it('hands each retry the attempt that failed', () => {
const seen: string[] = [];
let n = 0;
retryOnDeadGpu(() => ({ ok: false, measured: false, output: `attempt ${++n}` }),
(_noVerdict: boolean, last: { output: string }) => seen.push(last.output), 0);
expect(seen).toEqual(['attempt 1', 'attempt 2', 'attempt 3', 'attempt 4', 'attempt 5']);
});
it('does not retry a launch that never happened', () => {
const r = runReporting([{ ok: false, measured: false, output: 'pnpm did not start: ENOENT' }]);
expect(r.attempts).toBe(1);
expect(r.launchFailed).toBe(true);
expect(r.gpuDied).toBeFalsy();
});
it('still retries a run with no verdict, which is not a launch that never happened', () => {
const r = run([DEAD]);
expect(r.launchFailed).toBeFalsy();
expect(r.attempts).toBe(6);
});
});
describe('launchNeverHappened', () => {
it('recognises the ways electron does not start', () => {
expect(launchNeverHappened('npx did not start: ENOENT')).toBe(true);
expect(launchNeverHappened('Missing X server or $DISPLAY')).toBe(true);
expect(launchNeverHappened('chrome-sandbox is owned by root and has mode 4755')).toBe(true);
expect(launchNeverHappened('electron exited with signal SIGTRAP')).toBe(true);
});
it('is not what a dead GPU or a blank frame looks like', () => {
expect(launchNeverHappened(DEAD)).toBe(false);
expect(launchNeverHappened(BLANK)).toBe(false);
});
});