A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
* @file deadGpu.mjs — telling "the GPU never came up" from "the subject is wrong".
*
* A runner whose GPU process dies during init leaves a context-lost renderer that
* cannot draw anything. That is not a verdict about the subject: reporting it as
* one says the frame was wrong when there was no frame.
*
* The question is whether a MEASUREMENT HAPPENED, and the run itself is the only
* honest answer to that — `measured` below. Log text is not: this repo's Linux
* runner prints "Exiting GPU process due to errors during initialization" on runs
* that go on to succeed, so a policy keyed on that string reads every failure as
* a dead GPU, retries all six times, and buries the real reason. One job spent
* fourteen minutes and 71 retries that way, and the scene that actually broke was
* never named.
*
* One definition, because both pixel runners launch electron on the same runners
* and a second copy would drift — verify-render had this and verify-golden did
* not, so the same minute-old GPU failure was a retry in one and a red game in
* the other.
*/
/**
* Log-text evidence that a runner never got a GPU. UNRELIABLE ALONE — see the
* file header — so this is the fallback for an attempt that cannot say whether it
* reached a verdict, never the primary answer.
*/
export function gpuNeverCameUp(output) {
return /Exiting GPU process due to errors during initialization/.test(output)
// A lost device, then resources it can no longer make — textures as well as
// framebuffers. The runner's outage said "Failed to create texture from
// spec", which only the line above caught, and that one fires on every launch.
|| (/GPU device lost/.test(output)
&& /Failed to create (framebuffer|texture)|createTexture failed/.test(output))
|| /WebGL2 is not available/.test(output);
}
/**
* Evidence a GPU really failed, strong enough to blame it for a run that never
* reached a verdict: the engine could not make what it draws through, or WebGL2
* was refused. {@link gpuNeverCameUp} is not: its first line prints on every
* launch on the Linux runner, and every failure there read as a dead GPU.
*/
export function gpuFailed(output) {
return engineCouldNotDraw(output) || /WebGL2 is not available/.test(output);
}
/** Lines every launch on a GPU-less runner prints, whatever happens next. */
const NOISE = /Exiting GPU process due to errors during initialization|dbus\/|GPU stall due to ReadPixels|Fontconfig|GL Driver Message/;
/**
* The lines of a failed attempt that say what happened, for a log that is read on
* a runner nobody can open: its errors, timeouts and signals, noise removed, last
* few kept. An attempt that printed none says so rather than printing nothing.
*/
export function failureLines(output, max = 8) {
const lines = String(output ?? '').split('\n').map((l) => l.trim())
.filter((l) => l && !NOISE.test(l)
&& /error|fail|timed out|timeout|exception|fatal|signal|crash|lost|not available|unreachable|refused/i.test(l));
return lines.length ? lines.slice(-max) : ['(the attempt printed no error line)'];
}
/**
* Electron never started: no display, a sandbox helper that is not setuid root,
* a binary that is not there. Not the GPU, and not worth six retries — a nightly
* spent 24 minutes on eleven projects that never launched, and called it a dead GPU.
*/
export function launchNeverHappened(output) {
return /did not start:|Missing X server or \$DISPLAY|chrome-sandbox is owned by root|signal SIGTRAP/
.test(output);
}
/** What to report when the process never started. Shared for the same reason the
* detection is: the runners each named this differently, and none named it. */
export function launchNeverHappenedVerdict(subject) {
return `electron never started on this machine — this says nothing about ${subject}`;
}
/**
* What to report when a measurement never happened because the GPU never came up.
* Shared for the same reason the detection is: the two runners described the same
* failure differently, and one of them described it as the game's fault.
*/
export function deadGpuVerdict(subject) {
return `the GPU process never came up, on every attempt — this says nothing about ${subject}`;
}
/** Attempts one measurement is allowed, counting the first. Six, not four: four
* was measured when the first project needed three retries to draw, and a tier
* with one more pair in it spent them all on a game that was fine. */
const MAX_ATTEMPTS = 6;
/**
* Wait before relaunching, growing with each try. The retries were immediate,
* which relaunches into a GPU process still on its way down — three attempts
* inside a second are one attempt with extra steps. Synchronous because the
* launches are; `stepMs` is 0 in tests, which have no process to wait for.
*/
function backoff(attempt, stepMs) {
if (stepMs <= 0) return;
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, stepMs * (attempt + 1));
}
/**
* Measured on the runner: a GPU that goes takes about two minutes to come back,
* and six attempts two seconds apart spanned ninety seconds of it — every retry
* spent inside the same dead window. Five seconds a step covers it.
*/
const STEP_MS = 5000;
/**
* When this process last saw an outage, in any chain. A blank frame inside the
* two minutes a GPU takes to come back is aftermath even when the death was
* another launch's: golden captures the editor, then launches the package, and
* the package's blank frame was judged in a chain that had seen nothing.
*/
let lastOutageAt = 0;
const OUTAGE_SETTLES_MS = 2 * 60 * 1000;
/**
* The ENGINE saying it could not make what it draws through. A run that gets this
* far still prints its result — reporting an empty frame — so "it finished" is not
* the same as "it rendered", and only the latter is a verdict about the scene.
*/
export function engineCouldNotDraw(output) {
// A device the engine reports LOST draws nothing after, whether or not it
// went on to fail making anything: the flat frame a launcher judges after
// that line is the loss, not the game — input-actions was red for one.
return /Framebuffer is incomplete|Failed to create framebuffer|Failed to create texture|createTexture failed|GPU device lost/
.test(output);
}
/**
* Whether a driver's structured result is a MEASUREMENT or the harness saying it
* could not take one: an error with no capture beside it is the latter, since a
* verdict about the subject always carries the frame it judged. Here, not in a
* runner, for the reason the header gives.
*/
export function resultMeasured(result) {
if (result === null || result === undefined) return false;
return !(result.ok === false && Boolean(result.error) && !result.capture);
}
/**
* The runner's own words for a GPU that DID come up. OUTRANKS every heuristic
* here, which guess a cause from a failure where this reads the cause itself:
* an emptied readback is `ok:false` with no capture, which {@link resultMeasured}
* calls "no measurement" and the policy would then blame on a dead GPU.
*/
export function gpuCameUp(output) {
return /webgpu adapter:\s*\S/i.test(output) && /WebGPU device injected/i.test(output);
}
/**
* The device arrived and then went. Still an outage rather than the subject's
* fault, so it survives the veto above — "came up" is not "stayed up".
*/
function gpuWentAway(output) {
return /GPU device lost/.test(output);
}
/**
* Whether this attempt reached a verdict at all. `measured` is the run's own
* answer and wins, except that a run whose engine could not draw has measured
* nothing whatever else it managed to print.
*/
function reachedNoVerdict(last) {
const output = last.output ?? '';
// A GPU that demonstrably arrived and did not go makes every later failure
// the subject's own — including a capture that came back empty, which is a
// statement ABOUT the frame and not the absence of one to judge.
if (gpuCameUp(output) && !gpuWentAway(output)) return false;
if (engineCouldNotDraw(output)) return true;
if (typeof last.measured === 'boolean') return !last.measured;
return gpuNeverCameUp(output);
}
/**
* Run `attempt` ({ ok, output, measured?, drew? }) until it succeeds or fails in a way
* that is the subject's own; `note(noVerdict, last)` announces each retry. A verdict
* that was measured is never retried, and a run that gives up is `gpuDied` only on
* {@link gpuFailed}'s evidence.
*/
export function retryOnDeadGpu(attempt, note, stepMs = STEP_MS) {
// A chain that waits for nothing (tests) has no runner to remember across.
const remembers = stepMs > 0;
let sawOutage = remembers && Date.now() - lastOutageAt < OUTAGE_SETTLES_MS;
let last;
for (let i = 0; i < MAX_ATTEMPTS; i++) {
last = attempt();
if (last.ok) return { ...last, retried: i > 0 };
// A launch that never happened will not happen on the sixth try either,
// and every retry buries the reason further under identical output.
if (launchNeverHappened(last.output ?? '')) return { ...last, retried: i > 0, launchFailed: true };
const noVerdict = reachedNoVerdict(last);
if (noVerdict) {
sawOutage = true;
if (remembers) lastOutageAt = Date.now();
}
// `drew === true` is the only thing that rules out aftermath: an attempt
// that cannot say keeps the older, more cautious behaviour.
const aftermath = sawOutage && last.drew !== true;
// It measured, and what it measured was wrong. Retrying that is how a
// broken subject gets six chances to look like a flaky runner.
if (!noVerdict && !aftermath) return { ...last, retried: i > 0 };
if (i < MAX_ATTEMPTS - 1) {
note(noVerdict, last);
backoff(i, stepMs);
}
}
return { ...last, retried: true, gpuDied: gpuFailed(last.output ?? '') };
}