Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

fix(verify): a blank frame inside two minutes of any outage is aftermath, whichever chain saw it

yhh3eb0344d0a6b
diff --git a/tools/lib/deadGpu.mjs b/tools/lib/deadGpu.mjs
index 18fe93fe8..c971f3f48 100644
--- a/tools/lib/deadGpu.mjs
+++ b/tools/lib/deadGpu.mjs
@@ -84,6 +84,15 @@ function backoff(attempt, stepMs) {
*/
const STEP_MS = 5000;
+/**
+ * When this process last saw an outage, in any chain. A blank frame inside the
+ * two minutes a GPU takes to come back is aftermath even when the death was
+ * another launch's: golden captures the editor, then launches the package, and
+ * the package's blank frame was judged in a chain that had seen nothing.
+ */
+let lastOutageAt = 0;
+const OUTAGE_SETTLES_MS = 2 * 60 * 1000;
+
/**
* The ENGINE saying it could not make what it draws through. A run that gets this
* far still prints its result — reporting an empty frame — so "it finished" is not
@@ -127,7 +136,9 @@ function reachedNoVerdict(last) {
* screen? a blank frame right after an outage is that outage still settling).
*/
export function retryOnDeadGpu(attempt, note, stepMs = STEP_MS) {
- let sawOutage = false;
+ // A chain that waits for nothing (tests) has no runner to remember across.
+ const remembers = stepMs > 0;
+ let sawOutage = remembers && Date.now() - lastOutageAt < OUTAGE_SETTLES_MS;
let last;
for (let i = 0; i < MAX_ATTEMPTS; i++) {
last = attempt();
@@ -137,7 +148,10 @@ export function retryOnDeadGpu(attempt, note, stepMs = STEP_MS) {
if (launchNeverHappened(last.output ?? '')) return { ...last, retried: i > 0, launchFailed: true };
const noVerdict = reachedNoVerdict(last);
- if (noVerdict) sawOutage = true;
+ if (noVerdict) {
+ sawOutage = true;
+ if (remembers) lastOutageAt = Date.now();
+ }
// `drew === true` is the only thing that rules out aftermath: an attempt
// that cannot say keeps the older, more cautious behaviour.
const aftermath = sawOutage && last.drew !== true;
diff --git a/tools/tests/dead-gpu.test.ts b/tools/tests/dead-gpu.test.ts
index a320b6b28..0939f7496 100644
--- a/tools/tests/dead-gpu.test.ts
+++ b/tools/tests/dead-gpu.test.ts
@@ -183,6 +183,26 @@ describe('retryOnDeadGpu', () => {
// The runner ran out of GPU resources and every scene came back empty. Each
// run still PRINTED its result, so 'it finished' called it a measurement and
// 46 of 70 scenes were reported as broken renderers in one job.
+ it('retries a blank frame within two minutes of an outage in ANOTHER chain', () => {
+ // golden captures the editor in one chain and launches the package in the
+ // next; a GPU that died under the first is still coming back under the
+ // second. A chain that waits (stepMs > 0) remembers; 1 ms keeps this quick.
+ retryOnDeadGpu(() => ({ ok: false, output: DEAD }), () => {}, 1);
+ let attempts = 0;
+ const r = retryOnDeadGpu(() => {
+ attempts++;
+ return attempts === 1 ? { ok: false, output: '', measured: true, drew: false } : { ok: true, output: '' };
+ }, () => {}, 1);
+ expect(r.ok).toBe(true);
+ expect(attempts).toBe(2);
+ });
+ it('a chain that waits for nothing keeps no memory of outages', () => {
+ // The tests above run in one process; a remembered outage would make a
+ // game that draws nothing pass through a retry it must never get.
+ run([DEAD]);
+ const r = runReporting([{ ok: false, measured: true, drew: false }]);
+ expect(r.attempts).toBe(1);
+ });
it('a run whose engine could not draw has measured nothing', () => {
const broke = 'Framebuffer is incomplete! (size: 640x480, GL error 0x37442)';
expect(engineCouldNotDraw(broke)).toBe(true);