A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
fix(verify): a blank frame inside two minutes of any outage is aftermath, whichever chain saw it
yhh3eb0344d0a6b
diff --git a/tools/lib/deadGpu.mjs b/tools/lib/deadGpu.mjsindex 18fe93fe8..c971f3f48 100644--- a/tools/lib/deadGpu.mjs+++ b/tools/lib/deadGpu.mjs@@ -84,6 +84,15 @@ function backoff(attempt, stepMs) {*/const STEP_MS = 5000;+/**+ * When this process last saw an outage, in any chain. A blank frame inside the+ * two minutes a GPU takes to come back is aftermath even when the death was+ * another launch's: golden captures the editor, then launches the package, and+ * the package's blank frame was judged in a chain that had seen nothing.+ */+let lastOutageAt = 0;+const OUTAGE_SETTLES_MS = 2 * 60 * 1000;+/*** The ENGINE saying it could not make what it draws through. A run that gets this* far still prints its result — reporting an empty frame — so "it finished" is not@@ -127,7 +136,9 @@ function reachedNoVerdict(last) {* screen? a blank frame right after an outage is that outage still settling).*/export function retryOnDeadGpu(attempt, note, stepMs = STEP_MS) {- let sawOutage = false;+ // A chain that waits for nothing (tests) has no runner to remember across.+ const remembers = stepMs > 0;+ let sawOutage = remembers && Date.now() - lastOutageAt < OUTAGE_SETTLES_MS;let last;for (let i = 0; i < MAX_ATTEMPTS; i++) {last = attempt();@@ -137,7 +148,10 @@ export function retryOnDeadGpu(attempt, note, stepMs = STEP_MS) {if (launchNeverHappened(last.output ?? '')) return { ...last, retried: i > 0, launchFailed: true };const noVerdict = reachedNoVerdict(last);- if (noVerdict) sawOutage = true;+ if (noVerdict) {+ sawOutage = true;+ if (remembers) lastOutageAt = Date.now();+ }// `drew === true` is the only thing that rules out aftermath: an attempt// that cannot say keeps the older, more cautious behaviour.const aftermath = sawOutage && last.drew !== true;diff --git a/tools/tests/dead-gpu.test.ts b/tools/tests/dead-gpu.test.tsindex a320b6b28..0939f7496 100644--- a/tools/tests/dead-gpu.test.ts+++ b/tools/tests/dead-gpu.test.ts@@ -183,6 +183,26 @@ describe('retryOnDeadGpu', () => {// The runner ran out of GPU resources and every scene came back empty. Each// run still PRINTED its result, so 'it finished' called it a measurement and// 46 of 70 scenes were reported as broken renderers in one job.+ it('retries a blank frame within two minutes of an outage in ANOTHER chain', () => {+ // golden captures the editor in one chain and launches the package in the+ // next; a GPU that died under the first is still coming back under the+ // second. A chain that waits (stepMs > 0) remembers; 1 ms keeps this quick.+ retryOnDeadGpu(() => ({ ok: false, output: DEAD }), () => {}, 1);+ let attempts = 0;+ const r = retryOnDeadGpu(() => {+ attempts++;+ return attempts === 1 ? { ok: false, output: '', measured: true, drew: false } : { ok: true, output: '' };+ }, () => {}, 1);+ expect(r.ok).toBe(true);+ expect(attempts).toBe(2);+ });+ it('a chain that waits for nothing keeps no memory of outages', () => {+ // The tests above run in one process; a remembered outage would make a+ // game that draws nothing pass through a retry it must never get.+ run([DEAD]);+ const r = runReporting([{ ok: false, measured: true, drew: false }]);+ expect(r.attempts).toBe(1);+ });it('a run whose engine could not draw has measured nothing', () => {const broke = 'Framebuffer is incomplete! (size: 640x480, GL error 0x37442)';expect(engineCouldNotDraw(broke)).toBe(true);