Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

16.4 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
 * @file  dead-gpu.test.ts — when a measurement counts and when it is retaken.
 *
 * The property that matters is the one a retry policy can destroy: a game that
 * genuinely draws nothing must still fail, on the first attempt, with no retry to
 * hide it. Everything else here is about not reporting a runner's dead GPU as a
 * verdict on the game.
 */
import { describe, it, expect } from 'vitest';
// @ts-expect-error — a .mjs tool module, typed by its own JSDoc
import { retryOnDeadGpu, gpuNeverCameUp, deadGpuVerdict, engineCouldNotDraw, resultMeasured, failureLines,
    launchNeverHappened, gpuCameUp } from '../lib/deadGpu.mjs';

const DEAD = 'Exiting GPU process due to errors during initialization';
const BLANK = 'painted=true live=false errors=0';
/** What the runner prints when the device DID arrive, on the two lines above the
 *  failure. Verbatim from the run that exposed the frame-lifecycle regression. */
const GPU_UP = '[renderer] [engine] webgpu adapter: apple metal-3 (default)\n'
    + "[renderer] [wasm] [INFO ] WebGPU device injected for '#canvas' (640x360)";
/** The regression's own shape: a device that came up, and a readback that did not. */
const EMPTY_CAPTURE = `${GPU_UP}\ncaptureViewportPixels: no pixels — the engine refused the readback`;

/** Drive the policy over a scripted sequence of attempt outputs; `null` = success. */
function run(outputs: Array<string | null>) {
    let attempts = 0;
    const notes: boolean[] = [];
    const result = retryOnDeadGpu(
        () => {
            const out = outputs[Math.min(attempts++, outputs.length - 1)];
            return { ok: out === null, output: out ?? '' };
        },
        (died: boolean) => notes.push(died),
        0,   // no backoff: there is no GPU process to wait for here
    );
    return { ...result, attempts, notes };
}

/** The same, for runs that report whether they reached a verdict. */
type Attempt = { ok: boolean; output?: string; measured?: boolean; drew?: boolean };
function runReporting(attempts_: Attempt[]) {
    let attempts = 0;
    const notes: boolean[] = [];
    const result = retryOnDeadGpu(
        () => ({ output: '', ...attempts_[Math.min(attempts++, attempts_.length - 1)] }),
        (noVerdict: boolean) => notes.push(noVerdict),
        0,
    );
    return { ...result, attempts, notes };
}

describe('failureLines', () => {
    it('keeps what happened and drops what every launch prints', () => {
        const out = `${DEAD}\n[6065:ERROR:dbus/bus.cc:405] Failed to connect to the bus\n`
            + '[screenshot] waitFor timed out: play realm ready\nok frame 3\nelectron exited with signal SIGTRAP';
        expect(failureLines(out)).toEqual([
            '[screenshot] waitFor timed out: play realm ready',
            'electron exited with signal SIGTRAP',
        ]);
    });

    it('says so when an attempt printed no error at all', () => {
        expect(failureLines(`${DEAD}\nall quiet`)).toEqual(['(the attempt printed no error line)']);
    });
});

describe('gpuNeverCameUp', () => {
    it('recognises the three ways a runner says it has no GPU', () => {
        expect(gpuNeverCameUp(DEAD)).toBe(true);
        expect(gpuNeverCameUp('GPU device lost\nFailed to create framebuffer')).toBe(true);
        expect(gpuNeverCameUp('WebGL2 is not available')).toBe(true);
    });

    it('does not read a verdict about pixels as one about the GPU', () => {
        expect(gpuNeverCameUp(BLANK)).toBe(false);
        // Device loss alone is a scene some gates drive on purpose; only paired
        // with a framebuffer that could not be made is it a runner with no GPU.
        expect(gpuNeverCameUp('GPU device lost')).toBe(false);
    });
});

describe('resultMeasured', () => {
    it('reads a result with a capture as a measurement, whatever its verdict', () => {
        expect(resultMeasured({ ok: true, capture: { rendered: true } })).toBe(true);
        // The property the whole policy rests on: a scene that drew the wrong
        // pixels HAS measured, and must not be handed a retry to hide behind.
        expect(resultMeasured({ ok: false, capture: { rendered: true }, expect: { ok: false } })).toBe(true);
    });

    it('reads an error with no capture as the harness failing to measure', () => {
        // What a lost GL context leaves: the readback throws, so there is no frame
        // to judge and nothing about the scene has been established.
        expect(resultMeasured({ ok: false, error: 'captureViewportPixels: no pixels' })).toBe(false);
    });

    it('has nothing to read when the driver printed no result at all', () => {
        expect(resultMeasured(null)).toBe(false);
        expect(resultMeasured(undefined)).toBe(false);
    });

    it('takes a capture over an error when a result carries both', () => {
        // A run that captured and then tripped over something else still measured.
        expect(resultMeasured({ ok: false, error: 'late failure', capture: { rendered: true } })).toBe(true);
    });
});

describe('gpuCameUp', () => {
    it('reads the two lines that say a device arrived', () => {
        expect(gpuCameUp(GPU_UP)).toBe(true);
        expect(gpuCameUp(EMPTY_CAPTURE)).toBe(true);
    });

    it('an adapter alone is not a device', () => {
        expect(gpuCameUp('[renderer] [engine] webgpu adapter: apple metal-3 (default)')).toBe(false);
    });

    it('says nothing about a runner that printed neither', () => {
        expect(gpuCameUp(DEAD)).toBe(false);
        expect(gpuCameUp(BLANK)).toBe(false);
    });
});

describe('retryOnDeadGpu', () => {
    it('an empty capture from a device that CAME UP is the subject\u2019s failure', () => {
        // `measured: false` is the half that makes this invisible: an error with
        // no capture is what resultMeasured calls "no measurement", and without
        // the veto the policy retries six times and blames the runner.
        const r = runReporting([{ ok: false, output: EMPTY_CAPTURE, measured: false }]);
        expect(r.ok).toBe(false);
        expect(r.attempts).toBe(1);
        expect(r.notes).toEqual([]);
        expect(r.gpuDied).toBeUndefined();
    });

    it('still retries when the device that came up was then LOST', () => {
        // "Came up" is not "stayed up": a device that goes mid-run took the
        // measurement with it, and that is the runner's story after all.
        const r = runReporting([
            { ok: false, output: `${GPU_UP}\nGPU device lost`, measured: false },
            { ok: true },
        ]);
        expect(r.ok).toBe(true);
        expect(r.attempts).toBe(2);
        expect(r.notes).toEqual([true]);
    });


    it('a game that draws nothing fails on the first attempt', () => {
        // The whole point: no retry may stand between a black frame and a red gate.
        const r = run([BLANK]);
        expect(r.ok).toBe(false);
        expect(r.attempts).toBe(1);
        expect(r.notes).toEqual([]);
    });

    it('takes one measurement when the GPU is up', () => {
        const r = run([null]);
        expect(r.ok).toBe(true);
        expect(r.attempts).toBe(1);
        expect(r.retried).toBe(false);
    });

    it('measures again when the GPU died before it drew', () => {
        const r = run([DEAD, null]);
        expect(r.ok).toBe(true);
        expect(r.attempts).toBe(2);
        expect(r.notes).toEqual([true]);
    });

    it('measures again for a blank frame that FOLLOWS a death in the chain', () => {
        // The case that made this necessary: a GPU process that dies and restarts
        // leaves the next launch on a context-lost renderer, which paints a blank
        // frame and prints nothing about why.
        const r = run([DEAD, BLANK, null]);
        expect(r.ok).toBe(true);
        expect(r.attempts).toBe(3);
        expect(r.notes).toEqual([true, false]);
    });

    it('reads a lost device that then fails to make TEXTURES', () => {
        // Verbatim from the runner, the day the second clause was found to only
        // know about framebuffers.
        const real = [
            '[ERROR] GPU device lost: context-lost [backend=WebGL2, vendor=Google Inc.]',
            '[ERROR] Texture::initialize: createTexture failed for 70x70',
            '[ERROR] Failed to create texture from spec',
        ].join('\n');
        expect(gpuNeverCameUp(real)).toBe(true);
    });

    it('words the give-up once, for both runners', () => {
        // The two of them each had their own sentence for it, and one of them
        // blamed the game — the failure this file exists to keep apart.
        expect(deadGpuVerdict('the game')).toContain('says nothing about the game');
        expect(deadGpuVerdict('the scene')).toContain('never came up');
    });

    // The runner this policy runs on prints the dead-GPU line on runs that then
    // succeed, so text alone made every failure look like an outage: one job
    // spent 71 retries and never named the scene that broke.
    it('a run that reached a verdict is not retried, whatever the log says', () => {
        const r = runReporting([{ ok: false, output: DEAD, measured: true, drew: true }]);
        expect(r.ok).toBe(false);
        expect(r.attempts).toBe(1);
        expect(r.notes).toEqual([]);
    });

    it('retries a run that reached no verdict, whatever the log says', () => {
        // Nothing in this output mentions a GPU; the run simply never got far
        // enough to judge anything, which is not a verdict about the subject.
        const r = runReporting([
            { ok: false, output: 'launch timed out', measured: false },
            { ok: true },
        ]);
        expect(r.ok).toBe(true);
        expect(r.attempts).toBe(2);
        expect(r.notes).toEqual([true]);
    });

    it('still retries a blank frame that follows a run with no verdict', () => {
        const r = runReporting([
            { ok: false, measured: false },
            { ok: false, measured: true, drew: false },
            { ok: true },
        ]);
        expect(r.ok).toBe(true);
        expect(r.attempts).toBe(3);
        expect(r.notes).toEqual([true, false]);
    });

    it('stops retrying once a run draws, even after an outage', () => {
        // A drawn frame that fails its assertion is the subject's own failure;
        // the outage is over and further attempts only hide it.
        const r = runReporting([
            { ok: false, measured: false },
            { ok: false, measured: true, drew: true },
        ]);
        expect(r.ok).toBe(false);
        expect(r.attempts).toBe(2);
        expect(r.gpuDied).toBeUndefined();
    });

    // The runner ran out of GPU resources and every scene came back empty. Each
    // run still PRINTED its result, so 'it finished' called it a measurement and
    // 46 of 70 scenes were reported as broken renderers in one job.
    it('retries a blank frame within two minutes of an outage in ANOTHER chain', () => {
        // golden captures the editor in one chain and launches the package in the
        // next; a GPU that died under the first is still coming back under the
        // second. A chain that waits (stepMs > 0) remembers; 1 ms keeps this quick.
        retryOnDeadGpu(() => ({ ok: false, output: DEAD }), () => {}, 1);
        let attempts = 0;
        const r = retryOnDeadGpu(() => {
            attempts++;
            return attempts === 1 ? { ok: false, output: '', measured: true, drew: false } : { ok: true, output: '' };
        }, () => {}, 1);
        expect(r.ok).toBe(true);
        expect(attempts).toBe(2);
    });
    it('a chain that waits for nothing keeps no memory of outages', () => {
        // The tests above run in one process; a remembered outage would make a
        // game that draws nothing pass through a retry it must never get.
        run([DEAD]);
        const r = runReporting([{ ok: false, measured: true, drew: false }]);
        expect(r.attempts).toBe(1);
    });
    it('a lost device with nothing left to create is still no verdict', () => {
        // Verbatim from the runner: the context went, the launcher judged the flat
        // frame that followed as "drew nothing", and no texture failed because
        // nothing was asked for after the loss.
        const lost = '[ERROR] GPU device lost: context-lost [backend=WebGL2, vendor=Google Inc. (Google)]';
        expect(engineCouldNotDraw(lost)).toBe(true);
        const r = runReporting([{ ok: false, output: lost, measured: true, drew: false }, { ok: true }]);
        expect(r.ok).toBe(true);
        expect(r.attempts).toBe(2);
    });
    it('a run whose engine could not draw has measured nothing', () => {
        const broke = 'Framebuffer is incomplete! (size: 640x480, GL error 0x37442)';
        expect(engineCouldNotDraw(broke)).toBe(true);
        expect(engineCouldNotDraw(BLANK)).toBe(false);
        const r = runReporting([
            { ok: false, output: broke, measured: true, drew: false },
            { ok: true },
        ]);
        expect(r.ok).toBe(true);
        expect(r.attempts).toBe(2);
        expect(r.notes).toEqual([true]);
    });

    it('retakes a measurement the harness could not take, and stops when it can', () => {
        // The runner loses the first launch's GL context and holds it after that,
        // which had every run's FIRST scene red while the hundred behind it passed.
        let attempts = 0;
        const r = retryOnDeadGpu(() => {
            attempts++;
            const result = attempts === 1
                ? { ok: false, error: 'captureViewportPixels: no pixels' }
                : { ok: true, capture: { rendered: true } };
            return { ok: result.ok, output: '', measured: resultMeasured(result), drew: result.capture?.rendered };
        }, () => {}, 0);
        expect(attempts).toBe(2);
        expect(r.ok).toBe(true);
        expect(r.retried).toBe(true);
    });

    it('does not retake a measurement that happened and disagreed', () => {
        let attempts = 0;
        const r = retryOnDeadGpu(() => {
            attempts++;
            const result = { ok: false, capture: { rendered: true }, expect: { ok: false } };
            return { ok: false, output: '', measured: resultMeasured(result), drew: true };
        }, () => {}, 0);
        expect(attempts).toBe(1);
        expect(r.gpuDied).toBeFalsy();
    });

    it('gives up at the cap without blaming the GPU on the line every launch prints', () => {
        const r = run([DEAD]);
        expect(r.ok).toBe(false);
        expect(r.attempts).toBe(6);
        expect(r.gpuDied).toBe(false);
    });

    it('blames the GPU at the cap when the engine says it could not draw', () => {
        const r = run([`${DEAD}\n[ERROR] GPU device lost: context-lost [backend=WebGL2]\n[ERROR] Texture::initialize: createTexture failed for 70x70`]);
        expect(r.attempts).toBe(6);
        expect(r.gpuDied).toBe(true);
    });

    // golden reports measured:false when a launch fails, which is exactly what a
    // dead GPU looks like — so a nightly with no xvfb retried eleven projects
    // six times each and spent 24 minutes calling it a GPU that died.
    it('hands each retry the attempt that failed', () => {
        const seen: string[] = [];
        let n = 0;
        retryOnDeadGpu(() => ({ ok: false, measured: false, output: `attempt ${++n}` }),
            (_noVerdict: boolean, last: { output: string }) => seen.push(last.output), 0);
        expect(seen).toEqual(['attempt 1', 'attempt 2', 'attempt 3', 'attempt 4', 'attempt 5']);
    });

    it('does not retry a launch that never happened', () => {
        const r = runReporting([{ ok: false, measured: false, output: 'pnpm did not start: ENOENT' }]);
        expect(r.attempts).toBe(1);
        expect(r.launchFailed).toBe(true);
        expect(r.gpuDied).toBeFalsy();
    });

    it('still retries a run with no verdict, which is not a launch that never happened', () => {
        const r = run([DEAD]);
        expect(r.launchFailed).toBeFalsy();
        expect(r.attempts).toBe(6);
    });
});

describe('launchNeverHappened', () => {
    it('recognises the ways electron does not start', () => {
        expect(launchNeverHappened('npx did not start: ENOENT')).toBe(true);
        expect(launchNeverHappened('Missing X server or $DISPLAY')).toBe(true);
        expect(launchNeverHappened('chrome-sandbox is owned by root and has mode 4755')).toBe(true);
        expect(launchNeverHappened('electron exited with signal SIGTRAP')).toBe(true);
    });

    it('is not what a dead GPU or a blank frame looks like', () => {
        expect(launchNeverHappened(DEAD)).toBe(false);
        expect(launchNeverHappened(BLANK)).toBe(false);
    });
});