A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
fix(smoke): a black frame is asked twice, and "too slow" stops counting as a pass
esengine095e30f6ebff
diff --git a/tools/lib/smokeRetry.mjs b/tools/lib/smokeRetry.mjsnew file mode 100644index 000000000..bbdb26010--- /dev/null+++ b/tools/lib/smokeRetry.mjs@@ -0,0 +1,71 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file smokeRetry.mjs — which native-boot verdicts are worth a second launch,+ * and which of the three states a run ends in.+ *+ * The property a retry policy can destroy: a game that genuinely draws nothing+ * must still fail. So only two verdicts are relaunched, and both were measured+ * to vary on one build rather than assumed to:+ *+ * - a frame of ONE flat colour. Across four runs of the same commit, chat and+ * collision-layers took turns being the black one, each drew normally in the+ * runs it was not, and either drew every time it was launched by itself. The+ * boot record reported the same geometry in both cases (`1 draw(s), 14 tri,+ * 7 sprite`), so what varies is whether the frame reached the surface — and+ * one launch cannot tell that from a game that draws nothing.+ * - a run too slow to have reached the frame it is judged at. The first launch+ * established the budget ran out, so the second gets a longer one.+ *+ * Everything else repeats: a crash, a launch that never reported ready, a dialog+ * with the focus. Relaunching those turns a broken build into a slow green one.+ *+ * A run that never reached a judged frame is the third state, never a pass: a+ * question this machine could not ask is not an answer about the game.+ */++/** @typedef {{ok?: boolean, undetermined?: boolean, why?: string, unjudged?: string|null, offScreen?: string|null}} SmokeResult */++/** The verdict for a frame that has no colour in it but one. */+const FLAT_FRAME = / flat color after /;++/**+ * Worth launching a second time, on the same installed APK.+ *+ * @param {SmokeResult} r+ */+export function worthAnotherLaunch(r) {+ // A dialog over the game is the emulator having a bad minute and is already+ // retried inside one launch; reaching here with it set means that retry is+ // spent, and the frame under a dialog is not this check's to judge.+ if (r.offScreen) return false;+ if (r.undetermined) return true;+ return !r.ok && FLAT_FRAME.test(r.why ?? '');+}++/**+ * Pass, fail, or unanswered — the three a run can end in.+ *+ * @param {SmokeResult} r+ * @returns {'pass'|'fail'|'undetermined'}+ */+export function verdictOf(r) {+ if (r.undetermined) return 'undetermined';+ return r.ok ? 'pass' : 'fail';+}++/**+ * The process's exit code: 1 if anything broke, 2 if nothing broke and something+ * went unanswered, 0 only when every example was asked and answered.+ *+ * Failure outranks unanswered: a run with both has a defect to report, and+ * reporting "this machine could not tell" instead would bury it.+ *+ * @param {SmokeResult[]} results+ * @param {(r: SmokeResult) => boolean} [isKnownBroken]+ */+export function exitCodeFor(results, isKnownBroken = () => false) {+ if (results.some((r) => verdictOf(r) === 'fail' && !isKnownBroken(r))) return 1;+ if (results.some((r) => verdictOf(r) === 'undetermined')) return 2;+ return 0;+}diff --git a/tools/tests/smoke-retry.test.ts b/tools/tests/smoke-retry.test.tsnew file mode 100644index 000000000..2c653ab75--- /dev/null+++ b/tools/tests/smoke-retry.test.ts@@ -0,0 +1,70 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file smoke-retry.test.ts — when a launch is repeated and when it stands.+ *+ * The property a retry policy can destroy: a game that genuinely draws nothing+ * must still fail, and it must fail on the first launch with no second one to+ * hide it. The rest is about not reporting an emulator's black frame as a+ * verdict on the game, and about "this machine could not judge" never again+ * counting as a pass.+ */+import { describe, it, expect } from 'vitest';+// @ts-expect-error — a .mjs tool module, typed by its own JSDoc+import { worthAnotherLaunch, verdictOf, exitCodeFor } from '../lib/smokeRetry.mjs';++const flat = { ok: false, why: 'the frame is 1 flat color after 120 frame(s)' };+const drew = { ok: true, colors: 8 };+const slow = { ok: false, undetermined: true, unjudged: 'too slow to judge here — 10 frame(s) before the capture' };++describe('what earns a second launch', () => {+ it('repeats a frame of one flat colour', () => {+ expect(worthAnotherLaunch(flat)).toBe(true);+ });++ it('repeats a run that never reached the frame it is judged at', () => {+ expect(worthAnotherLaunch(slow)).toBe(true);+ });++ // The whole point of the policy: these are the ones that must not be given a+ // second chance, or a broken build becomes a slow green one.+ it('does not repeat a crash, a launch that never reported ready, or a boot error', () => {+ expect(worthAnotherLaunch({ ok: false, why: 'never reported ready' })).toBe(false);+ expect(worthAnotherLaunch({ ok: false, why: 'reported ready, then drew no frame at all' })).toBe(false);+ expect(worthAnotherLaunch({ ok: false, why: 'ERROR [asset] texture 4 failed to upload' })).toBe(false);+ expect(worthAnotherLaunch({ ok: false, why: 'after an activity recreate: ERROR [js] undefined' })).toBe(false);+ });++ it('does not repeat a game a dialog was covering — that retry is already spent', () => {+ expect(worthAnotherLaunch({ ok: false, why: 'the game was not on screen — a dialog has focus', offScreen: 'a dialog has focus' })).toBe(false);+ });++ it('leaves a run that drew alone', () => {+ expect(worthAnotherLaunch(drew)).toBe(false);+ });+});++describe('three states, not two', () => {+ it('separates unanswered from passed', () => {+ expect(verdictOf(drew)).toBe('pass');+ expect(verdictOf(flat)).toBe('fail');+ expect(verdictOf(slow)).toBe('undetermined');+ });++ // 0.59.0 shipped a black audio-demo because this returned 0.+ it('exits 2 when a run went unanswered and nothing broke', () => {+ expect(exitCodeFor([drew, slow])).toBe(2);+ });++ it('exits 1 when something broke, even beside an unanswered one', () => {+ expect(exitCodeFor([flat, slow])).toBe(1);+ });++ it('exits 0 only when every example was asked and answered', () => {+ expect(exitCodeFor([drew, drew])).toBe(0);+ });++ it('does not fail the run for an example already recorded as broken', () => {+ expect(exitCodeFor([{ ...flat, name: 'known' }, drew], (r) => (r as { name?: string }).name === 'known')).toBe(0);+ });+});diff --git a/tools/verify-native-boot.mjs b/tools/verify-native-boot.mjsindex b6674853b..183574769 100644--- a/tools/verify-native-boot.mjs+++ b/tools/verify-native-boot.mjs@@ -47,6 +47,7 @@ import path from 'node:path';import { fileURLToPath } from 'node:url';import { decodePng, distinctColors } from '../build-tools/utils/png.js';import { GOLDEN, launchTimeoutFor } from './goldenProjects.mjs';+import { worthAnotherLaunch, exitCodeFor } from './lib/smokeRetry.mjs';const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');const APP_ID = 'com.estella.game';@@ -566,7 +567,7 @@ function framesDrawn(log) {return n;}-async function verifyApp(driver, artifact, label, opts, judgeFrame = true) {+async function verifyApp(driver, artifact, label, opts, judgeFrame = true, round = 1) {driver.install(artifact);let log = '';@@ -596,7 +597,9 @@ async function verifyApp(driver, artifact, label, opts, judgeFrame = true) {const settleBy = Date.now() + Math.max(opts.settle, 1) * 1000;// Its own budget, not the ready timeout: a scene this slow is worth// waiting out, a hung one is not, and the set is 40-odd apps per device.- const framesBy = Date.now() + frameBudgetMs(label);+ // A second round gets a longer one: the first said the budget was what+ // ran out, so repeating it unchanged would only ask the same question.+ const framesBy = Date.now() + frameBudgetMs(label) * round;await sleep(Math.max(settleBy - Date.now(), 0));// Backing off rather than polling flat out: reading the record spawns a// process on the device, and asking a struggling app ninety times is the@@ -703,8 +706,11 @@ async function verifyApp(driver, artifact, label, opts, judgeFrame = true) {? `the frame is ${colors} flat color after ${frames} frame(s)`: recreateWhy;+ // Three states: `unjudged` is this machine failing to ask the question, and+ // is neither a pass nor a verdict on the game (smokeRetry.mjs).return {- ok: !why, ready, colors, frames, unjudged, readyLine, errors, offScreen, metrics,+ ok: !why && !unjudged, undetermined: !why && !!unjudged,+ ready, colors, frames, unjudged, readyLine, errors, offScreen, metrics,size: `${image.width}x${image.height}`, log, shot, why,};}@@ -856,6 +862,11 @@ for (const name of examples) {try {const { app, work } = packageExample(driver, name, opts);r = await verifyApp(driver, app, name, opts, !FRAME_NOT_JUDGED[name]);+ if (worthAnotherLaunch(r)) {+ const first = r;+ r = await verifyApp(driver, app, name, opts, !FRAME_NOT_JUDGED[name], 2);+ r.firstTry = first;+ }// Each APK carries the whole runtime template; keeping 40 of them around// is gigabytes for no reason.rmSync(work, { recursive: true, force: true });@@ -865,26 +876,37 @@ for (const name of examples) {results.push({ name, ...r });const frameNote = FRAME_NOT_JUDGED[name] ? ' (frame not judged)': r.unjudged ? ` — ${r.unjudged}` : '';- console.log(`[smoke] ${name.padEnd(22)} ${r.ok ? `✓ ${r.size}, ${r.colors} colors, ${r.frames} frames${frameNote}` : `✗ ${r.why.split('\n')[0]}`}`);+ // What the first launch said, kept in the line: a run that needed two is not+ // the same event as one that needed one, and a name that starts appearing+ // here every night is the black-frame defect getting worse, not a flake.+ const retryNote = r.firstTry+ ? ` (relaunched — first: ${r.firstTry.why?.split('\n')[0] ?? r.firstTry.unjudged})`+ : '';+ const verdict = r.ok ? `✓ ${r.size}, ${r.colors} colors, ${r.frames} frames${frameNote}`+ : r.undetermined ? `? ${r.unjudged} — twice`+ : `✗ ${r.why.split('\n')[0]}`;+ console.log(`[smoke] ${name.padEnd(22)} ${verdict}${retryNote}`);}const known = Object.keys(KNOWN_FAILURES);-const brokeUnexpectedly = results.filter((r) => !r.ok && !KNOWN_FAILURES[r.name]);+const brokeUnexpectedly = results.filter((r) => !r.ok && !r.undetermined && !KNOWN_FAILURES[r.name]);const fixedUnexpectedly = results.filter((r) => r.ok && KNOWN_FAILURES[r.name]);-// Counted, because a run that could not judge half its frames proved half of-// what it claims and should not read as a clean pass.-const tooSlow = results.filter((r) => r.ok && r.unjudged);+// Neither passed nor failed: twice the capture landed before the run reached the+// frame it judges at. Named, counted, and carried into the exit code, because a+// question this machine could not ask is not an answer about the game.+const tooSlow = results.filter((r) => r.undetermined);note(`### ${driver.name} — ${results.filter((r) => r.ok).length}/${results.length} examples started and drew`- + (tooSlow.length ? `, ${tooSlow.length} too slow to judge (${tooSlow.map((r) => r.name).join(', ')})` : ''));+ + (tooSlow.length ? `, ${tooSlow.length} unanswered here (${tooSlow.map((r) => r.name).join(', ')})` : ''));note('');note(`Device: \`${device}\``);note('');note('| example | result | frame |');note('| --- | --- | --- |');for (const r of results) {- const state = r.ok ? '✓'- : (KNOWN_FAILURES[r.name] ? `✗ known — ${KNOWN_FAILURES[r.name]}` : `✗ ${r.why.split('\n')[0]}`);+ const state = r.ok ? (r.firstTry ? '✓ (relaunched)' : '✓')+ : r.undetermined ? `? ${r.unjudged} — twice`+ : (KNOWN_FAILURES[r.name] ? `✗ known — ${KNOWN_FAILURES[r.name]}` : `✗ ${r.why.split('\n')[0]}`);note(`| ${r.name} | ${state} | ${r.ok ? `${r.size}, ${r.colors} colors` : '—'} |`);}@@ -917,4 +939,15 @@ if (brokeUnexpectedly.length || fixedUnexpectedly.length) {process.exit(1);}-console.log(`\n✓ ${driver.name}: ${results.filter((r) => r.ok).length}/${results.length} examples started and drew a frame`);+// 2, not 0: nothing here failed, and this run did not answer for every example+// it was given. A caller that treats 2 as a pass is making the same trade 0.59.0+// made silently, but at least it is making it out loud.+if (exitCodeFor(results, (r) => !!KNOWN_FAILURES[r.name]) === 2) {+ console.error(`\n— UNANSWERED: ${tooSlow.length} example(s) reached no judged frame in two launches`);+ for (const r of tooSlow) console.error(` ${r.name} — ${r.unjudged}`);+ process.exit(2);+}++const relaunched = results.filter((r) => r.firstTry);+console.log(`\n✓ ${driver.name}: ${results.filter((r) => r.ok).length}/${results.length} examples started and drew a frame`+ + (relaunched.length ? ` (${relaunched.length} on a second launch: ${relaunched.map((r) => r.name).join(', ')})` : ''));