Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

fix(smoke): a black frame is asked twice, and "too slow" stops counting as a pass

esengine095e30f6ebff
diff --git a/tools/lib/smokeRetry.mjs b/tools/lib/smokeRetry.mjs
new file mode 100644
index 000000000..bbdb26010
--- /dev/null
+++ b/tools/lib/smokeRetry.mjs
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file smokeRetry.mjs — which native-boot verdicts are worth a second launch,
+ * and which of the three states a run ends in.
+ *
+ * The property a retry policy can destroy: a game that genuinely draws nothing
+ * must still fail. So only two verdicts are relaunched, and both were measured
+ * to vary on one build rather than assumed to:
+ *
+ * - a frame of ONE flat colour. Across four runs of the same commit, chat and
+ * collision-layers took turns being the black one, each drew normally in the
+ * runs it was not, and either drew every time it was launched by itself. The
+ * boot record reported the same geometry in both cases (`1 draw(s), 14 tri,
+ * 7 sprite`), so what varies is whether the frame reached the surface — and
+ * one launch cannot tell that from a game that draws nothing.
+ * - a run too slow to have reached the frame it is judged at. The first launch
+ * established the budget ran out, so the second gets a longer one.
+ *
+ * Everything else repeats: a crash, a launch that never reported ready, a dialog
+ * with the focus. Relaunching those turns a broken build into a slow green one.
+ *
+ * A run that never reached a judged frame is the third state, never a pass: a
+ * question this machine could not ask is not an answer about the game.
+ */
+
+/** @typedef {{ok?: boolean, undetermined?: boolean, why?: string, unjudged?: string|null, offScreen?: string|null}} SmokeResult */
+
+/** The verdict for a frame that has no colour in it but one. */
+const FLAT_FRAME = / flat color after /;
+
+/**
+ * Worth launching a second time, on the same installed APK.
+ *
+ * @param {SmokeResult} r
+ */
+export function worthAnotherLaunch(r) {
+ // A dialog over the game is the emulator having a bad minute and is already
+ // retried inside one launch; reaching here with it set means that retry is
+ // spent, and the frame under a dialog is not this check's to judge.
+ if (r.offScreen) return false;
+ if (r.undetermined) return true;
+ return !r.ok && FLAT_FRAME.test(r.why ?? '');
+}
+
+/**
+ * Pass, fail, or unanswered — the three a run can end in.
+ *
+ * @param {SmokeResult} r
+ * @returns {'pass'|'fail'|'undetermined'}
+ */
+export function verdictOf(r) {
+ if (r.undetermined) return 'undetermined';
+ return r.ok ? 'pass' : 'fail';
+}
+
+/**
+ * The process's exit code: 1 if anything broke, 2 if nothing broke and something
+ * went unanswered, 0 only when every example was asked and answered.
+ *
+ * Failure outranks unanswered: a run with both has a defect to report, and
+ * reporting "this machine could not tell" instead would bury it.
+ *
+ * @param {SmokeResult[]} results
+ * @param {(r: SmokeResult) => boolean} [isKnownBroken]
+ */
+export function exitCodeFor(results, isKnownBroken = () => false) {
+ if (results.some((r) => verdictOf(r) === 'fail' && !isKnownBroken(r))) return 1;
+ if (results.some((r) => verdictOf(r) === 'undetermined')) return 2;
+ return 0;
+}
diff --git a/tools/tests/smoke-retry.test.ts b/tools/tests/smoke-retry.test.ts
new file mode 100644
index 000000000..2c653ab75
--- /dev/null
+++ b/tools/tests/smoke-retry.test.ts
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file smoke-retry.test.ts — when a launch is repeated and when it stands.
+ *
+ * The property a retry policy can destroy: a game that genuinely draws nothing
+ * must still fail, and it must fail on the first launch with no second one to
+ * hide it. The rest is about not reporting an emulator's black frame as a
+ * verdict on the game, and about "this machine could not judge" never again
+ * counting as a pass.
+ */
+import { describe, it, expect } from 'vitest';
+// @ts-expect-error — a .mjs tool module, typed by its own JSDoc
+import { worthAnotherLaunch, verdictOf, exitCodeFor } from '../lib/smokeRetry.mjs';
+
+const flat = { ok: false, why: 'the frame is 1 flat color after 120 frame(s)' };
+const drew = { ok: true, colors: 8 };
+const slow = { ok: false, undetermined: true, unjudged: 'too slow to judge here — 10 frame(s) before the capture' };
+
+describe('what earns a second launch', () => {
+ it('repeats a frame of one flat colour', () => {
+ expect(worthAnotherLaunch(flat)).toBe(true);
+ });
+
+ it('repeats a run that never reached the frame it is judged at', () => {
+ expect(worthAnotherLaunch(slow)).toBe(true);
+ });
+
+ // The whole point of the policy: these are the ones that must not be given a
+ // second chance, or a broken build becomes a slow green one.
+ it('does not repeat a crash, a launch that never reported ready, or a boot error', () => {
+ expect(worthAnotherLaunch({ ok: false, why: 'never reported ready' })).toBe(false);
+ expect(worthAnotherLaunch({ ok: false, why: 'reported ready, then drew no frame at all' })).toBe(false);
+ expect(worthAnotherLaunch({ ok: false, why: 'ERROR [asset] texture 4 failed to upload' })).toBe(false);
+ expect(worthAnotherLaunch({ ok: false, why: 'after an activity recreate: ERROR [js] undefined' })).toBe(false);
+ });
+
+ it('does not repeat a game a dialog was covering — that retry is already spent', () => {
+ expect(worthAnotherLaunch({ ok: false, why: 'the game was not on screen — a dialog has focus', offScreen: 'a dialog has focus' })).toBe(false);
+ });
+
+ it('leaves a run that drew alone', () => {
+ expect(worthAnotherLaunch(drew)).toBe(false);
+ });
+});
+
+describe('three states, not two', () => {
+ it('separates unanswered from passed', () => {
+ expect(verdictOf(drew)).toBe('pass');
+ expect(verdictOf(flat)).toBe('fail');
+ expect(verdictOf(slow)).toBe('undetermined');
+ });
+
+ // 0.59.0 shipped a black audio-demo because this returned 0.
+ it('exits 2 when a run went unanswered and nothing broke', () => {
+ expect(exitCodeFor([drew, slow])).toBe(2);
+ });
+
+ it('exits 1 when something broke, even beside an unanswered one', () => {
+ expect(exitCodeFor([flat, slow])).toBe(1);
+ });
+
+ it('exits 0 only when every example was asked and answered', () => {
+ expect(exitCodeFor([drew, drew])).toBe(0);
+ });
+
+ it('does not fail the run for an example already recorded as broken', () => {
+ expect(exitCodeFor([{ ...flat, name: 'known' }, drew], (r) => (r as { name?: string }).name === 'known')).toBe(0);
+ });
+});
diff --git a/tools/verify-native-boot.mjs b/tools/verify-native-boot.mjs
index b6674853b..183574769 100644
--- a/tools/verify-native-boot.mjs
+++ b/tools/verify-native-boot.mjs
@@ -47,6 +47,7 @@ import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { decodePng, distinctColors } from '../build-tools/utils/png.js';
import { GOLDEN, launchTimeoutFor } from './goldenProjects.mjs';
+import { worthAnotherLaunch, exitCodeFor } from './lib/smokeRetry.mjs';
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const APP_ID = 'com.estella.game';
@@ -566,7 +567,7 @@ function framesDrawn(log) {
return n;
}
-async function verifyApp(driver, artifact, label, opts, judgeFrame = true) {
+async function verifyApp(driver, artifact, label, opts, judgeFrame = true, round = 1) {
driver.install(artifact);
let log = '';
@@ -596,7 +597,9 @@ async function verifyApp(driver, artifact, label, opts, judgeFrame = true) {
const settleBy = Date.now() + Math.max(opts.settle, 1) * 1000;
// Its own budget, not the ready timeout: a scene this slow is worth
// waiting out, a hung one is not, and the set is 40-odd apps per device.
- const framesBy = Date.now() + frameBudgetMs(label);
+ // A second round gets a longer one: the first said the budget was what
+ // ran out, so repeating it unchanged would only ask the same question.
+ const framesBy = Date.now() + frameBudgetMs(label) * round;
await sleep(Math.max(settleBy - Date.now(), 0));
// Backing off rather than polling flat out: reading the record spawns a
// process on the device, and asking a struggling app ninety times is the
@@ -703,8 +706,11 @@ async function verifyApp(driver, artifact, label, opts, judgeFrame = true) {
? `the frame is ${colors} flat color after ${frames} frame(s)`
: recreateWhy;
+ // Three states: `unjudged` is this machine failing to ask the question, and
+ // is neither a pass nor a verdict on the game (smokeRetry.mjs).
return {
- ok: !why, ready, colors, frames, unjudged, readyLine, errors, offScreen, metrics,
+ ok: !why && !unjudged, undetermined: !why && !!unjudged,
+ ready, colors, frames, unjudged, readyLine, errors, offScreen, metrics,
size: `${image.width}x${image.height}`, log, shot, why,
};
}
@@ -856,6 +862,11 @@ for (const name of examples) {
try {
const { app, work } = packageExample(driver, name, opts);
r = await verifyApp(driver, app, name, opts, !FRAME_NOT_JUDGED[name]);
+ if (worthAnotherLaunch(r)) {
+ const first = r;
+ r = await verifyApp(driver, app, name, opts, !FRAME_NOT_JUDGED[name], 2);
+ r.firstTry = first;
+ }
// Each APK carries the whole runtime template; keeping 40 of them around
// is gigabytes for no reason.
rmSync(work, { recursive: true, force: true });
@@ -865,26 +876,37 @@ for (const name of examples) {
results.push({ name, ...r });
const frameNote = FRAME_NOT_JUDGED[name] ? ' (frame not judged)'
: r.unjudged ? ` — ${r.unjudged}` : '';
- console.log(`[smoke] ${name.padEnd(22)} ${r.ok ? `✓ ${r.size}, ${r.colors} colors, ${r.frames} frames${frameNote}` : `✗ ${r.why.split('\n')[0]}`}`);
+ // What the first launch said, kept in the line: a run that needed two is not
+ // the same event as one that needed one, and a name that starts appearing
+ // here every night is the black-frame defect getting worse, not a flake.
+ const retryNote = r.firstTry
+ ? ` (relaunched — first: ${r.firstTry.why?.split('\n')[0] ?? r.firstTry.unjudged})`
+ : '';
+ const verdict = r.ok ? `✓ ${r.size}, ${r.colors} colors, ${r.frames} frames${frameNote}`
+ : r.undetermined ? `? ${r.unjudged} — twice`
+ : `✗ ${r.why.split('\n')[0]}`;
+ console.log(`[smoke] ${name.padEnd(22)} ${verdict}${retryNote}`);
}
const known = Object.keys(KNOWN_FAILURES);
-const brokeUnexpectedly = results.filter((r) => !r.ok && !KNOWN_FAILURES[r.name]);
+const brokeUnexpectedly = results.filter((r) => !r.ok && !r.undetermined && !KNOWN_FAILURES[r.name]);
const fixedUnexpectedly = results.filter((r) => r.ok && KNOWN_FAILURES[r.name]);
-// Counted, because a run that could not judge half its frames proved half of
-// what it claims and should not read as a clean pass.
-const tooSlow = results.filter((r) => r.ok && r.unjudged);
+// Neither passed nor failed: twice the capture landed before the run reached the
+// frame it judges at. Named, counted, and carried into the exit code, because a
+// question this machine could not ask is not an answer about the game.
+const tooSlow = results.filter((r) => r.undetermined);
note(`### ${driver.name} — ${results.filter((r) => r.ok).length}/${results.length} examples started and drew`
- + (tooSlow.length ? `, ${tooSlow.length} too slow to judge (${tooSlow.map((r) => r.name).join(', ')})` : ''));
+ + (tooSlow.length ? `, ${tooSlow.length} unanswered here (${tooSlow.map((r) => r.name).join(', ')})` : ''));
note('');
note(`Device: \`${device}\``);
note('');
note('| example | result | frame |');
note('| --- | --- | --- |');
for (const r of results) {
- const state = r.ok ? '✓'
- : (KNOWN_FAILURES[r.name] ? `✗ known — ${KNOWN_FAILURES[r.name]}` : `✗ ${r.why.split('\n')[0]}`);
+ const state = r.ok ? (r.firstTry ? '✓ (relaunched)' : '✓')
+ : r.undetermined ? `? ${r.unjudged} — twice`
+ : (KNOWN_FAILURES[r.name] ? `✗ known — ${KNOWN_FAILURES[r.name]}` : `✗ ${r.why.split('\n')[0]}`);
note(`| ${r.name} | ${state} | ${r.ok ? `${r.size}, ${r.colors} colors` : '—'} |`);
}
@@ -917,4 +939,15 @@ if (brokeUnexpectedly.length || fixedUnexpectedly.length) {
process.exit(1);
}
-console.log(`\n✓ ${driver.name}: ${results.filter((r) => r.ok).length}/${results.length} examples started and drew a frame`);
+// 2, not 0: nothing here failed, and this run did not answer for every example
+// it was given. A caller that treats 2 as a pass is making the same trade 0.59.0
+// made silently, but at least it is making it out loud.
+if (exitCodeFor(results, (r) => !!KNOWN_FAILURES[r.name]) === 2) {
+ console.error(`\n— UNANSWERED: ${tooSlow.length} example(s) reached no judged frame in two launches`);
+ for (const r of tooSlow) console.error(` ${r.name} — ${r.unjudged}`);
+ process.exit(2);
+}
+
+const relaunched = results.filter((r) => r.firstTry);
+console.log(`\n✓ ${driver.name}: ${results.filter((r) => r.ok).length}/${results.length} examples started and drew a frame`
+ + (relaunched.length ? ` (${relaunched.length} on a second launch: ${relaunched.map((r) => r.name).join(', ')})` : ''));