Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(ci): hold every packaged game against the editor's own frame of it

esengine6137a28347a4
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ab1c37e99..20b814208 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -527,11 +527,11 @@ jobs:
ELECTRON_DISABLE_SANDBOX: '1'
run: xvfb-run -a node scripts/editor-checks/run.mjs
- # The golden corpus, packaged and opened the way a player opens it. Exporting
- # was checked and the engine was checked; the package itself never was, which
- # is where a wrong export default or a path that only resolves in the editor
- # survives every other green check and ships.
- - name: Launch the golden packages (PR tier)
+ # The golden corpus, packaged and opened the way a player opens it, then held
+ # against the editor's own frame of the same game. Exporting was checked and
+ # the engine was checked; the package never was — and "it launched" is not the
+ # claim, since a build that boots to its rotate-to-portrait gate launches fine.
+ - name: Launch the golden packages and compare them to the editor (PR tier)
env:
ELECTRON_DISABLE_SANDBOX: '1'
run: xvfb-run -a node tools/verify-golden.mjs --tier pr
diff --git a/desktop/scripts/launch-export.mjs b/desktop/scripts/launch-export.mjs
index e4ea751da..3c6523e91 100644
--- a/desktop/scripts/launch-export.mjs
+++ b/desktop/scripts/launch-export.mjs
@@ -100,7 +100,10 @@ async function main() {
const base = `http://127.0.0.1:${server.address().port}/`;
const win = new BrowserWindow({
- width: W, height: H, show: false,
+ // Content size, not window size: with the frame counted in, the surface came
+ // out shorter than asked for, and a capture compared against another surface
+ // is only meaningful when the size requested is the size rendered.
+ width: W, height: H, useContentSize: true, show: false,
webPreferences: { offscreen: false, backgroundThrottling: false },
});
await win.webContents.session.clearCache();
diff --git a/desktop/tests/frame-compare.test.ts b/desktop/tests/frame-compare.test.ts
new file mode 100644
index 000000000..6329a08c6
--- /dev/null
+++ b/desktop/tests/frame-compare.test.ts
@@ -0,0 +1,137 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file The score behind "the package shows what the editor showed".
+ *
+ * It has to separate two runs of one game from two different games by a
+ * wide margin, or the tolerance is guesswork. Measured on real captures:
+ * the same game lands at 0.001–0.006 and two different games at
+ * 0.37–0.48, which is the gap these cases pin down synthetically.
+ */
+import { describe, it, expect } from 'vitest';
+import { deflateSync } from 'node:zlib';
+// @ts-expect-error — plain-JS tool module, no d.ts
+import { downsample, gridDistance, frameDistance, DEFAULT_INSET } from '../../tools/frameCompare.mjs';
+
+interface Img { w: number; h: number; px(x: number, y: number): [number, number, number] }
+
+/** An image from a function of position, duck-typed like the PNG reader's result. */
+const img = (w: number, h: number, f: (x: number, y: number) => [number, number, number]): Img =>
+ ({ w, h, px: (x, y) => f(x, y) });
+
+const solid = (w: number, h: number, c: [number, number, number]) => img(w, h, () => c);
+/** Left half one colour, right half another — a layout the score must notice moving. */
+const split = (w: number, h: number, a: [number, number, number], b: [number, number, number]) =>
+ img(w, h, (x) => (x < w / 2 ? a : b));
+
+describe('downsample', () => {
+ it('averages a region to its mean colour', () => {
+ const g = downsample(solid(40, 20, [10, 20, 30]), 4, 2);
+ expect([...g.slice(0, 3)]).toEqual([10, 20, 30]);
+ expect(g).toHaveLength(4 * 2 * 3);
+ });
+
+ it('keeps left and right distinguishable at a coarse grid', () => {
+ const g = downsample(split(40, 20, [0, 0, 0], [255, 255, 255]), 2, 1);
+ expect(g[0]).toBeLessThan(10); // left cell
+ expect(g[3]).toBeGreaterThan(245); // right cell
+ });
+
+ it('drops the inset border — the editor draws a play frame around its surface', () => {
+ // A green ring one tenth deep around a black field: with the ring included the
+ // mean is polluted; inset past it and the field reads as the field.
+ const ring = img(100, 100, (x, y) =>
+ (x < 10 || y < 10 || x >= 90 || y >= 90 ? [0, 255, 0] : [0, 0, 0]));
+ const withRing = downsample(ring, 1, 1, 0);
+ const inside = downsample(ring, 1, 1, 0.12);
+ expect(withRing[1]).toBeGreaterThan(50);
+ expect(inside[1]).toBe(0);
+ });
+});
+
+describe('gridDistance', () => {
+ it('is zero for a grid against itself', () => {
+ const g = downsample(split(20, 10, [12, 34, 56], [200, 100, 50]), 4, 4);
+ expect(gridDistance(g, g)).toBe(0);
+ });
+
+ it('is one for black against white', () => {
+ const a = downsample(solid(8, 8, [0, 0, 0]), 2, 2);
+ const b = downsample(solid(8, 8, [255, 255, 255]), 2, 2);
+ expect(gridDistance(a, b)).toBeCloseTo(1, 6);
+ });
+
+ it('separates a shifted layout from an identical one by an order of magnitude', () => {
+ const base = downsample(split(64, 32, [20, 40, 60], [220, 180, 140]), 16, 8);
+ const jitter = downsample(
+ img(64, 32, (x) => (x < 33 ? [20, 40, 60] : [220, 180, 140])), 16, 8,
+ ); // the seam moved one pixel — drift, not a different game
+ const other = downsample(split(64, 32, [220, 180, 140], [20, 40, 60]), 16, 8); // mirrored
+ expect(gridDistance(base, jitter)).toBeLessThan(0.02);
+ expect(gridDistance(base, other)).toBeGreaterThan(0.3);
+ });
+
+ it('refuses grids of different sizes rather than scoring nonsense', () => {
+ expect(() => gridDistance(new Float64Array(3), new Float64Array(6))).toThrow(/mismatch/);
+ });
+});
+
+/** A minimal 8-bit RGB PNG, so the size guard can be tested on real input rather
+ * than on a buffer that fails to decode for unrelated reasons. */
+function png(w: number, h: number, c: [number, number, number]): Buffer {
+ const crcTable = Array.from({ length: 256 }, (_, n) => {
+ let v = n;
+ for (let k = 0; k < 8; k++) v = v & 1 ? 0xedb88320 ^ (v >>> 1) : v >>> 1;
+ return v >>> 0;
+ });
+ const crc = (b: Buffer) => {
+ let v = 0xffffffff;
+ for (const byte of b) v = crcTable[(v ^ byte) & 0xff] ^ (v >>> 8);
+ return (v ^ 0xffffffff) >>> 0;
+ };
+ const chunk = (type: string, data: Buffer) => {
+ const len = Buffer.alloc(4);
+ len.writeUInt32BE(data.length);
+ const body = Buffer.concat([Buffer.from(type, 'ascii'), data]);
+ const cr = Buffer.alloc(4);
+ cr.writeUInt32BE(crc(body));
+ return Buffer.concat([len, body, cr]);
+ };
+ const ihdr = Buffer.alloc(13);
+ ihdr.writeUInt32BE(w, 0);
+ ihdr.writeUInt32BE(h, 4);
+ ihdr[8] = 8; ihdr[9] = 2; // 8-bit, truecolour
+ const raw = Buffer.alloc(h * (w * 3 + 1));
+ for (let y = 0; y < h; y++) {
+ const row = y * (w * 3 + 1);
+ raw[row] = 0; // no filter
+ for (let x = 0; x < w; x++) {
+ raw[row + 1 + x * 3] = c[0];
+ raw[row + 2 + x * 3] = c[1];
+ raw[row + 3 + x * 3] = c[2];
+ }
+ }
+ return Buffer.concat([
+ Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
+ chunk('IHDR', ihdr), chunk('IDAT', deflateSync(raw)), chunk('IEND', Buffer.alloc(0)),
+ ]);
+}
+
+describe('frameDistance', () => {
+ it('scores two identical captures at zero', () => {
+ expect(frameDistance(png(32, 18, [40, 80, 120]), png(32, 18, [40, 80, 120]))).toBe(0);
+ });
+
+ it('scores black against white at one', () => {
+ expect(frameDistance(png(32, 18, [0, 0, 0]), png(32, 18, [255, 255, 255]))).toBeCloseTo(1, 6);
+ });
+
+ it('refuses captures of different sizes — they letterbox differently', () => {
+ expect(() => frameDistance(png(32, 18, [0, 0, 0]), png(64, 18, [0, 0, 0])))
+ .toThrow(/differ in size/);
+ });
+
+ it('insets by default, so a play border never counts as a difference', () => {
+ expect(DEFAULT_INSET).toBeGreaterThan(0);
+ });
+});
diff --git a/tools/check-golden.mjs b/tools/check-golden.mjs
index a422cf3c5..63be03e68 100644
--- a/tools/check-golden.mjs
+++ b/tools/check-golden.mjs
@@ -16,7 +16,7 @@ import { existsSync, readFileSync } from 'node:fs';
import path from 'node:path';
import {
GOLDEN, CAPABILITIES, KNOWN_GAPS, TIERS, TARGETS,
- atTier, uncoveredCapabilities, nonGoldenExamples, projectDir,
+ atTier, uncoveredCapabilities, nonGoldenExamples, projectDir, parityFor,
} from './goldenProjects.mjs';
const problems = [];
@@ -53,6 +53,14 @@ for (const g of GOLDEN) {
for (const t of g.targets ?? []) {
if (!TARGETS.includes(t)) fail(`"${g.id}" names target "${t}" (have: ${TARGETS.join(', ')})`);
}
+ // Parity is the release's own claim, so opting out of it is a sentence, not a flag.
+ if (g.parityGap !== undefined && !(typeof g.parityGap === 'string' && g.parityGap.trim())) {
+ fail(`"${g.id}" sets parityGap without a reason — say why its frames cannot be compared`);
+ }
+ if (g.parity !== undefined && !(typeof g.parity === 'number' && g.parity > 0 && g.parity < 1)) {
+ fail(`"${g.id}" has parity ${g.parity} — expected a tolerance between 0 and 1`);
+ }
+ if (g.parity !== undefined && g.parityGap) fail(`"${g.id}" both sets a parity tolerance and opts out`);
if (!g.certifies?.length) fail(`"${g.id}" certifies nothing — say what it is in the suite for`);
for (const c of g.certifies ?? []) {
if (!CAPABILITIES.includes(c)) fail(`"${g.id}" certifies "${c}", which is not a declared capability`);
@@ -82,11 +90,13 @@ if (problems.length) {
process.exit(1);
}
+const noParity = GOLDEN.filter((g) => parityFor(g) === null);
const gaps = Object.keys(KNOWN_GAPS).length;
const pairs = TIERS.map((t) => `${t} ${atTier(t).length}`).join(' / ');
console.log(
`check-golden: ${GOLDEN.length} golden project(s) certify ${covered.size}/${CAPABILITIES.length} capabilities`
+ `, ${gaps} declared gap(s) — ok (${pairs})`,
);
+for (const g of noParity) console.log(` no editor/package comparison for ${g.id}: ${g.parityGap}`);
const rest = nonGoldenExamples();
if (rest.length) console.log(` ${rest.length} example(s) outside the corpus (smoke-tested, not certified)`);
diff --git a/tools/frameCompare.mjs b/tools/frameCompare.mjs
new file mode 100644
index 000000000..5d99d9c39
--- /dev/null
+++ b/tools/frameCompare.mjs
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file frameCompare.mjs — is this the same picture, coarsely?
+ *
+ * For asking whether a packaged game shows what the editor showed. Both sides
+ * must be captured at the SAME surface size — the caller's job, and an exact
+ * one, since the editor reports the size of its play surface and a package can
+ * be opened at any size. Given that, comparison is a resample and a score.
+ *
+ * Coarse on purpose: the failures worth catching (wrong scene, missing assets,
+ * lost configuration, wrong design resolution) move whole regions of the image,
+ * while drift between two independently-timed runs moves a few pixels.
+ */
+import { readPNG } from '../desktop/scripts/lib/editorDriver.mjs';
+
+export { readPNG };
+
+/** Fraction of each edge dropped before comparing — the editor draws a play
+ * border around its surface, and it is chrome, not game. */
+export const DEFAULT_INSET = 0.02;
+
+/**
+ * Box-average `img` onto a `cols × rows` grid as a flat RGB array, ignoring an
+ * `inset` fraction of every edge. Resampling is what lets two captures be
+ * compared at all; averaging is what makes the score ignore edge shimmer.
+ */
+export function downsample(img, cols, rows, inset = 0) {
+ const mx = Math.floor(img.w * inset);
+ const my = Math.floor(img.h * inset);
+ const x0 = mx, y0 = my;
+ const bw = Math.max(1, img.w - mx * 2);
+ const bh = Math.max(1, img.h - my * 2);
+ const out = new Float64Array(cols * rows * 3);
+ for (let cy = 0; cy < rows; cy++) {
+ const yA = y0 + Math.floor((cy * bh) / rows);
+ const yB = y0 + Math.max(Math.floor((cy * bh) / rows) + 1, Math.floor(((cy + 1) * bh) / rows));
+ for (let cx = 0; cx < cols; cx++) {
+ const xA = x0 + Math.floor((cx * bw) / cols);
+ const xB = x0 + Math.max(Math.floor((cx * bw) / cols) + 1, Math.floor(((cx + 1) * bw) / cols));
+ let r = 0, g = 0, b = 0, n = 0;
+ for (let y = yA; y < yB && y < y0 + bh; y++) {
+ for (let x = xA; x < xB && x < x0 + bw; x++) {
+ const p = img.px(x, y);
+ r += p[0]; g += p[1]; b += p[2]; n++;
+ }
+ }
+ const i = (cy * cols + cx) * 3;
+ out[i] = n ? r / n : 0;
+ out[i + 1] = n ? g / n : 0;
+ out[i + 2] = n ? b / n : 0;
+ }
+ }
+ return out;
+}
+
+/** Mean per-channel difference of two same-length grids, normalised to 0..1. */
+export function gridDistance(a, b) {
+ if (a.length !== b.length) throw new Error(`grid size mismatch: ${a.length} vs ${b.length}`);
+ let sum = 0;
+ for (let i = 0; i < a.length; i++) sum += Math.abs(a[i] - b[i]);
+ return sum / a.length / 255;
+}
+
+/**
+ * How different two captures are, 0 (identical) to 1. Refuses different sizes
+ * rather than stretching: two shapes letterbox differently and put screen-space
+ * UI elsewhere, so the score would measure the harness, not the game.
+ */
+export function frameDistance(pngA, pngB, { cols = 48, rows = 27, inset = DEFAULT_INSET } = {}) {
+ const a = readPNG(pngA);
+ const b = readPNG(pngB);
+ if (a.w !== b.w || a.h !== b.h) {
+ throw new Error(`captures differ in size (${a.w}x${a.h} vs ${b.w}x${b.h}) — compare like for like`);
+ }
+ return gridDistance(downsample(a, cols, rows, inset), downsample(b, cols, rows, inset));
+}
diff --git a/tools/goldenProjects.mjs b/tools/goldenProjects.mjs
index 63c52bfaa..14b4a50d1 100644
--- a/tools/goldenProjects.mjs
+++ b/tools/goldenProjects.mjs
@@ -28,6 +28,13 @@ export const TIERS = ['pr', 'nightly', 'release'];
/** Export targets a golden project can be asked to package + launch for. */
export const TARGETS = ['web', 'playable', 'desktop', 'wechat', 'android', 'ios'];
+/**
+ * How far a packaged frame may sit from the editor's frame of the same game.
+ * Measured: the same game scores ~0.01 and two different games 0.37–0.48, so
+ * this sits an order of magnitude below anything that is actually a difference.
+ */
+export const DEFAULT_PARITY = 0.06;
+
/**
* What the suite claims to cover. A capability here with no project behind it
* is a hole in the release argument, so the gate fails on one that is not in
@@ -66,6 +73,9 @@ export const KNOWN_GAPS = {
* The corpus. `certifies` is the claim, `targets` the packages that must build
* AND launch, `tier` the cheapest run that pays for it. Existing examples on
* purpose — a parallel suite would be a second set of games to keep alive.
+ *
+ * `parity` overrides {@link DEFAULT_PARITY} for a game whose opening seconds
+ * move too much to compare that tightly; `parityGap` opts out with a reason.
*/
export const GOLDEN = [
{
@@ -161,6 +171,12 @@ export function nonGoldenExamples() {
.sort();
}
+/** The parity tolerance in force for a project, or null when it opted out. */
+export function parityFor(g) {
+ if (g.parityGap) return null;
+ return typeof g.parity === 'number' ? g.parity : DEFAULT_PARITY;
+}
+
/** Absolute path to a golden project's directory. */
export function projectDir(id) {
return path.join(EXAMPLES, id);
diff --git a/tools/verify-golden.mjs b/tools/verify-golden.mjs
index 297ed56a7..c95e9ff83 100644
--- a/tools/verify-golden.mjs
+++ b/tools/verify-golden.mjs
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
- * @file verify-golden.mjs — carry each golden project through package → launch.
+ * @file verify-golden.mjs — carry each golden project through package → launch → compare.
*
* The registry says which projects a tier certifies and for which targets; this
* runs that matrix. Web and playable are driven here because both launch in the
@@ -16,9 +16,10 @@
* node tools/verify-golden.mjs --tier pr --shots <dir>
*/
import { spawnSync } from 'node:child_process';
-import { mkdirSync, rmSync } from 'node:fs';
+import { mkdirSync, rmSync, readFileSync, existsSync } from 'node:fs';
import path from 'node:path';
-import { atTier, projectDir, ROOT } from './goldenProjects.mjs';
+import { atTier, projectDir, parityFor, ROOT } from './goldenProjects.mjs';
+import { frameDistance, readPNG } from './frameCompare.mjs';
const argv = process.argv.slice(2);
const flag = (n, d) => {
@@ -30,6 +31,7 @@ const TIER = flag('tier', 'pr');
const ONLY = flag('only', '');
const SHOTS = flag('shots', '');
const WORK = flag('work', path.join(ROOT, '.golden'));
+const NO_PARITY = argv.includes('--no-parity');
/** Targets this runner owns; the rest belong to the native/desktop verifiers. */
const OWNED = new Set(['web', 'playable']);
@@ -45,6 +47,47 @@ const deferred = projects.flatMap((g) => g.targets.filter((t) => !OWNED.has(t)).
console.log(`golden ${TIER}: ${projects.length} project(s), ${pairs.length} pair(s) here`
+ (deferred.length ? `, ${deferred.length} left to the platform verifiers` : ''));
+/** The shape the game is authored for. A package derives its orientation gate
+ * from this, so the comparison surface has to agree with it. */
+function designAspect(id) {
+ try {
+ const r = JSON.parse(readFileSync(path.join(projectDir(id), 'project.esproject'), 'utf8')).designResolution;
+ if (r?.width > 0 && r?.height > 0) return { w: r.width, h: r.height };
+ } catch { /* fall through to the default shape */ }
+ return { w: 16, h: 9 };
+}
+
+/**
+ * The editor's frame of the running game, through the screenshot hook the UI
+ * shots use. The play panel takes the project's aspect first: a portrait game on
+ * a landscape surface draws the package's rotate gate against the editor's
+ * letterbox — a difference of surface, not of game.
+ */
+function captureEditorFrame(id, out) {
+ const a = designAspect(id);
+ const major = 820;
+ const panel = a.h >= a.w
+ ? { width: Math.round((major * a.w) / a.h), height: major }
+ : { width: major, height: Math.round((major * a.h) / a.w) };
+ const r = spawnSync('npx', ['electron', '.'], {
+ encoding: 'utf8',
+ cwd: DESKTOP,
+ env: {
+ ...process.env,
+ ESTELLA_SHOT: out,
+ ESTELLA_SHOT_PROJECT: projectDir(id),
+ ESTELLA_SHOT_PLAY: '1',
+ ESTELLA_SHOT_CROP: 'iframe[title="Game"]',
+ ESTELLA_SHOT_EVAL: `window.__estellaEditor.setPanelSize('viewport', ${JSON.stringify(panel)})`,
+ ESTELLA_WIN_W: '1500',
+ ESTELLA_WIN_H: '1040',
+ },
+ });
+ if (!existsSync(out)) return { ok: false, why: (r.stdout || r.stderr || '').trim().slice(-300) };
+ const png = readPNG(readFileSync(out));
+ return { ok: true, w: png.w, h: png.h };
+}
+
const results = [];
for (const { id, target } of pairs) {
const out = path.join(WORK, `${id}-${target}`);
@@ -61,17 +104,51 @@ for (const { id, target } of pairs) {
continue;
}
+ // Parity compares like for like, so the package is opened at exactly the size
+ // the editor's play surface came out — never a guessed one.
+ const golden = atTier(TIER).find((g) => g.id === id);
+ const tolerance = target === 'web' && !NO_PARITY ? parityFor(golden) : null;
+ const editorPng = path.join(WORK, `${id}-editor.png`);
+ const editor = tolerance != null ? captureEditorFrame(id, editorPng) : null;
+ if (editor && !editor.ok) {
+ results.push({ id, target, stage: 'editor-frame', ok: false, why: editor.why });
+ console.log(`✗ ${id} ${target} — the editor never produced a play frame`);
+ continue;
+ }
+
+ const packagePng = SHOTS ? path.join(SHOTS, `${id}-${target}.png`) : path.join(WORK, `${id}-${target}.png`);
const launch = spawnSync('npx', [
'electron', path.join('scripts', 'launch-export.mjs'),
- '--dir', out,
- ...(SHOTS ? ['--out', path.join(SHOTS, `${id}-${target}.png`)] : []),
+ '--dir', out, '--out', packagePng,
+ ...(editor ? ['--w', String(editor.w), '--h', String(editor.h)] : []),
], { encoding: 'utf8', cwd: DESKTOP });
const line = (launch.stdout || '').split('\n').find((l) => l.startsWith('✓') || l.startsWith('✗')) ?? '';
- const ok = launch.status === 0;
- results.push({ id, target, stage: 'launch', ok, why: ok ? '' : (line || (launch.stdout || launch.stderr || '').trim().slice(-300)) });
- console.log(`${ok ? '✓' : '✗'} ${id} ${target}${ok ? '' : ` — ${line || 'launch failed'}`}`);
- if (!ok) for (const l of (launch.stdout || '').split('\n').slice(-6)) if (l.trim()) console.log(` ${l}`);
+ if (launch.status !== 0) {
+ results.push({ id, target, stage: 'launch', ok: false, why: line || 'launch failed' });
+ console.log(`✗ ${id} ${target} — ${line || 'launch failed'}`);
+ for (const l of (launch.stdout || '').split('\n').slice(-6)) if (l.trim()) console.log(` ${l}`);
+ continue;
+ }
+
+ if (tolerance == null) {
+ results.push({ id, target, stage: 'launch', ok: true });
+ console.log(`✓ ${id} ${target}`);
+ continue;
+ }
+
+ let distance;
+ try {
+ distance = frameDistance(readFileSync(editorPng), readFileSync(packagePng));
+ } catch (e) {
+ results.push({ id, target, stage: 'parity', ok: false, why: e.message });
+ console.log(`✗ ${id} ${target} — parity: ${e.message}`);
+ continue;
+ }
+ const ok = distance <= tolerance;
+ results.push({ id, target, stage: 'parity', ok, why: ok ? '' : `distance ${distance.toFixed(4)} > ${tolerance}` });
+ console.log(`${ok ? '✓' : '✗'} ${id} ${target} — parity ${distance.toFixed(4)} (limit ${tolerance})`);
+ if (!ok) console.log(` the package is not showing what the editor showed; compare ${editorPng} and ${packagePng}`);
}
const bad = results.filter((r) => !r.ok);