Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(tools): a driver that plays the game instead of guessing at it

esengine402b99628fff
diff --git a/desktop/scripts/play-through.mjs b/desktop/scripts/play-through.mjs
new file mode 100644
index 000000000..0555c8da1
--- /dev/null
+++ b/desktop/scripts/play-through.mjs
@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file play-through.mjs — can the shipped game be played from start to finish?
+ *
+ * launch-export proves a package boots and answers input. It cannot prove a
+ * playthrough: its input is a list of "hold this key from frame A to frame B",
+ * which is open-loop. Any enemy standing in the way silently turns the rest of
+ * the script into someone walking into a wall, and the run still reports green
+ * because the frame is still lively. A route written that way is a route that
+ * has to be re-timed by hand every time the level changes.
+ *
+ * So this one plays: it asks the running game where things are (`?headless` →
+ * `__estellaCooked.probe`), steers toward the next waypoint, swings on the way,
+ * and moves on when it arrives. What it asserts is the game's own progression —
+ * the areas reached, in order — not a pixel.
+ *
+ * electron desktop/scripts/play-through.mjs --dir <exportDir> --route <file.json>
+ * --out <file.png> write the final frame
+ * --w / --h surface size (default 960x540)
+ * --budget <n> give up after this many frames (default 6000)
+ * --log <regex> also print console lines matching this
+ * --trace print what the driver decided, periodically
+ *
+ * A route is a list of legs: `{ area, goal, arriveWithin?, swing?, timeout? }`.
+ * `goal` is an entity name the probe can find, so a leg survives the level being
+ * re-authored — the door moves, the route does not.
+ *
+ * NOT a gate yet, and deliberately not wired into verify. Runs of the same
+ * package still disagree with each other: a leg occasionally reports arrival
+ * from a pickup that was not taken, and the run then fails later at a door that
+ * had every right to stay shut. Reading on a frame boundary and refusing a
+ * world mid-transition fixed most of it; what remains is timing-sensitive
+ * enough that adding console output changes the outcome, which is the shape of
+ * a race that has not been found yet.
+ */
+import { app, BrowserWindow } from 'electron';
+import http from 'node:http';
+import { readFile, writeFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import { onRendererConsole } from './rendererConsole.mjs';
+
+app.commandLine.appendSwitch('force-color-profile', 'srgb');
+
+const argv = process.argv.slice(2);
+const flag = (name, fallback) => {
+ const i = argv.indexOf(`--${name}`);
+ return i >= 0 && argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[i + 1] : fallback;
+};
+
+const DIR = path.resolve(flag('dir', ''));
+const ROUTE = flag('route', '');
+const OUT = flag('out', '');
+const W = Number(flag('w', '960'));
+const H = Number(flag('h', '540'));
+const BUDGET = Number(flag('budget', '6000'));
+const LOG = flag('log', '');
+const logRe = LOG ? new RegExp(LOG, 'i') : null;
+const TRACE = argv.includes('--trace');
+
+const MIME = {
+ '.html': 'text/html', '.js': 'text/javascript', '.mjs': 'text/javascript',
+ '.json': 'application/json', '.wasm': 'application/wasm', '.png': 'image/png',
+ '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.webp': 'image/webp', '.ktx2': 'image/ktx2',
+ '.mp3': 'audio/mpeg', '.ogg': 'audio/ogg', '.wav': 'audio/wav', '.m4a': 'audio/mp4',
+ '.mp4': 'video/mp4', '.esv': 'video/mp4', '.css': 'text/css', '.svg': 'image/svg+xml',
+ '.atlas': 'text/plain', '.skel': 'application/octet-stream', '.txt': 'text/plain',
+};
+
+function serve(root) {
+ const server = http.createServer(async (req, res) => {
+ try {
+ let rel = decodeURIComponent(new URL(req.url, 'http://x').pathname).replace(/^\/+/, '');
+ if (rel === '') rel = 'index.html';
+ const abs = path.join(root, rel);
+ if (!abs.startsWith(root)) { res.writeHead(403).end(); return; }
+ const bytes = await readFile(abs);
+ res.writeHead(200, { 'content-type': MIME[path.extname(abs).toLowerCase()] ?? 'application/octet-stream' })
+ .end(bytes);
+ } catch {
+ res.writeHead(404).end();
+ }
+ });
+ return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve(server)));
+}
+
+/**
+ * The page-side half: hold a set of keys for `frames` frames, then release.
+ * Keys go in as real DOM events, which is what the engine's input actually
+ * listens to — a driver that pokes state instead would be testing itself.
+ */
+const stepScript = (keys, frames, tapKey) => `
+(() => {
+ const target = document.querySelector('canvas') ?? window;
+ const send = (type, code) => {
+ const e = new KeyboardEvent(type, { code, key: code, bubbles: true, cancelable: true });
+ target.dispatchEvent(e);
+ if (target !== window) window.dispatchEvent(new KeyboardEvent(type, { code, key: code, bubbles: true }));
+ };
+ const held = ${JSON.stringify(keys)};
+ for (const k of held) send('keydown', k);
+ ${tapKey ? `send('keydown', ${JSON.stringify(tapKey)});` : ''}
+ return new Promise((resolve) => {
+ let n = 0;
+ const tick = () => {
+ n++;
+ ${tapKey ? `if (n === 2) send('keyup', ${JSON.stringify(tapKey)});` : ''}
+ if (n >= ${frames}) {
+ for (const k of held) send('keyup', k);
+ return resolve(true);
+ }
+ requestAnimationFrame(tick);
+ };
+ requestAnimationFrame(tick);
+ });
+})()
+`;
+
+/** Which keys walk from `from` toward `to` on the ground plane. */
+function keysToward(from, to) {
+ const dx = to.x - from.x;
+ const dy = to.y - from.y;
+ const keys = [];
+ // The dead band keeps a driver from stuttering between two opposite keys when
+ // it is already lined up on one axis.
+ if (Math.abs(dx) > 40) keys.push(dx > 0 ? 'KeyD' : 'KeyA');
+ if (Math.abs(dy) > 40) keys.push(dy > 0 ? 'KeyW' : 'KeyS');
+ return keys;
+}
+
+async function main() {
+ if (!DIR || !existsSync(path.join(DIR, 'index.html'))) {
+ console.error(`✗ no index.html under ${DIR || '(--dir not given)'}`);
+ app.exit(2);
+ return;
+ }
+ if (!ROUTE || !existsSync(ROUTE)) {
+ console.error(`✗ no route file at ${ROUTE || '(--route not given)'}`);
+ app.exit(2);
+ return;
+ }
+ const route = JSON.parse(await readFile(ROUTE, 'utf8'));
+ const server = await serve(DIR);
+ const base = `http://127.0.0.1:${server.address().port}/?headless`;
+
+ const win = new BrowserWindow({
+ width: W, height: H, useContentSize: true, show: false,
+ webPreferences: { offscreen: false, backgroundThrottling: false },
+ });
+ await win.webContents.session.clearCache();
+
+ const errors = [];
+ const stop = onRendererConsole(win.webContents, (msg) => {
+ if (/error|uncaught|failed/i.test(msg)) errors.push(msg.slice(0, 300));
+ if (msg.startsWith('[engine]') || logRe?.test(msg)) console.log(` ${msg}`);
+ });
+
+ await win.loadURL(base);
+ const exec = (js) => win.webContents.executeJavaScript(js);
+
+ // The handle appears as soon as the host builds it, which is before the first
+ // scene has spawned anything — asking then is asking an empty world.
+ let ready = false;
+ for (let i = 0; i < 300 && !ready; i++) {
+ ready = await exec(`(() => {
+ const c = window.__estellaCooked;
+ if (!c || !c.probe) return false;
+ const s = c.probe(['Lyra_Player']);
+ return !!(s.scene && s.at.Lyra_Player);
+ })()`).catch(() => false);
+ if (!ready) await new Promise((r) => setTimeout(r, 100));
+ }
+ if (!ready) {
+ console.log('✗ the package never exposed a probe — is this a cooked build served with ?headless');
+ stop(); server.close(); app.exit(1);
+ return;
+ }
+
+ // Read on a frame boundary. Poked from outside the loop, a read can land
+ // between the systems that despawn something and the ones that replace it,
+ // and every such read is a world nobody was ever in.
+ const probe = (names) => exec(
+ `new Promise((r) => requestAnimationFrame(() => r(window.__estellaCooked.probe(${JSON.stringify(names)}))))`,
+ );
+ const pathTo = (goal) => exec(`window.__estellaCooked.pathBetween("Lyra_Player", ${JSON.stringify(goal)})`);
+ // Every goal the route names, before a step is taken: a leg whose goal never
+ // existed would otherwise report as "already done" and the run would fail
+ // later, somewhere else, for a reason that is not the reason.
+ const goals = [...new Set(route.legs.map((l) => l.goal))];
+ const opening = await probe(goals);
+ const missing = goals.filter((g) => !(g in opening.at));
+ console.log(` route names ${goals.length} goal(s); ${goals.length - missing.length} are in ${opening.scene}`
+ + (missing.length ? `, not yet: ${missing.join(', ')}` : ''));
+
+ const STEP = 8;
+ // Nothing vanishes into a pack from across the room: a goal that goes while
+ // Lyra is nowhere near it went for some other reason, and crediting that is
+ // how a run reports legs it never walked.
+ const REACH = 220;
+ let frames = 0;
+ const done = [];
+ let failure = null;
+
+ for (const leg of route.legs) {
+ const label = `${leg.area}:${leg.goal}`;
+ const timeout = Number(leg.timeout ?? 1800);
+ const arriveWithin = Number(leg.arriveWithin ?? 90);
+ let spent = 0;
+ let arrived = false;
+ let sawArea = false;
+ let sawGoal = false;
+ let lastGap = Infinity;
+ let plan = [];
+ let replan = 0;
+ let last = null;
+
+ // Something picked up on the way to somewhere else is still picked up. A
+ // leg whose goal is already gone before it starts is a leg already walked —
+ // and a goal that is merely misspelled fails the legs that depend on it.
+ const opening = await probe([leg.goal, 'Lyra_Player']);
+ if (opening.at.Lyra_Player && !opening.at[leg.goal] && !leg.throughDoor) {
+ console.log(` ${label} was already done (probe saw ${JSON.stringify(Object.keys(opening.at))} in ${opening.scene})`);
+ done.push(label);
+ continue;
+ }
+
+ while (spent < timeout && frames < BUDGET) {
+ const state = await probe([leg.goal, 'Lyra_Player']);
+ // Nothing is true about a world that is halfway through being replaced.
+ if (state.transitioning) { await exec(stepScript([], STEP, null)); spent += STEP; frames += STEP; continue; }
+ if (state.scene === leg.area) sawArea = true;
+ // The leg is over when the area it was aiming for has been left behind:
+ // walking into a door IS the arrival, and the door is gone by the time
+ // anyone could measure standing next to it.
+ if (sawArea && state.scene !== leg.area) { arrived = true; break; }
+
+ const goal = state.at[leg.goal];
+ const me = state.at.Lyra_Player;
+ // A goal that was there and now is not was reached — taken, or killed —
+ // but only while Lyra is beside it: an area rebuilding after a death
+ // empties the world for a moment, and that is not an arrival.
+ if (sawGoal && !goal && me && lastGap < REACH && !leg.throughDoor) { arrived = true; break; }
+ if (goal) sawGoal = true;
+ if (me && goal) {
+ const gap = Math.hypot(goal.x - me.x, goal.y - me.y);
+ last = { me, goal, gap };
+ lastGap = gap;
+ if (gap <= arriveWithin && !leg.throughDoor) { arrived = true; break; }
+ // Re-planned every few steps rather than followed to the end: the route
+ // is walked with enemies pushing back, so a plan is a heading, not a rail.
+ if (!plan.length || replan <= 0) {
+ plan = (await pathTo(leg.goal)) ?? [];
+ replan = 40;
+ }
+ replan -= STEP;
+ while (plan.length && Math.hypot(plan[0].x - me.x, plan[0].y - me.y) < 70) plan.shift();
+ const aim = plan[0] ?? goal;
+ const keys = keysToward(me, aim);
+ if (TRACE && spent % 160 === 0) {
+ console.log(` ${label} @${spent}: at ${Math.round(me.x)},${Math.round(me.y)} `
+ + `aim ${Math.round(aim.x)},${Math.round(aim.y)} (${plan.length} waypoints) keys ${keys.join('+') || 'none'} gap ${Math.round(gap)}`);
+ }
+ await exec(stepScript(keys, STEP, leg.swing ? 'Space' : null));
+ } else {
+ // Between areas: the next scene has not handed out its entities yet.
+ await exec(stepScript([], STEP, null));
+ }
+ spent += STEP;
+ frames += STEP;
+ }
+
+ if (!arrived) {
+ failure = `${label} — ${spent >= timeout ? 'ran out of leg budget' : 'ran out of run budget'} after ${spent} frames`
+ + (last ? `; last seen ${Math.round(last.me.x)},${Math.round(last.me.y)} with the goal at `
+ + `${Math.round(last.goal.x)},${Math.round(last.goal.y)} (${Math.round(last.gap)} away)` : '; never saw both');
+ break;
+ }
+ done.push(label);
+ console.log(` reached ${label} (${frames} frames in)`);
+ }
+
+ if (OUT) await writeFile(OUT, (await win.webContents.capturePage()).toPNG());
+ const final = await probe([]).catch(() => ({ scene: null }));
+ stop();
+ server.close();
+
+ const ok = !failure && errors.length === 0;
+ console.log(`${ok ? '✓' : '✗'} ${path.basename(DIR)} — ${done.length}/${route.legs.length} legs, `
+ + `${frames} frames, ended in ${final.scene ?? '(nothing)'}, errors=${errors.length}`);
+ if (failure) console.log(` ${failure}`);
+ for (const e of errors.slice(0, 5)) console.log(` ${e}`);
+ app.exit(ok ? 0 : 1);
+}
+
+app.whenReady().then(main).catch((e) => {
+ console.error('✗ play-through failed:', e);
+ app.exit(2);
+});
diff --git a/desktop/src/gameHost.ts b/desktop/src/gameHost.ts
index 659953a23..bc07096cd 100644
--- a/desktop/src/gameHost.ts
+++ b/desktop/src/gameHost.ts
@@ -15,7 +15,7 @@ import {
createWebApp, setEditorMode, setPlayMode, Assets,
indexPackagedManifest, createPackagedAssetSource, applyAssetRefResolvers, initRuntime,
HttpBackend, fetchDecodePixels, registerPackagedSideModules,
- packagedAppOptions, packagedRuntimeInit,
+ packagedAppOptions, packagedRuntimeInit, Transform, SceneManager, Nav,
} from 'esengine';
import type { ESEngineModule, SceneData, AddressableManifest, PackagedGameConfig } from 'esengine';
@@ -94,6 +94,47 @@ async function boot(): Promise<void> {
gl.readPixels(0, 0, w, h, gl.RGBA, gl.UNSIGNED_BYTE, rgba);
return { width: w, height: h, rgba };
},
+ /**
+ * Where things are, for a driver that has to play rather than just look.
+ * Capture answers "did it draw"; walking a route needs "am I there yet",
+ * and guessing that from frame counts breaks on the first enemy in the
+ * way. Named entities only, and only the names asked for.
+ */
+ probe(names: string[] = []): {
+ scene: string | null;
+ transitioning: boolean;
+ at: Record<string, { x: number; y: number }>;
+ } {
+ const at: Record<string, { x: number; y: number }> = {};
+ for (const name of names) {
+ const entity = app.world.findEntityByName(name);
+ if (entity === null || !app.world.has(entity, Transform)) continue;
+ const p = app.world.get(entity, Transform).position;
+ at[name] = { x: p.x, y: p.y };
+ }
+ const scenes = app.hasResource(SceneManager) ? app.getResource(SceneManager) : null;
+ // A scene arrives entity by entity, and a driver polling from outside
+ // the frame loop can land in the middle of that. Half a world reads as
+ // "the thing I was walking to is gone", which is a lie with a cost.
+ return { scene: scenes?.getActive() ?? null, transitioning: scenes?.isTransitioning() ?? false, at };
+ },
+ /**
+ * A way from one named entity to another, over the same navigation grid
+ * the game's own enemies walk. A driver that steers straight at its goal
+ * is a driver that walks into the first wall between them; asking the
+ * game means the route stays right when the level is re-authored.
+ */
+ pathBetween(fromName: string, toName: string): Array<{ x: number; y: number }> | null {
+ if (!app.hasResource(Nav)) return null;
+ const ends = [fromName, toName].map((name) => {
+ const entity = app.world.findEntityByName(name);
+ if (entity === null || !app.world.has(entity, Transform)) return null;
+ const p = app.world.get(entity, Transform).position;
+ return { x: p.x, y: p.y };
+ });
+ if (!ends[0] || !ends[1]) return null;
+ return app.getResource(Nav).findWorldPath(ends[0], ends[1]);
+ },
/** Drive a hot update against a served (CDN) manifest: fetch + diff + apply.
* Rebinding the visuals is the game's job (via Assets.onInvalidate); a
* driver settles frames after this before re-capturing. */
diff --git a/examples/celestial-heights/playthrough.json b/examples/celestial-heights/playthrough.json
new file mode 100644
index 000000000..fcfc9de1c
--- /dev/null
+++ b/examples/celestial-heights/playthrough.json
@@ -0,0 +1,67 @@
+{
+ "name": "Celestial Heights \u2014 start to Vesper",
+ "legs": [
+ {
+ "area": "main",
+ "goal": "Drop_petal_20_21",
+ "swing": true,
+ "timeout": 2400
+ },
+ {
+ "area": "main",
+ "goal": "Drop_petal_30_4",
+ "swing": true,
+ "timeout": 2400
+ },
+ {
+ "area": "main",
+ "goal": "Drop_petal_43_5",
+ "swing": true,
+ "timeout": 1200
+ },
+ {
+ "area": "main",
+ "goal": "Gate",
+ "swing": true,
+ "throughDoor": true,
+ "timeout": 2400
+ },
+ {
+ "area": "moonlit-gallery",
+ "goal": "Drop_shard_22_5",
+ "swing": true,
+ "timeout": 2400
+ },
+ {
+ "area": "moonlit-gallery",
+ "goal": "Drop_shard_22_34",
+ "swing": true,
+ "timeout": 2400
+ },
+ {
+ "area": "moonlit-gallery",
+ "goal": "Drop_shard_41_34",
+ "swing": true,
+ "timeout": 2400
+ },
+ {
+ "area": "moonlit-gallery",
+ "goal": "Drop_shard_53_24",
+ "swing": true,
+ "timeout": 2400
+ },
+ {
+ "area": "moonlit-gallery",
+ "goal": "Gate",
+ "swing": true,
+ "throughDoor": true,
+ "timeout": 2400
+ },
+ {
+ "area": "spire",
+ "goal": "Vesper",
+ "swing": true,
+ "timeout": 3000
+ }
+ ]
+}
\ No newline at end of file