Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(editor): headless automation/observation surface + render verification

esengine35eae5a10aab
diff --git a/desktop/headless.html b/desktop/headless.html
new file mode 100644
index 000000000..156a61b14
--- /dev/null
+++ b/desktop/headless.html
@@ -0,0 +1,17 @@
+<!doctype html>
+<html lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <!-- Mirror the editor shell's CSP exactly so the headless host is a faithful
+ reproduction of the live renderer (same embind 'unsafe-eval', same asset
+ origins) — a render that fails here for a CSP reason would fail there too. -->
+ <meta
+ http-equiv="Content-Security-Policy"
+ content="default-src 'self'; script-src 'self' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: estella:; font-src 'self' data:; connect-src 'self' ws: estella:; worker-src 'self' blob:;"
+ />
+ <title>Estella Headless</title>
+ </head>
+ <body>
+ <script type="module" src="/src/headless.ts"></script>
+ </body>
+</html>
diff --git a/desktop/package.json b/desktop/package.json
index 2ac10a35a..8d18d0f1a 100644
--- a/desktop/package.json
+++ b/desktop/package.json
@@ -11,7 +11,8 @@
"preview": "vite preview",
"typecheck": "tsc --noEmit",
"test": "vitest run",
- "test:watch": "vitest"
+ "test:watch": "vitest",
+ "verify:render": "vite build && electron scripts/headless-verify.mjs"
},
"dependencies": {
"esengine": "workspace:*",
diff --git a/desktop/scripts/headless-verify.mjs b/desktop/scripts/headless-verify.mjs
new file mode 100644
index 000000000..b00f7eed2
--- /dev/null
+++ b/desktop/scripts/headless-verify.mjs
@@ -0,0 +1,111 @@
+/**
+ * @file Headless render verification runner (docs/REARCH_EDITOR_AUTOMATION.md P1).
+ *
+ * A standalone Electron entry — NOT the editor's production main — that proves a
+ * scene renders by driving the headless render host: it static-serves the built
+ * dist/ over loopback http (an http origin is required; the engine resolves its
+ * wasm glue from location.origin, which file:// roots wrong), opens a show:false
+ * window on headless.html, then loadScene → step → captureViewport and asserts
+ * the frame is actually rasterized (color variation, not a uniform clear).
+ *
+ * This is the repeatable, in-repo form of the capability — `pnpm verify:render`.
+ * It runs by Electron (for a real WebGL2 context), so it imports only `electron`
+ * and node built-ins. Parametrize via env to point it at other scenes:
+ * ESTELLA_VERIFY_SCENE scene url (default the dev sprite scene)
+ * ESTELLA_VERIFY_MANIFEST texture manifest url
+ * ESTELLA_VERIFY_W / _H capture size (default 640×480)
+ * ESTELLA_VERIFY_STEPS fixed-dt frames to advance before capture (default 30)
+ */
+import { app, BrowserWindow } from 'electron';
+import http from 'node:http';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const DIST = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'dist');
+
+const SCENE = process.env.ESTELLA_VERIFY_SCENE ?? '/scenes/sprite-rendering.esscene';
+const MANIFEST = process.env.ESTELLA_VERIFY_MANIFEST ?? '/scenes/sprite-rendering.textures.json';
+const W = Number(process.env.ESTELLA_VERIFY_W) || 640;
+const H = Number(process.env.ESTELLA_VERIFY_H) || 480;
+const STEPS = Number(process.env.ESTELLA_VERIFY_STEPS) || 30;
+
+// Headless / GPU-less (CI) WebGL2 falls back to SwiftShader; harmless with a GPU.
+app.commandLine.appendSwitch('enable-unsafe-swiftshader');
+process.env.ELECTRON_DISABLE_SECURITY_WARNINGS = 'true';
+
+const MIME = {
+ '.html': 'text/html', '.js': 'text/javascript', '.mjs': 'text/javascript',
+ '.css': 'text/css', '.json': 'application/json', '.esscene': 'application/json',
+ '.wasm': 'application/wasm', '.png': 'image/png', '.jpg': 'image/jpeg',
+ '.webp': 'image/webp', '.woff': 'font/woff', '.woff2': 'font/woff2', '.svg': 'image/svg+xml',
+};
+
+function serveDist() {
+ const server = http.createServer(async (req, res) => {
+ try {
+ const rel = decodeURIComponent(new URL(req.url, 'http://x').pathname).replace(/^\/+/, '') || 'index.html';
+ const abs = path.join(DIST, rel);
+ if (!abs.startsWith(DIST)) {
+ res.writeHead(403).end();
+ return;
+ }
+ const bytes = await readFile(abs);
+ res.writeHead(200, { 'content-type': MIME[path.extname(abs).toLowerCase()] ?? 'application/octet-stream' });
+ res.end(bytes);
+ } catch {
+ res.writeHead(404).end('not found');
+ }
+ });
+ return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve(server)));
+}
+
+function finish(result, server) {
+ const ok = result.ok && result.capture?.rendered;
+ console.log(`\n[verify:render] ${ok ? 'PASS' : 'FAIL'} — ${SCENE}`);
+ console.log('DRIVE_RESULT ' + JSON.stringify(result));
+ process.exitCode = ok ? 0 : 1;
+ try {
+ server?.close();
+ } catch {
+ /* ignore */
+ }
+ app.quit();
+}
+
+app.whenReady().then(async () => {
+ let server;
+ try {
+ server = await serveDist();
+ const url = `http://127.0.0.1:${server.address().port}/headless.html?w=${W}&h=${H}`;
+
+ const win = new BrowserWindow({ show: false, width: W, height: H, webPreferences: { offscreen: false } });
+ win.webContents.on('console-message', (...args) => {
+ const msg = args.map((a) => (a && typeof a === 'object' ? a.message ?? '' : String(a))).join(' ');
+ if (/error|fail|unwind|exception|webgl/i.test(msg)) console.log('[renderer]', msg.slice(0, 240));
+ });
+ await win.loadURL(url);
+
+ const exec = (code) => win.webContents.executeJavaScript(code, true);
+ await exec('window.__estellaHeadless.ready');
+ const entityCount = await exec(
+ `window.__estellaHeadless.api.loadScene(${JSON.stringify(SCENE)}, ${JSON.stringify(MANIFEST)})`,
+ );
+ await exec(`window.__estellaHeadless.api.step(${STEPS}, 1 / 60)`);
+ const capture = await exec(`(() => {
+ const c = window.__estellaHeadless.api.captureViewport();
+ const px = c.rgba; const min = [255, 255, 255], max = [0, 0, 0]; let nonZero = 0;
+ for (let i = 0; i < px.length; i += 4) {
+ for (let k = 0; k < 3; k++) { const v = px[i + k]; if (v < min[k]) min[k] = v; if (v > max[k]) max[k] = v; }
+ if (px[i] | px[i + 1] | px[i + 2]) nonZero++;
+ }
+ const spread = (max[0] - min[0]) + (max[1] - min[1]) + (max[2] - min[2]);
+ return { w: c.width, h: c.height, totalPixels: px.length / 4, nonZeroPixels: nonZero, min, max, spread, rendered: spread > 16 };
+ })()`);
+ finish({ ok: true, entityCount, capture }, server);
+ } catch (e) {
+ finish({ ok: false, error: String((e && e.stack) || e) }, server);
+ }
+});
+
+setTimeout(() => finish({ ok: false, error: 'timeout' }), 45000);
diff --git a/desktop/src/engine/EditorControlSurface.ts b/desktop/src/engine/EditorControlSurface.ts
new file mode 100644
index 000000000..a88fc9cf2
--- /dev/null
+++ b/desktop/src/engine/EditorControlSurface.ts
@@ -0,0 +1,174 @@
+/**
+ * @file EditorControlSurface.ts
+ * @brief The single canonical programmatic entry to the editor — lifecycle,
+ * commands, queries, and observation — composed over the existing
+ * engine modules. One surface, three consumers: the React UI, the
+ * headless verification host, and (later) the editor MCP server, which
+ * is a transport adapter over this object rather than a parallel API.
+ *
+ * Design and phasing: docs/REARCH_EDITOR_AUTOMATION.md. This adds no new truth —
+ * commands route through SceneCommands (the undoable write boundary), reads
+ * through SceneQuery / SceneModel, and observation reads the live canvas/World.
+ */
+import type {
+ EntityId,
+ InspectorComponent,
+ InspectorFieldType,
+ InspectorFieldValue,
+ NodeKind,
+ SceneNode,
+} from '@/types';
+import type { SceneData } from 'esengine';
+import { EngineHost } from './EngineHost';
+import { SceneCommands } from './SceneCommands';
+import { SceneQuery } from './SceneQuery';
+import { SceneModel } from './SceneModel';
+import { EditorHistory } from './EditorHistory';
+
+/** A captured viewport frame: raw RGBA pixels (GL order: bottom-up rows). */
+export interface ViewportCapture {
+ rgba: Uint8Array;
+ width: number;
+ height: number;
+}
+
+export const EditorControlSurface = {
+ // =========================================================================
+ // Lifecycle — drive the engine deterministically and switch run mode
+ // =========================================================================
+
+ /**
+ * Load a scene into the live World, resolving `@uuid:` asset refs via the
+ * manifest. Returns the spawned entity count. The async-driving counterpart to
+ * opening a scene in the UI; used by verification to set up a known scene.
+ */
+ async loadScene(sceneUrl: string, manifestUrl?: string): Promise<number> {
+ return EngineHost.loadScene(sceneUrl, manifestUrl);
+ },
+
+ /**
+ * Switch edit↔play. Play snapshots the scene; Stop restores it (gameplay never
+ * dirties the edit scene). Returns true if a play→edit restore happened (entity
+ * ids changed — drop any cached selection).
+ */
+ setRunMode(playing: boolean, paused = false): boolean {
+ return EngineHost.setRunMode(playing, paused);
+ },
+
+ /**
+ * Advance the engine by exactly `frames` fixed-delta ticks — no rAF, no
+ * wall-clock — so a subsequent capture is reproducible. Use after loadScene /
+ * setRunMode and before captureViewport. Drives the engine directly, so it
+ * belongs to a host that does NOT run app.run()'s loop (the headless host).
+ */
+ async step(frames = 1, dt = 1 / 60): Promise<void> {
+ for (let i = 0; i < frames; i++) await EngineHost.tick(dt);
+ },
+
+ undo(): void {
+ EditorHistory.undo();
+ },
+ redo(): void {
+ EditorHistory.redo();
+ },
+ canUndo(): boolean {
+ return EditorHistory.canUndo();
+ },
+ canRedo(): boolean {
+ return EditorHistory.canRedo();
+ },
+
+ // =========================================================================
+ // Commands — mutations, all undoable, through the SceneCommands write door
+ // =========================================================================
+
+ addEntity(): EntityId | null {
+ return SceneCommands.addEntity();
+ },
+ deleteEntity(id: EntityId): void {
+ SceneCommands.deleteEntity(id);
+ },
+ duplicateEntity(id: EntityId): EntityId | null {
+ return SceneCommands.duplicateEntity(id);
+ },
+ renameEntity(id: EntityId, name: string): void {
+ SceneCommands.renameEntity(id, name);
+ },
+ setField(
+ entity: EntityId,
+ component: string,
+ key: string,
+ type: InspectorFieldType,
+ value: InspectorFieldValue,
+ ): void {
+ SceneCommands.setField(entity, component, key, type, value);
+ },
+ setEntityXY(id: EntityId, x: number, y: number): void {
+ SceneCommands.setEntityXY(id, x, y);
+ },
+ /** Coalesce a burst of setField writes (a drag) into one undo step. */
+ beginGesture(label: string): void {
+ SceneCommands.beginGesture(label);
+ },
+ endGesture(): void {
+ SceneCommands.endGesture();
+ },
+
+ // =========================================================================
+ // Queries — read-only reflection of the live World / scene truth
+ // =========================================================================
+
+ worldVersion(): number {
+ return SceneQuery.worldVersion();
+ },
+ getSceneTree(): SceneNode[] {
+ return SceneQuery.readSceneTree();
+ },
+ getEntity(id: EntityId): { name: string; kind: NodeKind; components: string[] } | null {
+ return SceneQuery.readEntity(id);
+ },
+ getInspector(entity: EntityId): InspectorComponent[] {
+ return SceneQuery.readInspector(entity);
+ },
+ getFieldValue(entity: EntityId, component: string, key: string): InspectorFieldValue | null {
+ return SceneQuery.getFieldValue(entity, component, key);
+ },
+ /** The lossless JSON-first scene truth (deep clone), or null if none loaded. */
+ serializeScene(): SceneData | null {
+ return SceneModel.serialize();
+ },
+
+ // =========================================================================
+ // Observation — runtime evidence for verification (the headless-gap closer)
+ // =========================================================================
+
+ /**
+ * Read the rendered viewport back as raw RGBA pixels. Requires a render host
+ * with a live WebGL2 canvas (the live editor viewport, or the headless editor
+ * window) — throws in a context without one (e.g. the pure-node test harness).
+ * Capture right after a step(): the drawing buffer is valid until the next
+ * frame, and with no rAF loop the headless host's buffer persists.
+ */
+ captureViewport(): ViewportCapture {
+ const canvas = EngineHost.canvas;
+ if (!canvas) {
+ throw new Error(
+ 'captureViewport requires a render host (no canvas) — run under the live viewport or the headless editor window',
+ );
+ }
+ const gl = canvas.getContext('webgl2') as WebGL2RenderingContext | null;
+ if (!gl) throw new Error('captureViewport: no WebGL2 context on the viewport canvas');
+ const width = canvas.width;
+ const height = canvas.height;
+ const rgba = new Uint8Array(width * height * 4);
+ gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, rgba);
+ return { rgba, width, height };
+ },
+
+ /** Live counts for quick assertions (entity count is headless-friendly). */
+ getStats(): { entities: number } {
+ return { entities: EngineHost.world?.getAllEntities().length ?? 0 };
+ },
+};
+
+export type EditorControlSurfaceT = typeof EditorControlSurface;
diff --git a/desktop/src/engine/EngineHost.ts b/desktop/src/engine/EngineHost.ts
index 031828e14..f497831be 100644
--- a/desktop/src/engine/EngineHost.ts
+++ b/desktop/src/engine/EngineHost.ts
@@ -162,6 +162,29 @@ class EngineHostImpl {
return restored;
}
+ // — Headless / automation drive (see docs/REARCH_EDITOR_AUTOMATION.md) —
+ // The live editor lets the engine drive its own rAF loop (app.run()); a
+ // headless render host or a verification/automation driver instead advances
+ // frames itself, so it can capture a deterministic, reproducible frame. These
+ // mediate the private App so encapsulation (the two-door world access) holds.
+
+ /**
+ * Advance the engine by exactly one frame with a fixed delta — no rAF, no
+ * wall-clock. The same per-frame work app.run()'s loop does, driven manually.
+ * No-op until booted. Do not mix with a running app.run() loop.
+ */
+ async tick(delta: number): Promise<void> {
+ await this.app_?.tick(delta);
+ }
+
+ /**
+ * Bulk-load a scene into the live World, resolving asset refs via the manifest
+ * (same door the boot bootstrap uses). Returns the spawned entity count.
+ */
+ async loadScene(sceneUrl: string, manifestUrl?: string): Promise<number> {
+ return this.app_ ? SceneLoader.loadInto(this.app_, sceneUrl, manifestUrl) : 0;
+ }
+
private resize() {
const canvas = this.canvas_;
const parent = canvas?.parentElement;
@@ -177,76 +200,111 @@ class EngineHostImpl {
if (this.booted) return;
this.booted = true;
this.setStatus('booting');
-
try {
- // Early build-consistency check: compare the wasm's stamped manifest
- // (variant / ABI / provenance) against this SDK before the heavy
- // instantiate. Advisory only — the runtime bridge handshake is the
- // authoritative fatal layout check (reads the real binary).
- const guard = await checkEngineBuild();
- if (guard.level === 'warn') console.warn('[engine]', guard.message);
- else console.info('[engine]', guard.message);
-
- // The glue lives under public/. Vite's import-analysis rejects static
- // imports of public files, so build the specifier at runtime from the
- // origin — non-analyzable, so Vite emits a native dynamic import (allowed
- // by CSP script-src 'self', no eval needed).
- // NOTE: works in dev (http origin); production packaging will need a
- // custom protocol or relative base since file:// roots differently.
- const glueUrl = `${location.origin}/wasm/esengine.js`;
- const { default: createModule } = (await import(/* @vite-ignore */ glueUrl)) as {
- default: (options?: Record<string, unknown>) => Promise<ESEngineModule>;
- };
const canvas = this.ensureCanvas();
+ await this.bootCore(canvas, { runLoop: true, loadInitialScene: true });
+ } catch (err) {
+ this.swallowUnwind(err);
+ }
+ }
- const module = await createModule({
- canvas,
- // The glue resolves esengine.wasm relative to itself; pin it explicitly.
- locateFile: (path: string) => `/wasm/${path}`,
- print: (text: string) => console.log('[wasm]', text),
- printErr: (text: string) => console.warn('[wasm]', text),
- });
- this.module_ = module;
-
- // Bind the renderer to a context WE create on this canvas (rather than the
- // engine's default '#canvas' selector) so the viewport works embedded
- // under any element id. Mirrors the wechat runtime path.
- const gl = canvas.getContext('webgl2', {
- alpha: false,
- antialias: true,
- depth: true,
- stencil: true,
- premultipliedAlpha: false,
- }) as WebGL2RenderingContext | null;
- if (!gl) throw new Error('WebGL2 is not available in this renderer.');
-
- const glHandle = module.GL.registerContext(gl, {
- majorVersion: 2,
- minorVersion: 0,
- enableExtensionsByDefault: true,
- });
-
- const app = createWebApp(module, {
- glContextHandle: glHandle,
- getViewportSize: () => ({ width: canvas.width, height: canvas.height }),
- // The per-version spine side modules are served next to esengine.wasm
- // (same /wasm/ dir as locateFile above), so the web spine provider can
- // load 3.8/4.1 assets in the viewport, not just the engine-linked 4.2.
- wasmBaseUrl: '/wasm',
- });
- this.app_ = app;
-
- // Mark this an editor host and open in edit mode: gameplay systems
- // (particle/animation/physics/timeline/…, gated on env.playModeOnly) are
- // frozen so simulation doesn't fight edits, while render/transform/camera
- // keep ticking. Play mode is toggled later via setRunMode.
- setEditorMode(true);
- setPlayMode(false);
-
- // Register the reactive bridge before the scene spawns so the initial
- // entities push through too.
- SceneStore.install();
+ /**
+ * Boot the engine without a DOM viewport or a self-driving loop, for the
+ * headless render host (see docs/REARCH_EDITOR_AUTOMATION.md): a fixed-size
+ * offscreen canvas, no initial scene (the driver loads one), and frames
+ * advanced manually via tick() so captures are deterministic. Resolves once
+ * the engine is ready; the driver then does loadScene → step → captureViewport.
+ */
+ async bootHeadless(size: { width: number; height: number }): Promise<void> {
+ if (this.booted) return;
+ this.booted = true;
+ this.setStatus('booting');
+ try {
+ const canvas = this.ensureCanvas();
+ canvas.width = size.width;
+ canvas.height = size.height;
+ await this.bootCore(canvas, { runLoop: false, loadInitialScene: false });
+ } catch (err) {
+ this.swallowUnwind(err);
+ }
+ }
+ // The shared boot sequence: instantiate the wasm, bind a WebGL2 context, build
+ // the App, open in edit mode. The DOM viewport (boot) then drives the engine's
+ // own rAF loop and loads an initial scene; the headless host (bootHeadless)
+ // does neither — it advances frames via tick() and loads scenes on demand.
+ private async bootCore(
+ canvas: HTMLCanvasElement,
+ opts: { runLoop: boolean; loadInitialScene: boolean },
+ ) {
+ // Early build-consistency check: compare the wasm's stamped manifest
+ // (variant / ABI / provenance) against this SDK before the heavy
+ // instantiate. Advisory only — the runtime bridge handshake is the
+ // authoritative fatal layout check (reads the real binary).
+ const guard = await checkEngineBuild();
+ if (guard.level === 'warn') console.warn('[engine]', guard.message);
+ else console.info('[engine]', guard.message);
+
+ // The glue lives under public/. Vite's import-analysis rejects static
+ // imports of public files, so build the specifier at runtime from the
+ // origin — non-analyzable, so Vite emits a native dynamic import (allowed
+ // by CSP script-src 'self', no eval needed).
+ // NOTE: works in dev (http origin); production packaging will need a
+ // custom protocol or relative base since file:// roots differently.
+ const glueUrl = `${location.origin}/wasm/esengine.js`;
+ const { default: createModule } = (await import(/* @vite-ignore */ glueUrl)) as {
+ default: (options?: Record<string, unknown>) => Promise<ESEngineModule>;
+ };
+
+ const module = await createModule({
+ canvas,
+ // The glue resolves esengine.wasm relative to itself; pin it explicitly.
+ locateFile: (path: string) => `/wasm/${path}`,
+ print: (text: string) => console.log('[wasm]', text),
+ printErr: (text: string) => console.warn('[wasm]', text),
+ });
+ this.module_ = module;
+
+ // Bind the renderer to a context WE create on this canvas (rather than the
+ // engine's default '#canvas' selector) so the viewport works embedded
+ // under any element id. Mirrors the wechat runtime path.
+ const gl = canvas.getContext('webgl2', {
+ alpha: false,
+ antialias: true,
+ depth: true,
+ stencil: true,
+ premultipliedAlpha: false,
+ }) as WebGL2RenderingContext | null;
+ if (!gl) throw new Error('WebGL2 is not available in this renderer.');
+
+ const glHandle = module.GL.registerContext(gl, {
+ majorVersion: 2,
+ minorVersion: 0,
+ enableExtensionsByDefault: true,
+ });
+
+ const app = createWebApp(module, {
+ glContextHandle: glHandle,
+ getViewportSize: () => ({ width: canvas.width, height: canvas.height }),
+ // The per-version spine side modules are served next to esengine.wasm
+ // (same /wasm/ dir as locateFile above), so the web spine provider can
+ // load 3.8/4.1 assets in the viewport, not just the engine-linked 4.2.
+ wasmBaseUrl: '/wasm',
+ });
+ this.app_ = app;
+
+ // Mark this an editor host and open in edit mode: gameplay systems
+ // (particle/animation/physics/timeline/…, gated on env.playModeOnly) are
+ // frozen so simulation doesn't fight edits, while render/transform/camera
+ // keep ticking. Play mode is toggled later via setRunMode.
+ setEditorMode(true);
+ setPlayMode(false);
+
+ // Register the reactive bridge before the scene spawns so the initial
+ // entities push through too.
+ SceneStore.install();
+
+ if (opts.loadInitialScene) {
// Load the opened project's scene if the launcher set a bootstrap;
// otherwise the in-repo placeholder scene (dev). Falls back to the
// in-code scene on failure.
@@ -260,12 +318,12 @@ class EngineHostImpl {
console.warn('[engine] scene load failed; using placeholder', err);
this.setupScene(app);
}
+ }
- // Loop started — report ready before run() in case it never resolves.
- this.setStatus('ready');
+ // Report ready before run() (the DOM path) in case the loop never resolves.
+ this.setStatus('ready');
+ if (opts.runLoop) {
void Promise.resolve(app.run()).catch((err) => this.swallowUnwind(err));
- } catch (err) {
- this.swallowUnwind(err);
}
}
diff --git a/desktop/src/headless.ts b/desktop/src/headless.ts
new file mode 100644
index 000000000..abf226088
--- /dev/null
+++ b/desktop/src/headless.ts
@@ -0,0 +1,34 @@
+/**
+ * @file headless.ts
+ * @brief Headless boot entry for the editor's render host
+ * (docs/REARCH_EDITOR_AUTOMATION.md P1). Loaded by a show:false
+ * Electron window; boots the engine with no React UI and publishes
+ * EditorControlSurface on `window` so a driver — the main process via
+ * executeJavaScript today, the editor MCP server later — can open a
+ * scene, advance frames deterministically, and read the rendered pixels
+ * back for reproducible verification.
+ */
+import { EngineHost } from './engine/EngineHost';
+import { EditorControlSurface, type EditorControlSurfaceT } from './engine/EditorControlSurface';
+
+declare global {
+ interface Window {
+ /** Headless driving surface — present only in the headless render host. */
+ __estellaHeadless?: {
+ /** Resolves once the engine is booted and ready to load a scene. */
+ ready: Promise<void>;
+ api: EditorControlSurfaceT;
+ };
+ }
+}
+
+// Viewport size is driver-controlled via the query string (?w=&h=) so captures
+// are a known resolution; default to a common 16:9 frame.
+const params = new URLSearchParams(location.search);
+const width = Number(params.get('w')) || 1280;
+const height = Number(params.get('h')) || 720;
+
+window.__estellaHeadless = {
+ ready: EngineHost.bootHeadless({ width, height }),
+ api: EditorControlSurface,
+};
diff --git a/desktop/tests/editor-control-surface.test.ts b/desktop/tests/editor-control-surface.test.ts
new file mode 100644
index 000000000..c4c8705e9
--- /dev/null
+++ b/desktop/tests/editor-control-surface.test.ts
@@ -0,0 +1,96 @@
+/**
+ * @file Regression net for EditorControlSurface — the one canonical
+ * programmatic entry to the editor (docs/REARCH_EDITOR_AUTOMATION.md).
+ * Proves the surface delegates correctly to the command/query core and
+ * that step() drives deterministic ticks, all against a real headless
+ * World. captureViewport needs a WebGL2 canvas, so here we only assert it
+ * fails clearly without a render host (covered end-to-end by the headless
+ * editor window, not the pure-node harness).
+ *
+ * EngineHost (the boot singleton that needs a canvas) is mocked to a per-test
+ * headless World; tick() drives the real App so step() exercises the engine.
+ */
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+import { App, Transform } from 'esengine';
+import type { ESEngineModule } from 'esengine';
+import { loadWasmModule, HAS_WASM } from './helpers/loadWasm';
+
+// Per-test state, injected into the mocked EngineHost. `vi.hoisted` so the mock
+// factory (hoisted above imports) can close over it.
+const host = vi.hoisted(() => ({
+ world: null as unknown as App['world'],
+ app: null as unknown as App,
+ ticks: 0,
+ runMode: null as null | [boolean, boolean],
+}));
+
+vi.mock('@/engine/EngineHost', () => ({
+ EngineHost: {
+ mutableWorld: () => host.world,
+ get world() {
+ return host.world;
+ },
+ getResource: () => undefined,
+ get canvas() {
+ return null; // no render host in the node harness → captureViewport must fail clearly
+ },
+ tick: async (dt: number) => {
+ host.ticks++;
+ await host.app?.tick(dt);
+ },
+ setRunMode: (playing: boolean, paused: boolean) => {
+ host.runMode = [playing, paused];
+ return false;
+ },
+ loadScene: async () => 0,
+ },
+}));
+
+import { EditorControlSurface as API } from '@/engine/EditorControlSurface';
+import { EditorHistory } from '@/engine/EditorHistory';
+
+describe.skipIf(!HAS_WASM)('EditorControlSurface (headless World)', () => {
+ let module: ESEngineModule;
+ beforeAll(async () => {
+ module = await loadWasmModule();
+ });
+ beforeEach(() => {
+ const app = App.new();
+ const registry = new module.Registry();
+ app.connectCpp(registry as never, module);
+ host.world = app.world;
+ host.app = app;
+ host.ticks = 0;
+ host.runMode = null;
+ EditorHistory.clear();
+ });
+
+ it('addEntity surfaces through the scene tree and stats', () => {
+ const id = API.addEntity();
+ expect(id).not.toBeNull();
+ expect(API.getSceneTree().length).toBe(1);
+ expect(API.getStats().entities).toBe(1);
+ });
+
+ it('setField writes a component field; surface undo reverts it', () => {
+ const id = API.addEntity()!;
+ API.setField(id, 'Transform', 'position', 'vec3', [10, 20, 30]);
+ expect(host.world.get(id, Transform).position).toMatchObject({ x: 10, y: 20, z: 30 });
+ API.undo();
+ expect(host.world.get(id, Transform).position).toMatchObject({ x: 0, y: 0, z: 0 });
+ });
+
+ it('step(n) drives exactly n deterministic ticks', async () => {
+ await API.step(3, 1 / 60);
+ expect(host.ticks).toBe(3);
+ });
+
+ it('setRunMode delegates play/pause state to the host', () => {
+ API.setRunMode(true, true);
+ expect(host.runMode).toEqual([true, true]);
+ });
+
+ it('captureViewport fails clearly without a render host', () => {
+ expect(() => API.captureViewport()).toThrow(/render host/);
+ });
+});
diff --git a/desktop/vite.config.ts b/desktop/vite.config.ts
index 5cbc90664..3d3551588 100644
--- a/desktop/vite.config.ts
+++ b/desktop/vite.config.ts
@@ -12,6 +12,16 @@ export default defineConfig({
'@': fileURLToPath(new URL('./src', import.meta.url)),
},
},
+ build: {
+ rollupOptions: {
+ // Two renderer entries: the editor shell (index) and the headless render
+ // host (headless) used for automation/verification — see headless.html.
+ input: {
+ index: fileURLToPath(new URL('./index.html', import.meta.url)),
+ headless: fileURLToPath(new URL('./headless.html', import.meta.url)),
+ },
+ },
+ },
plugins: [
react(),
electron({
diff --git a/docs/REARCH_EDITOR_AUTOMATION.md b/docs/REARCH_EDITOR_AUTOMATION.md
new file mode 100644
index 000000000..00daaaf57
--- /dev/null
+++ b/docs/REARCH_EDITOR_AUTOMATION.md
@@ -0,0 +1,218 @@
+# REARCH — Editor Automation & Observation Surface (toward an editor MCP)
+
+## Why
+
+Two needs converge on the same architecture:
+
+1. **Verification is blocked at the pixel surface.** Logic and state can be verified
+ headlessly today (the Spine 3.8 module load + animate test runs the real
+ `spine38.wasm` in node and asserts a bone advances). But "does it actually
+ rasterize on screen" cannot be observed without a human looking at the editor —
+ the headless test path has no GL context, and an external agent cannot reach the
+ live Electron renderer. Every render-correctness change ends in a manual smoke
+ test.
+2. **A future editor MCP** wants exactly the same thing: a clean, typed surface an
+ agent can call to open a project, mutate a scene, drive a frame, and read back
+ what happened — including the rendered pixels.
+
+The answer to both is one canonical, transport-agnostic **EditorControlSurface**
+(command + query + observation), plus the two primitives that make render
+verification meaningful and reachable: a deterministic engine **step**, and a
+**headless render host** that owns a real WebGL2 context. The MCP server is then a
+thin transport adapter over that surface — not a parallel API.
+
+## Diagnosis — the programmatic surface that exists today
+
+| Layer | Where | What it gives | Gap |
+|---|---|---|---|
+| Mutations | `desktop/src/engine/SceneCommands.ts` | `addEntity` / `deleteEntity` / `duplicateEntity` / `renameEntity` / `setField` / `setEntityXY` / `beginGesture` / `endGesture`; dual-writes World + `SceneModel`; every edit undoable via `EditorHistory` | No `setComponent` (add/remove a whole component), no asset-ref set helper |
+| Reads | `desktop/src/engine/SceneQuery.ts` | `worldVersion` / `readSceneTree` / `readEntity` / `readInspector` / `getFieldValue` | — |
+| Truth | `desktop/src/engine/SceneModel.ts` | JSON-first `SceneData` (lossless: unknown components, `@uuid:` refs, `visible:false` entities); `serialize()` | — |
+| Engine bridge | `desktop/src/engine/EngineHost.ts` (renderer) | owns `module`, **WebGL2 `canvas`**, live `World`; `setRunMode(isPlaying,isPaused)`; `getResource` | Loop is Emscripten rAF (`app.run()`) — **no deterministic `step(dt)`**; capture is whatever frame rAF landed on |
+| Picking / transforms | `desktop/src/engine/ViewportController.ts` | `pickEntity` / `getEntityScreenRect` / `getEntityXY` / `canvasToWorld` | — |
+| Telemetry | `desktop/src/engine/StatsStore.ts` | `{ fps, entities, cursor }` (sampled, not per-frame) | No draw-call / batch stats; no per-entity render state |
+| Asset load | `desktop/src/engine/SceneLoader.ts` | `loadInto(app, sceneUrl, manifestUrl)` — resolves `@uuid:` → texture handles, spawns | — |
+| Transport | `desktop/electron/{main,preload}.ts` | pure `ipcMain.handle` + `estella://` protocol | **No local server**; nothing an external process can attach to |
+| Headless test | `desktop/tests/**`, `tests/helpers/loadWasm.ts` | constructs `App` + `Registry` in **pure node, no canvas** | **No GL → cannot rasterize → cannot pixel-verify** |
+
+The command/query core is already correct and already unit-tested. What's missing is
+(a) a single composed entry point over it, (b) the two render primitives, and
+(c) a transport an agent can reach.
+
+## Target architecture
+
+```
+ Consumers
+ ─────────
+ React UI (existing) vitest (logic/state) Agent / editor MCP client
+ │ │ │
+ │ direct calls │ direct calls ┌─────▼──────────────┐
+ │ (renderer) │ (in-proc) │ Editor MCP server │ P2
+ │ │ │ main process │
+ │ │ │ Streamable HTTP │ (attach to live editor)
+ │ │ │ + stdio │ (spawn headless host)
+ │ │ └─────┬──────────────┘
+ │ │ │ editor:rpc IPC
+ ▼ ▼ ▼
+ ┌──────────────────────────────────────────────────────────────────────────┐
+ │ EditorControlSurface — the ONE canonical programmatic API (renderer) │ P1
+ │ │
+ │ Lifecycle openProject · loadScene · saveScene · setRunMode · step │
+ │ Commands addEntity · deleteEntity · setComponent · setField · … │ (over SceneCommands)
+ │ Queries getSceneTree · getEntity · getInspector · getProjectInfo │ (over SceneQuery / SceneModel)
+ │ Observe captureViewport · getEntityRenderState · getStats · pickAt │ (over canvas/GL + ViewportController)
+ └──────────────────────────────────────────────────────────────────────────┘
+ │ │ │
+ ▼ ▼ ▼
+ SceneCommands SceneQuery/Model EngineHost (canvas, WebGL2, World, step)
+```
+
+### The EditorControlSurface (P1) — one API, four pillars
+
+A new renderer-side module `desktop/src/engine/EditorControlSurface.ts` that **composes
+the existing modules** — it adds no new truth, it is the single front door.
+
+- **Lifecycle** — `openProject(root)`, `loadScene(rel)`, `saveScene()`,
+ `setRunMode({playing,paused})`, and the new **`step(frames, dt)`** (see below).
+- **Commands** — pass-throughs to `SceneCommands` plus two small additions the agent
+ needs that the UI does today by other means: `setComponent(entity, type, data)` /
+ `removeComponent(entity, type)` (add/remove a whole component), and
+ `setAssetRef(entity, type, key, uuid)` (set a `@uuid:` ref via `SceneModel`, the way
+ the inspector's asset picker does). Everything stays undoable.
+- **Queries** — pass-throughs to `SceneQuery` + `SceneModel.serialize()` +
+ `getProjectInfo()` (root, scene list, build/schema-cache status).
+- **Observe** — the verification-closing pillar:
+ - `captureViewport({region?, maxWidth?}) → { png: Uint8Array, w, h }` — read the
+ WebGL2 framebuffer (`gl.readPixels` into a canvas → PNG, or `canvas.toDataURL`).
+ The agent literally sees the frame.
+ - `getEntityRenderState(id) → { worldTransform, screenRect, visible, inFrustum }` —
+ `ViewportController.getEntityScreenRect` + world transform. Cheap, deterministic,
+ diff-friendly — often enough to verify without a screenshot.
+ - `getStats() → { fps, entities, drawCalls?, batches? }` — `StatsStore` plus
+ draw-call/batch counters surfaced from the renderer (small engine add).
+ - `pickAt(x, y) → EntityId | null` — `ViewportController.pickEntity`.
+
+The surface is **transport-agnostic**: the same object is called directly by the
+React UI, in-process by a test, and (via IPC marshalling) by the MCP server. It is
+directly unit-testable in the existing vitest harness.
+
+### Determinism primitive — engine `step(dt)` (already exists)
+
+Render verification is only meaningful if it is reproducible. Today the editor drives
+`app.run()` (Emscripten rAF) — a capture reflects whatever frame rAF happened to be on.
+
+**The primitive already exists: `App.tick(delta)` (`sdk/src/app.ts:465`).** `tick` does
+the same lazy init `run()` does (SystemRunner, `Time` resource, `finishPlugins_`), then
+`flushStartupSystems_()` + `runFrame_(delta)` — exactly one frame, with the supplied
+delta, no wall-clock, **no rAF, no `'unwind'`**. `run()` is just that same setup plus a
+self-driving `mainLoop` that computes wall-clock delta and `requestAnimationFrame`s
+`runFrame_`. So the surface's `step(frames, dt)` is `for … await app.tick(dt)` — and the
+headless host simply calls `tick()` N times **instead of** `run()`. **No engine/SDK
+change.** Canonical recipe: `loadScene → setRunMode(play) → step(30, 1/60) →
+captureViewport → assert`.
+
+The same wall-clock-free drive serves the isolated play realm (REARCH_EDITOR_REALM
+R0–R2). Most of the surface (commands, queries, lifecycle, `step` via `tick`) is also
+testable in the existing **pure-node vitest** (which already constructs `App` + mocks
+`EngineHost`); only `captureViewport` needs the WebGL2 window.
+
+### Render host — where pixels actually rasterize
+
+`captureViewport` needs a real **WebGL2** context. Two hosts provide the surface; pick
+per task:
+
+| Host | GL | Use | Notes |
+|---|---|---|---|
+| **Live editor** (renderer) | real WebGL2 | interactive / MCP-over-HTTP | the running GUI a human also uses |
+| **Headless editor** (Electron `BrowserWindow {show:false}` / offscreen) | real WebGL2 (Chromium SwiftShader/ANGLE) | agent verification, CI, MCP-over-stdio | boots the editor with no visible window, drives the surface programmatically |
+| pure-node vitest | **none** | logic/state only | keeps verifying World/command correctness; **cannot** pixel-verify |
+
+The decisive choice: **the headless render host is a headless boot of the real editor**,
+not a separate `headless-gl` rig. `headless-gl` (`gl` npm) is WebGL1-only — a dead end
+for a WebGL2 engine. A `show:false` Electron window gives a real Chromium WebGL2
+context, reuses the actual boot path (`createWebApp(..., wasmBaseUrl:'/wasm')`,
+`registerContext`), and **doubles as the MCP-over-stdio host**. One host serves both
+"my verification driver" and "the editor MCP transport."
+
+## MCP mapping (P2) — tools, resources, transport
+
+Grounded in MCP tool-design practice: dedicated typed tools (not a bash escape hatch)
+so the editor can validate, gate, and audit each action; strict JSON schemas
+(`additionalProperties:false`, explicit `required`); prescriptive descriptions that
+state *when* to call.
+
+- **Commands → MCP tools** (side effects, need gating). `editor_add_entity`,
+ `editor_delete_entity`, `editor_set_component`, `editor_set_field`,
+ `editor_load_scene`, `editor_set_run_mode`, `editor_step`.
+- **Observations → MCP tools returning image content** — `editor_capture_viewport`
+ returns an MCP **image content block**, so the agent sees the frame inline.
+ `editor_get_render_state`, `editor_get_stats`, `editor_pick_at` return JSON.
+- **Stable reads → MCP resources** (read-only, addressable, cacheable context, kept
+ out of the tool list): `editor://scene/tree`, `editor://scene/{entity}/inspector`,
+ `editor://project`. Parametric reads stay tools.
+
+**Transport: Streamable HTTP as primary, stdio for headless.**
+- **Streamable HTTP** — the editor's main process hosts `http://127.0.0.1:<port>/mcp`;
+ an agent attaches to the **already-running** GUI. Right because the editor is a
+ long-lived process a human also uses (stdio would force the client to own the editor
+ as a subprocess).
+- **stdio** — for `--headless` boots where the agent spawns the editor (CI, autonomous
+ verification). Same surface, different framing.
+
+Both bind localhost only and sit behind a **permission policy**:
+- `read` — queries + observations always allowed.
+- `write` — mutations gated; default off, enabled with `--allow-writes` or a UI toggle,
+ so an agent cannot silently rewrite the user's project. Transient verification
+ scenes (load → step → capture, no save) need only `read` + ephemeral mutation, never
+ `saveScene`.
+- Destructive/outward ops (`saveScene` overwrite, `openProject`) always require explicit
+ opt-in.
+
+The MCP server is `desktop/electron/mcp/` in the main process; it marshals each tool
+call to the renderer surface over a new `editor:rpc` IPC channel pair and marshals the
+result (PNG bytes → image content) back.
+
+## Phases
+
+- **P1 — EditorControlSurface + `step` + headless render host.**
+ - `EditorControlSurface.ts` composing the existing modules (commands/queries/observe).
+ - Engine `app.tick(dt)` manual-step entry; surface `step(frames, dt)`.
+ - Headless editor boot (`show:false`) that instantiates the surface.
+ - **Deliverable that closes the current gap:** a verification run that does
+ `loadScene(spine 3.8 spineboy) → step(30,1/60) → captureViewport → assert the
+ spine region is non-blank / matches a reference`. This pixel-verifies Spine S1
+ (and every future render change) **without a human**. P1 is self-contained and
+ immediately useful even before any MCP exists.
+- **P2 — Editor MCP server.**
+ - `desktop/electron/mcp/` MCP server (official SDK), Streamable HTTP + stdio.
+ - Tools/resources per the mapping above; `capture_viewport` → image content.
+ - `editor:rpc` IPC bridge main↔renderer; permission policy (`read` / `write`).
+ - **Deliverable:** an agent connects to a live or headless editor and drives the
+ full loop (open project · add a sprite/spine entity · set its asset · step · see
+ the viewport · read the scene tree).
+
+## Open risks
+
+- ~~**WebGL2 in headless Electron**~~ — **RETIRED.** A `show:false` `BrowserWindow` gives
+ a usable WebGL2 context with the engine's exact context options, and `gl.readPixels`
+ returns the expected color. Evidence (de-risk spike, electron 42 on macOS):
+ `{"ok":true,"version":"WebGL 2.0 (OpenGL ES 3.0 Chromium)","pixel":[255,0,0,255]}`.
+ (`headless-gl` was never an option — WebGL1 only.) CI/Linux-headless still to confirm
+ (SwiftShader fallback; `enable-unsafe-swiftshader` already wired in the spike).
+- ~~**`step(dt)` vs the Emscripten main loop**~~ — **RETIRED.** `App.tick(delta)` already
+ exists and shares `runFrame_` with `run()`'s `mainLoop`; the headless host calls
+ `tick()` instead of `run()`, so there is no rAF loop to co-exist with. No engine change.
+- **Capture determinism** — texture upload / async asset load must be settled before
+ the captured frame; `loadScene` resolves textures before spawn, but verify nothing
+ uploads lazily on first draw (one warm-up `step` before the asserted capture if so).
+- **Pixel comparison** — reference-image diffing needs a tolerance (driver AA, SwiftShader
+ vs hardware). Prefer structural assertions (region non-blank, bounding box, draw-call
+ count, `getEntityRenderState`) over exact pixel equality where possible.
+
+## Cross-links
+
+- Determinism/host shares the wall-clock-free drive of REARCH_EDITOR_REALM (R0–R2,
+ isolated play realm) — build the step primitive once.
+- First consumer is the Spine rearch (REARCH_SPINE S1 pixel verification, then S2
+ pixel-parity vs native 4.2) — the spineboy asset ships for 3.8/4.1/4.2 at the same
+ example path, ideal for reference-image parity.