Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(pipeline): a project's promised systems build to wasm

yhh0a1686f2ccb5
diff --git a/compiler/src/frontend.ts b/compiler/src/frontend.ts
index 7d4e9bbb6..57dd401c8 100644
--- a/compiler/src/frontend.ts
+++ b/compiler/src/frontend.ts
@@ -791,7 +791,7 @@ export function lowerProgram(files: readonly string[], builtins: ReadonlyMap<str
// Constants and components first: a system in one file names both from another.
for (const sf of sources) {
- walkTop(sf, (node, binding) => {
+ walkTop(sf, (node) => {
// The declaration's own name, not the binding walkTop passes DOWN to
// an initializer: at the declaration node itself that is still null.
if (!ts.isVariableDeclaration(node) || !ts.isIdentifier(node.name) || !node.initializer) return;
diff --git a/compiler/src/interp.ts b/compiler/src/interp.ts
index 8ed31fd03..12f14b8e7 100644
--- a/compiler/src/interp.ts
+++ b/compiler/src/interp.ts
@@ -13,7 +13,7 @@
* obviously correct, not fast; anything clever here would be a second
* place for a bug to hide from the backend it exists to check.
*/
-import type { EirFn, EirSystem, Expr, Local, Place, QueryArg, Stmt } from './eir';
+import type { EirFn, EirSystem, Expr, Place, QueryArg, Stmt } from './eir';
/** The module's pure functions, threaded rather than held in a module global. */
export type Fns = ReadonlyMap<string, EirFn>;
diff --git a/pipeline/src/bundle/buildCompiledSystems.ts b/pipeline/src/bundle/buildCompiledSystems.ts
new file mode 100644
index 000000000..2aef2d51b
--- /dev/null
+++ b/pipeline/src/bundle/buildCompiledSystems.ts
@@ -0,0 +1,194 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file AOT step: a project's `@compiled` systems, as wasm (docs/REARCH_AOT.md).
+ *
+ * The rest of `buildScripts` hands the realm JavaScript. This hands it a module
+ * of machine code for the systems the author PROMISED would compile, built with
+ * emcc from the C `@estella/compiler` emits.
+ *
+ * It imports the engine's memory rather than owning one, which is what keeps
+ * this cheap: the engine wasm stays prebuilt and only the game's own systems are
+ * compiled here — a few hundred bytes, not a link (REARCH_AOT.md §6.1).
+ *
+ * Never throws. A project with no marked system is not a failure and not a
+ * build step; a marked system that will not compile IS one, because that is what
+ * the marker means.
+ *
+ * NOT yet called by `estella build`. Wiring it in makes emcc a requirement for
+ * any project carrying a marker, and whether that is an error or a warning is a
+ * question the dev / release / ship modes answer — they do not exist yet, and
+ * failing every build on a machine without emsdk is the wrong default to pick
+ * by accident.
+ */
+import { existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+import { brokenPromises, lowerProgram } from '../../../compiler/src/frontend';
+import { verifySystem } from '../../../compiler/src/verify';
+import { inlineSystem } from '../../../compiler/src/inline';
+import { builtinShapes } from '../../../compiler/src/builtins';
+import { packLayout, planFor } from '../../../compiler/src/abi';
+import { CFLAGS, cSymbol, emitC } from '../../../compiler/src/codegen';
+
+/** Where the C and the wasm land, beside the script bundle. */
+const CACHE_DIR = '.esengine/cache/aot';
+const WASM = 'systems.wasm';
+const MANIFEST = 'systems.json';
+
+/** What the runtime needs in order to call one compiled system. */
+export interface CompiledSystemInfo {
+ /** The system's declared name, as the schedule knows it. */
+ name: string;
+ /** The exported wasm symbol. */
+ symbol: string;
+ /** One entry per declared Query: its components, in the order it names them. */
+ queries: { comp: string; mut: boolean }[][];
+ /** One entry per declared Res, in declaration order. */
+ resources: string[];
+}
+
+export interface CompiledSystemsManifest {
+ /** The §2.5 contract hash. The loader refuses a module that disagrees. */
+ contractHash: string;
+ systems: CompiledSystemInfo[];
+}
+
+export interface BuildCompiledResult {
+ ok: boolean;
+ /** Absolute path to the wasm, or null when there was nothing to build. */
+ wasmPath: string | null;
+ manifest: CompiledSystemsManifest | null;
+ errors: string[];
+ /** Systems the subset refused that were NOT promised — information, not failure. */
+ notes: string[];
+}
+
+/** Every `.ts` under `src/`, which is the unit a project's program is. */
+function sources(root: string): string[] {
+ const out: string[] = [];
+ const walk = (dir: string): void => {
+ if (!existsSync(dir)) return;
+ for (const name of readdirSync(dir)) {
+ const p = path.join(dir, name);
+ if (statSync(p).isDirectory()) walk(p);
+ else if (p.endsWith('.ts') && !p.endsWith('.d.ts')) out.push(p);
+ }
+ };
+ walk(path.join(root, 'src'));
+ return out;
+}
+
+/**
+ * Compile the project's `@compiled` systems to `<root>/.esengine/cache/aot/`.
+ *
+ * `emcc` is the caller's to supply, because finding it is the toolchain's job
+ * and not this module's; without one a project that promised nothing still
+ * builds, and one that promised something says so.
+ */
+export async function buildCompiledSystems(
+ root: string,
+ opts: {
+ /** Absolute path to emcc, or null when the toolchain has none. */
+ emcc: string | null;
+ /** Runs a command; the toolchain's own runner, so env and logging match. */
+ run: (cmd: string, args: string[], cwd: string) => Promise<{ code: number; stderr: string }>;
+ },
+): Promise<BuildCompiledResult> {
+ const files = sources(root);
+ if (files.length === 0) {
+ return { ok: true, wasmPath: null, manifest: null, errors: [], notes: [] };
+ }
+
+ let lowered;
+ try {
+ lowered = lowerProgram(files, builtinShapes());
+ } catch (err) {
+ return { ok: false, wasmPath: null, manifest: null, notes: [], errors: [String(err)] };
+ }
+
+ const broken = brokenPromises(lowered);
+ if (broken.length > 0) {
+ // The marker's whole point: a refusal against a promised system is an error
+ // with a line, not a silent fall back to the interpreter.
+ return {
+ ok: false,
+ wasmPath: null,
+ manifest: null,
+ notes: [],
+ errors: broken.map((d) => `${d.file}:${d.line}: ${d.system ?? ''} is @compiled but ${d.message}`),
+ };
+ }
+
+ const promised = new Set(lowered.required);
+ const chosen = lowered.module.systems.filter((s) => promised.has(s.name));
+ const notes = lowered.diagnostics
+ .filter((d) => d.severity === 'note' && d.system)
+ .map((d) => `${d.system}: ${d.message}`);
+ if (chosen.length === 0) {
+ // Nothing was promised, so there is nothing to build and nothing to say.
+ return { ok: true, wasmPath: null, manifest: null, errors: [], notes };
+ }
+
+ const bad = chosen.flatMap((s) => verifySystem(s, lowered.module.comps, lowered.module.fns));
+ if (bad.length > 0) {
+ return {
+ ok: false, wasmPath: null, manifest: null, notes,
+ errors: bad.map((e) => `${e.system}: ${e.message}`),
+ };
+ }
+
+ if (!opts.emcc) {
+ return {
+ ok: false, wasmPath: null, manifest: null, notes,
+ errors: [`${chosen.length} system(s) are marked @compiled but there is no emcc`
+ + ' — run `pnpm emsdk:setup`, or remove the marker'],
+ };
+ }
+
+ const layout = packLayout(lowered.module.comps);
+ const inlined = chosen.map((s) => inlineSystem(s, lowered.module.fns));
+ const c = emitC(lowered.module, layout, inlined);
+
+ const dir = path.join(root, CACHE_DIR);
+ rmSync(dir, { recursive: true, force: true });
+ mkdirSync(dir, { recursive: true });
+ writeFileSync(path.join(dir, 'estella_abi.h'), c.header);
+ writeFileSync(path.join(dir, 'estella_offsets.h'), c.offsets);
+ writeFileSync(path.join(dir, 'systems.c'), c.source);
+
+ const wasmPath = path.join(dir, WASM);
+ const exported = c.symbols.map((s) => `_${s}`).join(',');
+ const built = await opts.run(opts.emcc, [
+ ...CFLAGS, '-Wall', '-Wextra',
+ // No entry point and no JS glue, so the module's only import is the memory
+ // it is given. §6.5 is a property of this command line as much as of the C.
+ '--no-entry', '-sSTANDALONE_WASM', '-sIMPORTED_MEMORY', '-sERROR_ON_UNDEFINED_SYMBOLS=1',
+ `-sEXPORTED_FUNCTIONS=${exported}`,
+ '-o', wasmPath, path.join(dir, 'systems.c'),
+ ], dir);
+ if (built.code !== 0 || !existsSync(wasmPath)) {
+ return { ok: false, wasmPath: null, manifest: null, notes, errors: [built.stderr.trim()] };
+ }
+
+ const manifest: CompiledSystemsManifest = {
+ contractHash: c.hash,
+ systems: inlined.map((sys) => {
+ const plan = planFor(sys);
+ return {
+ name: sys.name,
+ symbol: cSymbol(sys.name),
+ queries: plan.queries.map((q) => q.map((a) => ({ comp: a.comp, mut: a.mut }))),
+ resources: [...plan.resources],
+ };
+ }),
+ };
+ writeFileSync(path.join(dir, MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`);
+ return { ok: true, wasmPath, manifest, errors: [], notes };
+}
+
+/** Read back what a previous build wrote, or null. */
+export function readCompiledManifest(root: string): CompiledSystemsManifest | null {
+ const at = path.join(root, CACHE_DIR, MANIFEST);
+ if (!existsSync(at)) return null;
+ return JSON.parse(readFileSync(at, 'utf8')) as CompiledSystemsManifest;
+}
diff --git a/pipeline/tests/build-compiled-systems.test.ts b/pipeline/tests/build-compiled-systems.test.ts
new file mode 100644
index 000000000..140e194cb
--- /dev/null
+++ b/pipeline/tests/build-compiled-systems.test.ts
@@ -0,0 +1,159 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file The AOT build step: a project's `@compiled` systems become wasm.
+ *
+ * @details Everything upstream of this proved the compiler and the contract.
+ * This proves the BUILD: point it at a project directory and get back a
+ * module the engine can load, or an error naming the line that stopped
+ * it. Real emcc where emsdk is unpacked, and a loud skip where it is
+ * not — a gate that never saw its subject is worse than a missing one.
+ */
+import { describe, it, expect } from 'vitest';
+import { spawn } from 'node:child_process';
+import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { buildCompiledSystems, readCompiledManifest } from '../src/bundle/buildCompiledSystems';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
+
+function findEmcc(): string | null {
+ const at = path.join(ROOT, 'tools/emsdk/upstream/emscripten',
+ process.platform === 'win32' ? 'emcc.bat' : 'emcc');
+ return existsSync(at) ? at : null;
+}
+
+const EMCC = findEmcc();
+
+/** The toolchain's runner, in miniature: emcc is a .bat on Windows. */
+const run = (cmd: string, args: string[], cwd: string): Promise<{ code: number; stderr: string }> =>
+ new Promise((done) => {
+ const child = spawn(cmd, args, { cwd, shell: process.platform === 'win32' });
+ let stderr = '';
+ child.stderr.on('data', (d) => { stderr += String(d); });
+ child.on('close', (code) => done({ code: code ?? 1, stderr }));
+ });
+
+/** A project on disk: `src/` with whatever files the case needs. */
+function project(files: Record<string, string>): string {
+ const root = mkdtempSync(path.join(tmpdir(), 'estella-aot-project-'));
+ for (const [rel, body] of Object.entries(files)) {
+ const at = path.join(root, rel);
+ mkdirSync(path.dirname(at), { recursive: true });
+ writeFileSync(at, body);
+ }
+ return root;
+}
+
+const COMPONENTS = `import { defineComponent } from 'esengine';
+export const Mover = defineComponent('Mover', { speed: 100, directionX: 1, directionY: 0 });
+`;
+
+const PROMISED = `import { defineSystem, Query, Mut, Res, Time, Transform } from 'esengine';
+import { Mover } from './components';
+
+/**
+ * @compiled
+ */
+export const moveSystem = defineSystem(
+ [Query(Mut(Transform), Mover), Res(Time)],
+ (query, time) => {
+ for (const [, transform, mover] of query) {
+ transform.position.x += mover.directionX * mover.speed * time.delta;
+ }
+ },
+ { name: 'MoveSystem' },
+);
+`;
+
+/** Promised, and outside the subset: trig is refused on purpose (§3.3). */
+const BROKEN = PROMISED
+ .replace('mover.directionX * mover.speed', 'Math.sin(mover.directionX) * mover.speed')
+ .replace("{ name: 'MoveSystem' }", "{ name: 'BrokenSystem' }");
+
+/** Inside the subset, but nobody promised it. */
+const UNMARKED = PROMISED.replace(`/**
+ * @compiled
+ */
+`, '').replace("{ name: 'MoveSystem' }", "{ name: 'QuietSystem' }");
+
+describe('the AOT build step', () => {
+ it('reports whether this gate could run at all', () => {
+ if (EMCC) console.log(`[aot-build] emcc at ${EMCC}`);
+ else console.warn('[aot-build] NO EMSDK — the wasm half did NOT run (pnpm emsdk:setup).');
+ expect(true).toBe(true);
+ });
+
+ it('a project that promised nothing is not a build step', async () => {
+ const root = project({ 'src/components.ts': COMPONENTS, 'src/systems.ts': UNMARKED });
+ const out = await buildCompiledSystems(root, { emcc: EMCC, run });
+ expect(out.ok).toBe(true);
+ expect(out.wasmPath).toBeNull();
+ // The refusal is still reported, as information: §3.2's fallback is the
+ // design, and a build that shouted about it would be crying wolf.
+ expect(out.errors).toEqual([]);
+ });
+
+ it('a project with no sources at all is fine', async () => {
+ const out = await buildCompiledSystems(project({}), { emcc: EMCC, run });
+ expect(out).toMatchObject({ ok: true, wasmPath: null, errors: [] });
+ });
+
+ it('a broken promise fails the build, naming the file and the line', async () => {
+ const root = project({ 'src/components.ts': COMPONENTS, 'src/systems.ts': BROKEN });
+ const out = await buildCompiledSystems(root, { emcc: EMCC, run });
+ expect(out.ok).toBe(false);
+ expect(out.wasmPath).toBeNull();
+ expect(out.errors).toHaveLength(1);
+ expect(out.errors[0]).toContain('BrokenSystem');
+ expect(out.errors[0]).toMatch(/Math\.sin is not exactly specified/);
+ expect(out.errors[0]).toMatch(/systems\.ts:\d+/);
+ });
+
+ it('says what is missing when a promise needs a toolchain that is absent', async () => {
+ const root = project({ 'src/components.ts': COMPONENTS, 'src/systems.ts': PROMISED });
+ const out = await buildCompiledSystems(root, { emcc: null, run });
+ expect(out.ok).toBe(false);
+ expect(out.errors[0]).toMatch(/marked @compiled but there is no emcc/);
+ });
+
+ it.skipIf(!EMCC)('builds a module the engine can load, and a manifest for it', async () => {
+ const root = project({ 'src/components.ts': COMPONENTS, 'src/systems.ts': PROMISED });
+ const out = await buildCompiledSystems(root, { emcc: EMCC, run });
+ expect(out.errors).toEqual([]);
+ expect(out.ok).toBe(true);
+ expect(out.wasmPath).not.toBeNull();
+
+ const bytes = await import('node:fs').then((fs) => fs.readFileSync(out.wasmPath!));
+ const module = new WebAssembly.Module(bytes as unknown as BufferSource);
+ // The property the whole shape rests on: the engine's memory, and nothing
+ // else — no function import, so no second channel back into the engine.
+ expect(WebAssembly.Module.imports(module))
+ .toEqual([{ module: 'env', name: 'memory', kind: 'memory' }]);
+ expect(WebAssembly.Module.exports(module).map((e) => e.name)).toContain('es_sys_MoveSystem');
+
+ // The manifest is what turns a symbol into a call: which components fill
+ // which row slot, and in what order.
+ expect(out.manifest).toEqual(readCompiledManifest(root));
+ expect(out.manifest!.systems).toEqual([{
+ name: 'MoveSystem',
+ symbol: 'es_sys_MoveSystem',
+ queries: [[{ comp: 'Transform', mut: true }, { comp: 'Mover', mut: false }]],
+ resources: ['Time'],
+ }]);
+ expect(out.manifest!.contractHash).toMatch(/^[0-9a-f]{16}$/);
+ });
+
+ it.skipIf(!EMCC)('rebuilding is not additive: the cache is what this build made', async () => {
+ const root = project({ 'src/components.ts': COMPONENTS, 'src/systems.ts': PROMISED });
+ await buildCompiledSystems(root, { emcc: EMCC, run });
+ writeFileSync(path.join(root, '.esengine/cache/aot/stale.c'), 'int leftover;\n');
+
+ await buildCompiledSystems(root, { emcc: EMCC, run });
+ // A file from a previous shape of the project must not survive into the one
+ // the engine loads next.
+ expect(existsSync(path.join(root, '.esengine/cache/aot/stale.c'))).toBe(false);
+ });
+});
diff --git a/tools/check-pipeline-boundary.mjs b/tools/check-pipeline-boundary.mjs
index d249a8cec..f091bde81 100644
--- a/tools/check-pipeline-boundary.mjs
+++ b/tools/check-pipeline-boundary.mjs
@@ -23,9 +23,11 @@ const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const PIPELINE = path.join(ROOT, 'pipeline');
const SOURCE = /\.(ts|mts|js|mjs)$/;
-/** Repo areas the pipeline is allowed to read: the engine it packages, and the
- * format writers it packages with. */
-const REACHABLE = ['sdk/src', 'build-tools', 'tools'];
+/** Repo areas the pipeline is allowed to read: the engine it packages, the
+ * format writers it packages with, and the AOT compiler — which is a thing that
+ * BUILDS a project, the same category as build-tools, not a thing that edits
+ * one (docs/REARCH_AOT.md Stage 4). */
+const REACHABLE = ['sdk/src', 'build-tools', 'tools', 'compiler/src'];
/** Packages that only exist inside a running editor. */
const HOST_ONLY = new Set(['electron', 'react', 'react-dom', 'react-dom/client', 'zustand', 'dockview']);