A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(pipeline): a project's promised systems build to wasm
yhh0a1686f2ccb5
diff --git a/compiler/src/frontend.ts b/compiler/src/frontend.tsindex 7d4e9bbb6..57dd401c8 100644--- a/compiler/src/frontend.ts+++ b/compiler/src/frontend.ts@@ -791,7 +791,7 @@ export function lowerProgram(files: readonly string[], builtins: ReadonlyMap<str// Constants and components first: a system in one file names both from another.for (const sf of sources) {- walkTop(sf, (node, binding) => {+ walkTop(sf, (node) => {// The declaration's own name, not the binding walkTop passes DOWN to// an initializer: at the declaration node itself that is still null.if (!ts.isVariableDeclaration(node) || !ts.isIdentifier(node.name) || !node.initializer) return;diff --git a/compiler/src/interp.ts b/compiler/src/interp.tsindex 8ed31fd03..12f14b8e7 100644--- a/compiler/src/interp.ts+++ b/compiler/src/interp.ts@@ -13,7 +13,7 @@* obviously correct, not fast; anything clever here would be a second* place for a bug to hide from the backend it exists to check.*/-import type { EirFn, EirSystem, Expr, Local, Place, QueryArg, Stmt } from './eir';+import type { EirFn, EirSystem, Expr, Place, QueryArg, Stmt } from './eir';/** The module's pure functions, threaded rather than held in a module global. */export type Fns = ReadonlyMap<string, EirFn>;diff --git a/pipeline/src/bundle/buildCompiledSystems.ts b/pipeline/src/bundle/buildCompiledSystems.tsnew file mode 100644index 000000000..2aef2d51b--- /dev/null+++ b/pipeline/src/bundle/buildCompiledSystems.ts@@ -0,0 +1,194 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file AOT step: a project's `@compiled` systems, as wasm (docs/REARCH_AOT.md).+ *+ * The rest of `buildScripts` hands the realm JavaScript. This hands it a module+ * of machine code for the systems the author PROMISED would compile, built with+ * emcc from the C `@estella/compiler` emits.+ *+ * It imports the engine's memory rather than owning one, which is what keeps+ * this cheap: the engine wasm stays prebuilt and only the game's own systems are+ * compiled here — a few hundred bytes, not a link (REARCH_AOT.md §6.1).+ *+ * Never throws. A project with no marked system is not a failure and not a+ * build step; a marked system that will not compile IS one, because that is what+ * the marker means.+ *+ * NOT yet called by `estella build`. Wiring it in makes emcc a requirement for+ * any project carrying a marker, and whether that is an error or a warning is a+ * question the dev / release / ship modes answer — they do not exist yet, and+ * failing every build on a machine without emsdk is the wrong default to pick+ * by accident.+ */+import { existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync, readFileSync } from 'node:fs';+import path from 'node:path';+import { brokenPromises, lowerProgram } from '../../../compiler/src/frontend';+import { verifySystem } from '../../../compiler/src/verify';+import { inlineSystem } from '../../../compiler/src/inline';+import { builtinShapes } from '../../../compiler/src/builtins';+import { packLayout, planFor } from '../../../compiler/src/abi';+import { CFLAGS, cSymbol, emitC } from '../../../compiler/src/codegen';++/** Where the C and the wasm land, beside the script bundle. */+const CACHE_DIR = '.esengine/cache/aot';+const WASM = 'systems.wasm';+const MANIFEST = 'systems.json';++/** What the runtime needs in order to call one compiled system. */+export interface CompiledSystemInfo {+ /** The system's declared name, as the schedule knows it. */+ name: string;+ /** The exported wasm symbol. */+ symbol: string;+ /** One entry per declared Query: its components, in the order it names them. */+ queries: { comp: string; mut: boolean }[][];+ /** One entry per declared Res, in declaration order. */+ resources: string[];+}++export interface CompiledSystemsManifest {+ /** The §2.5 contract hash. The loader refuses a module that disagrees. */+ contractHash: string;+ systems: CompiledSystemInfo[];+}++export interface BuildCompiledResult {+ ok: boolean;+ /** Absolute path to the wasm, or null when there was nothing to build. */+ wasmPath: string | null;+ manifest: CompiledSystemsManifest | null;+ errors: string[];+ /** Systems the subset refused that were NOT promised — information, not failure. */+ notes: string[];+}++/** Every `.ts` under `src/`, which is the unit a project's program is. */+function sources(root: string): string[] {+ const out: string[] = [];+ const walk = (dir: string): void => {+ if (!existsSync(dir)) return;+ for (const name of readdirSync(dir)) {+ const p = path.join(dir, name);+ if (statSync(p).isDirectory()) walk(p);+ else if (p.endsWith('.ts') && !p.endsWith('.d.ts')) out.push(p);+ }+ };+ walk(path.join(root, 'src'));+ return out;+}++/**+ * Compile the project's `@compiled` systems to `<root>/.esengine/cache/aot/`.+ *+ * `emcc` is the caller's to supply, because finding it is the toolchain's job+ * and not this module's; without one a project that promised nothing still+ * builds, and one that promised something says so.+ */+export async function buildCompiledSystems(+ root: string,+ opts: {+ /** Absolute path to emcc, or null when the toolchain has none. */+ emcc: string | null;+ /** Runs a command; the toolchain's own runner, so env and logging match. */+ run: (cmd: string, args: string[], cwd: string) => Promise<{ code: number; stderr: string }>;+ },+): Promise<BuildCompiledResult> {+ const files = sources(root);+ if (files.length === 0) {+ return { ok: true, wasmPath: null, manifest: null, errors: [], notes: [] };+ }++ let lowered;+ try {+ lowered = lowerProgram(files, builtinShapes());+ } catch (err) {+ return { ok: false, wasmPath: null, manifest: null, notes: [], errors: [String(err)] };+ }++ const broken = brokenPromises(lowered);+ if (broken.length > 0) {+ // The marker's whole point: a refusal against a promised system is an error+ // with a line, not a silent fall back to the interpreter.+ return {+ ok: false,+ wasmPath: null,+ manifest: null,+ notes: [],+ errors: broken.map((d) => `${d.file}:${d.line}: ${d.system ?? ''} is @compiled but ${d.message}`),+ };+ }++ const promised = new Set(lowered.required);+ const chosen = lowered.module.systems.filter((s) => promised.has(s.name));+ const notes = lowered.diagnostics+ .filter((d) => d.severity === 'note' && d.system)+ .map((d) => `${d.system}: ${d.message}`);+ if (chosen.length === 0) {+ // Nothing was promised, so there is nothing to build and nothing to say.+ return { ok: true, wasmPath: null, manifest: null, errors: [], notes };+ }++ const bad = chosen.flatMap((s) => verifySystem(s, lowered.module.comps, lowered.module.fns));+ if (bad.length > 0) {+ return {+ ok: false, wasmPath: null, manifest: null, notes,+ errors: bad.map((e) => `${e.system}: ${e.message}`),+ };+ }++ if (!opts.emcc) {+ return {+ ok: false, wasmPath: null, manifest: null, notes,+ errors: [`${chosen.length} system(s) are marked @compiled but there is no emcc`+ + ' — run `pnpm emsdk:setup`, or remove the marker'],+ };+ }++ const layout = packLayout(lowered.module.comps);+ const inlined = chosen.map((s) => inlineSystem(s, lowered.module.fns));+ const c = emitC(lowered.module, layout, inlined);++ const dir = path.join(root, CACHE_DIR);+ rmSync(dir, { recursive: true, force: true });+ mkdirSync(dir, { recursive: true });+ writeFileSync(path.join(dir, 'estella_abi.h'), c.header);+ writeFileSync(path.join(dir, 'estella_offsets.h'), c.offsets);+ writeFileSync(path.join(dir, 'systems.c'), c.source);++ const wasmPath = path.join(dir, WASM);+ const exported = c.symbols.map((s) => `_${s}`).join(',');+ const built = await opts.run(opts.emcc, [+ ...CFLAGS, '-Wall', '-Wextra',+ // No entry point and no JS glue, so the module's only import is the memory+ // it is given. §6.5 is a property of this command line as much as of the C.+ '--no-entry', '-sSTANDALONE_WASM', '-sIMPORTED_MEMORY', '-sERROR_ON_UNDEFINED_SYMBOLS=1',+ `-sEXPORTED_FUNCTIONS=${exported}`,+ '-o', wasmPath, path.join(dir, 'systems.c'),+ ], dir);+ if (built.code !== 0 || !existsSync(wasmPath)) {+ return { ok: false, wasmPath: null, manifest: null, notes, errors: [built.stderr.trim()] };+ }++ const manifest: CompiledSystemsManifest = {+ contractHash: c.hash,+ systems: inlined.map((sys) => {+ const plan = planFor(sys);+ return {+ name: sys.name,+ symbol: cSymbol(sys.name),+ queries: plan.queries.map((q) => q.map((a) => ({ comp: a.comp, mut: a.mut }))),+ resources: [...plan.resources],+ };+ }),+ };+ writeFileSync(path.join(dir, MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`);+ return { ok: true, wasmPath, manifest, errors: [], notes };+}++/** Read back what a previous build wrote, or null. */+export function readCompiledManifest(root: string): CompiledSystemsManifest | null {+ const at = path.join(root, CACHE_DIR, MANIFEST);+ if (!existsSync(at)) return null;+ return JSON.parse(readFileSync(at, 'utf8')) as CompiledSystemsManifest;+}diff --git a/pipeline/tests/build-compiled-systems.test.ts b/pipeline/tests/build-compiled-systems.test.tsnew file mode 100644index 000000000..140e194cb--- /dev/null+++ b/pipeline/tests/build-compiled-systems.test.ts@@ -0,0 +1,159 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file The AOT build step: a project's `@compiled` systems become wasm.+ *+ * @details Everything upstream of this proved the compiler and the contract.+ * This proves the BUILD: point it at a project directory and get back a+ * module the engine can load, or an error naming the line that stopped+ * it. Real emcc where emsdk is unpacked, and a loud skip where it is+ * not — a gate that never saw its subject is worse than a missing one.+ */+import { describe, it, expect } from 'vitest';+import { spawn } from 'node:child_process';+import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import path from 'node:path';+import { fileURLToPath } from 'node:url';+import { buildCompiledSystems, readCompiledManifest } from '../src/bundle/buildCompiledSystems';++const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');++function findEmcc(): string | null {+ const at = path.join(ROOT, 'tools/emsdk/upstream/emscripten',+ process.platform === 'win32' ? 'emcc.bat' : 'emcc');+ return existsSync(at) ? at : null;+}++const EMCC = findEmcc();++/** The toolchain's runner, in miniature: emcc is a .bat on Windows. */+const run = (cmd: string, args: string[], cwd: string): Promise<{ code: number; stderr: string }> =>+ new Promise((done) => {+ const child = spawn(cmd, args, { cwd, shell: process.platform === 'win32' });+ let stderr = '';+ child.stderr.on('data', (d) => { stderr += String(d); });+ child.on('close', (code) => done({ code: code ?? 1, stderr }));+ });++/** A project on disk: `src/` with whatever files the case needs. */+function project(files: Record<string, string>): string {+ const root = mkdtempSync(path.join(tmpdir(), 'estella-aot-project-'));+ for (const [rel, body] of Object.entries(files)) {+ const at = path.join(root, rel);+ mkdirSync(path.dirname(at), { recursive: true });+ writeFileSync(at, body);+ }+ return root;+}++const COMPONENTS = `import { defineComponent } from 'esengine';+export const Mover = defineComponent('Mover', { speed: 100, directionX: 1, directionY: 0 });+`;++const PROMISED = `import { defineSystem, Query, Mut, Res, Time, Transform } from 'esengine';+import { Mover } from './components';++/**+ * @compiled+ */+export const moveSystem = defineSystem(+ [Query(Mut(Transform), Mover), Res(Time)],+ (query, time) => {+ for (const [, transform, mover] of query) {+ transform.position.x += mover.directionX * mover.speed * time.delta;+ }+ },+ { name: 'MoveSystem' },+);+`;++/** Promised, and outside the subset: trig is refused on purpose (§3.3). */+const BROKEN = PROMISED+ .replace('mover.directionX * mover.speed', 'Math.sin(mover.directionX) * mover.speed')+ .replace("{ name: 'MoveSystem' }", "{ name: 'BrokenSystem' }");++/** Inside the subset, but nobody promised it. */+const UNMARKED = PROMISED.replace(`/**+ * @compiled+ */+`, '').replace("{ name: 'MoveSystem' }", "{ name: 'QuietSystem' }");++describe('the AOT build step', () => {+ it('reports whether this gate could run at all', () => {+ if (EMCC) console.log(`[aot-build] emcc at ${EMCC}`);+ else console.warn('[aot-build] NO EMSDK — the wasm half did NOT run (pnpm emsdk:setup).');+ expect(true).toBe(true);+ });++ it('a project that promised nothing is not a build step', async () => {+ const root = project({ 'src/components.ts': COMPONENTS, 'src/systems.ts': UNMARKED });+ const out = await buildCompiledSystems(root, { emcc: EMCC, run });+ expect(out.ok).toBe(true);+ expect(out.wasmPath).toBeNull();+ // The refusal is still reported, as information: §3.2's fallback is the+ // design, and a build that shouted about it would be crying wolf.+ expect(out.errors).toEqual([]);+ });++ it('a project with no sources at all is fine', async () => {+ const out = await buildCompiledSystems(project({}), { emcc: EMCC, run });+ expect(out).toMatchObject({ ok: true, wasmPath: null, errors: [] });+ });++ it('a broken promise fails the build, naming the file and the line', async () => {+ const root = project({ 'src/components.ts': COMPONENTS, 'src/systems.ts': BROKEN });+ const out = await buildCompiledSystems(root, { emcc: EMCC, run });+ expect(out.ok).toBe(false);+ expect(out.wasmPath).toBeNull();+ expect(out.errors).toHaveLength(1);+ expect(out.errors[0]).toContain('BrokenSystem');+ expect(out.errors[0]).toMatch(/Math\.sin is not exactly specified/);+ expect(out.errors[0]).toMatch(/systems\.ts:\d+/);+ });++ it('says what is missing when a promise needs a toolchain that is absent', async () => {+ const root = project({ 'src/components.ts': COMPONENTS, 'src/systems.ts': PROMISED });+ const out = await buildCompiledSystems(root, { emcc: null, run });+ expect(out.ok).toBe(false);+ expect(out.errors[0]).toMatch(/marked @compiled but there is no emcc/);+ });++ it.skipIf(!EMCC)('builds a module the engine can load, and a manifest for it', async () => {+ const root = project({ 'src/components.ts': COMPONENTS, 'src/systems.ts': PROMISED });+ const out = await buildCompiledSystems(root, { emcc: EMCC, run });+ expect(out.errors).toEqual([]);+ expect(out.ok).toBe(true);+ expect(out.wasmPath).not.toBeNull();++ const bytes = await import('node:fs').then((fs) => fs.readFileSync(out.wasmPath!));+ const module = new WebAssembly.Module(bytes as unknown as BufferSource);+ // The property the whole shape rests on: the engine's memory, and nothing+ // else — no function import, so no second channel back into the engine.+ expect(WebAssembly.Module.imports(module))+ .toEqual([{ module: 'env', name: 'memory', kind: 'memory' }]);+ expect(WebAssembly.Module.exports(module).map((e) => e.name)).toContain('es_sys_MoveSystem');++ // The manifest is what turns a symbol into a call: which components fill+ // which row slot, and in what order.+ expect(out.manifest).toEqual(readCompiledManifest(root));+ expect(out.manifest!.systems).toEqual([{+ name: 'MoveSystem',+ symbol: 'es_sys_MoveSystem',+ queries: [[{ comp: 'Transform', mut: true }, { comp: 'Mover', mut: false }]],+ resources: ['Time'],+ }]);+ expect(out.manifest!.contractHash).toMatch(/^[0-9a-f]{16}$/);+ });++ it.skipIf(!EMCC)('rebuilding is not additive: the cache is what this build made', async () => {+ const root = project({ 'src/components.ts': COMPONENTS, 'src/systems.ts': PROMISED });+ await buildCompiledSystems(root, { emcc: EMCC, run });+ writeFileSync(path.join(root, '.esengine/cache/aot/stale.c'), 'int leftover;\n');++ await buildCompiledSystems(root, { emcc: EMCC, run });+ // A file from a previous shape of the project must not survive into the one+ // the engine loads next.+ expect(existsSync(path.join(root, '.esengine/cache/aot/stale.c'))).toBe(false);+ });+});diff --git a/tools/check-pipeline-boundary.mjs b/tools/check-pipeline-boundary.mjsindex d249a8cec..f091bde81 100644--- a/tools/check-pipeline-boundary.mjs+++ b/tools/check-pipeline-boundary.mjs@@ -23,9 +23,11 @@ const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');const PIPELINE = path.join(ROOT, 'pipeline');const SOURCE = /\.(ts|mts|js|mjs)$/;-/** Repo areas the pipeline is allowed to read: the engine it packages, and the- * format writers it packages with. */-const REACHABLE = ['sdk/src', 'build-tools', 'tools'];+/** Repo areas the pipeline is allowed to read: the engine it packages, the+ * format writers it packages with, and the AOT compiler — which is a thing that+ * BUILDS a project, the same category as build-tools, not a thing that edits+ * one (docs/REARCH_AOT.md Stage 4). */+const REACHABLE = ['sdk/src', 'build-tools', 'tools', 'compiler/src'];/** Packages that only exist inside a running editor. */const HOST_ONLY = new Set(['electron', 'react', 'react-dom', 'react-dom/client', 'zustand', 'dockview']);