A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
* @file AOT step: a project's `@compiled` systems, as wasm.
*
* The rest of `buildScripts` hands the realm JavaScript. This hands it a module
* of machine code for the systems the author PROMISED would compile, built with
* emcc from the C `@estella/compiler` emits.
*
* It imports the engine's memory rather than owning one, which is what keeps
* this cheap: the engine wasm stays prebuilt and only the game's own systems are
* compiled here — a few hundred bytes, not a link.
*
* Never throws. A project with no marked system is not a failure and not a
* build step; a marked system that will not compile IS one, because that is what
* the marker means.
*
* The mode decides whether it runs at all. `dev` never compiles, because the
* editor's preview always interprets: a machine with no emsdk builds and runs
* every project. `release` and `ship` do compile, and there a promise the subset
* cannot keep is an error rather than a quiet fallback — which is the difference
* the marker exists to make.
*/
import { existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync, readFileSync } from 'node:fs';
import path from 'node:path';
import { brokenPromises, lowerProgram } from '../../../compiler/src/frontend';
import { verifySystem } from '../../../compiler/src/verify';
import { inlineSystem } from '../../../compiler/src/inline';
import { builtinShapes } from '../../../compiler/src/builtins';
import { packLayout } from '../../../compiler/src/abi';
import { CFLAGS, WASM_LINK_FLAGS, emitC, moduleDeclarations } from '../../../compiler/src/codegen';
import { nativeLinkFlags, nativeModuleExt } from '../../../compiler/src/hostCC';
import { AOT_MANIFEST, AOT_WASM } from './aotArtifacts';
/** Where the C and the wasm land, beside the script bundle. */
const CACHE_DIR = '.esengine/cache/aot';
const WASM = AOT_WASM;
const MANIFEST = AOT_MANIFEST;
/** What the runtime needs in order to call one compiled system. */
export interface CompiledSystemInfo {
/** The system's declared name, as the schedule knows it. */
name: string;
/** The exported wasm symbol. */
symbol: string;
/** One entry per declared Query: its components, in the order it names them. */
queries: { comp: string; mut: boolean }[][];
/**
* One per declared Res/ResMut, in declaration order. `mut` is what makes the
* runtime write a mirrored resource back after the call; `fields` is present
* for a resource the PROJECT declared, because its layout is not something
* the engine knows — it was derived from the declaration.
*/
resources: { name: string; mut: boolean; fields?: string[] }[];
/** Event readers and writers, with the payload layout each one uses. */
readers: { slot: number; event: string; fields: string[] }[];
writers: { slot: number; event: string; fields: string[] }[];
/**
* Whether the body appends a command record. A road that cannot flush one to
* the world the interpreter would have flushed it to leaves the system to the
* interpreter, so this is what that decision is made on.
*/
commands: boolean;
}
export interface CompiledSystemsManifest {
/**
* What this module baked in, so a loader can say WHICH one moved: the engine
* it was built against, and the project's own component shapes. The fixes
* differ — rebuild the module, or rebuild the project — so the numbers do too.
*/
engineAbi: string;
projectShapes: string;
/** Which BUILD this is: the artifact bakes the same number, so a loader can
* refuse a module paired with another build's sidecar. */
moduleContract: string;
systems: CompiledSystemInfo[];
}
/**
* Which build this is. Not a verbosity setting: it decides whether a `@compiled`
* marker is a promise anyone is collecting on.
*/
export type BuildMode = 'dev' | 'release' | 'ship';
/**
* Which machine the systems are compiled FOR.
*
* `wasm` imports the engine's memory; `native` is a library a host loads into
* its own process, where an address is a pointer and not an offset into one
* block. The same C either way — the header picks the typedef.
*/
export type CompileTarget = 'wasm' | 'native';
/** Everything that differs between the two, in one place rather than in ifs. */
interface TargetPlan {
/** `sizeof(es_addr_t)` there. It rides the handshake, so an artifact built
* for one host is refused by the other rather than reading half a pointer. */
readonly addressBytes: 4 | 8;
/** What the module is called in the cache. */
readonly module: string;
/** Link flags, plus whatever names the exports on this target. */
linkFlags(symbols: readonly string[]): string[];
/** Whether the declaration table is compiled in. It is the DATA half: a wasm
* module must not carry it, since its data section would land at an address
* the engine already owns, and a library in the host's process is who does. */
readonly compilesDecls: boolean;
/** What to say when this machine has no compiler for it. */
readonly missing: string;
}
const TARGETS: Record<CompileTarget, TargetPlan> = {
wasm: {
addressBytes: 4,
module: WASM,
linkFlags: (symbols) => [
// No entry point and no JS glue, so the module's only import is the memory
// it is given. An empty import section is a property of this command line
// as much as of the C.
...WASM_LINK_FLAGS,
`-sEXPORTED_FUNCTIONS=${symbols.map((s) => `_${s}`).join(',')}`,
],
compilesDecls: false,
missing: 'there is no emcc — install the emscripten toolchain and point EMSDK'
+ ' at it (in this repo: `pnpm emsdk:setup`)',
},
native: {
addressBytes: 8,
module: `systems${nativeModuleExt()}`,
linkFlags: () => [...nativeLinkFlags()],
compilesDecls: true,
missing: 'this machine has no C compiler — install clang or gcc and put it on PATH',
},
};
export interface BuildCompiledResult {
ok: boolean;
/** Absolute path to the built module, or null when there was nothing to
* build. Which KIND of module is the target the caller asked for. */
modulePath: string | null;
/** With `outputs`: each output's module, by its id (the first is `modulePath`). */
modules?: Record<string, string>;
manifest: CompiledSystemsManifest | null;
errors: string[];
/** Systems the subset refused that were NOT promised — information, not failure. */
notes: string[];
}
/** Every `.ts` under `src/`, which is the unit a project's program is. */
function sources(root: string): string[] {
const out: string[] = [];
const walk = (dir: string): void => {
if (!existsSync(dir)) return;
for (const name of readdirSync(dir)) {
const p = path.join(dir, name);
if (statSync(p).isDirectory()) walk(p);
else if (p.endsWith('.ts') && !p.endsWith('.d.ts')) out.push(p);
}
};
walk(path.join(root, 'src'));
return out;
}
/** A marker anywhere in a file. Text, not a parse: this only decides whether the
* expensive answer is worth computing. */
const anyPromise = (files: string[]): boolean =>
files.some((f) => readFileSync(f, 'utf8').includes('@compiled'));
/**
* Whether this project promises any compilation at all.
*
* The cheap half of what {@link buildCompiledSystems} answers, exported because
* the editor's platform catalog must say a target needs emcc BEFORE an export
* spawns one. Both read this, so they cannot disagree about what was promised.
*/
export function promisesCompilation(root: string): boolean {
return anyPromise(sources(root));
}
/**
* Compile the project's `@compiled` systems to `<root>/.esengine/cache/aot/`.
*
* The compiler is the caller's to supply, because finding it is the toolchain's
* job and not this module's; without one a project that promised nothing still
* builds, and one that promised something says so.
*/
export async function buildCompiledSystems(
root: string,
opts: {
/** dev never compiles; release and ship require what the markers promised. */
mode: BuildMode;
/** Which machine this build is for. Absent ⇒ wasm, the road every
* browser and mini-game host takes. */
target?: CompileTarget;
/** Absolute path to the compiler FOR that target — emcc for wasm, the
* host's own cc for native — or null when this machine has none. */
cc: string | null;
/** Runs a command; the toolchain's own runner, so env and logging match. */
run: (cmd: string, args: string[], cwd: string) => Promise<{ code: number; stderr: string }>;
/**
* Native only: the machines to build the one C for, each with its own
* compiler and flags — a cross-compiled package carries one module per ABI.
* Replaces `cc` and the host's link flags; an empty list means no compiler
* here, said with `missing`.
*/
outputs?: ReadonlyArray<{ id: string; cc: string; flags: readonly string[]; module: string }>;
/** What to say when `outputs` is empty. */
missing?: string;
},
): Promise<BuildCompiledResult> {
const plan = TARGETS[opts.target ?? 'wasm'];
if (opts.mode === 'dev') {
// Nothing is compiled and nothing is checked, so a marker costs a dev build
// nothing at all — not a toolchain, not a second of build time, not a
// failure on a machine that has neither.
return { ok: true, modulePath: null, manifest: null, errors: [], notes: [] };
}
const files = sources(root);
if (files.length === 0) {
return { ok: true, modulePath: null, manifest: null, errors: [], notes: [] };
}
// A project with no `@compiled` anywhere promised nothing, so it must not pay
// for a TypeScript Program over its sources. Nothing is decided here: the parse
// below still decides, this only skips having nothing to decide.
if (!anyPromise(files)) {
return { ok: true, modulePath: null, manifest: null, errors: [], notes: [] };
}
let lowered;
try {
lowered = lowerProgram(files, builtinShapes());
} catch (err) {
return { ok: false, modulePath: null, manifest: null, notes: [], errors: [String(err)] };
}
const broken = brokenPromises(lowered);
if (broken.length > 0) {
// The marker's whole point: a refusal against a promised system is an error
// with a line, not a silent fall back to the interpreter.
return {
ok: false,
modulePath: null,
manifest: null,
notes: [],
errors: broken.map((d) => `${d.file}:${d.line}: ${d.system ?? ''} is @compiled but ${d.message}`),
};
}
const promised = new Set(lowered.required);
const chosen = lowered.module.systems.filter((s) => promised.has(s.name));
const notes = lowered.diagnostics
.filter((d) => d.severity === 'note' && d.system)
.map((d) => `${d.system}: ${d.message}`);
if (chosen.length === 0) {
// Nothing was promised, so there is nothing to build and nothing to say.
return { ok: true, modulePath: null, manifest: null, errors: [], notes };
}
const bad = chosen.flatMap((s) => verifySystem(s, lowered.module.comps, lowered.module.fns));
if (bad.length > 0) {
return {
ok: false, modulePath: null, manifest: null, notes,
errors: bad.map((e) => `${e.system}: ${e.message}`),
};
}
if (opts.outputs ? opts.outputs.length === 0 : !opts.cc) {
return {
ok: false, modulePath: null, manifest: null, notes,
errors: [`${chosen.length} system(s) are marked @compiled but ${opts.outputs ? opts.missing ?? plan.missing : plan.missing}`
+ ', or remove the marker'],
};
}
const layout = packLayout(lowered.module.comps);
const inlined = chosen.map((s) => inlineSystem(s, lowered.module.fns));
const c = emitC(lowered.module, layout, inlined, plan.addressBytes);
const dir = path.join(root, CACHE_DIR);
rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
mkdirSync(dir, { recursive: true });
writeFileSync(path.join(dir, 'estella_abi.h'), c.header);
writeFileSync(path.join(dir, 'estella_offsets.h'), c.offsets);
writeFileSync(path.join(dir, 'systems.c'), c.source);
const units = [path.join(dir, 'systems.c')];
if (plan.compilesDecls) {
writeFileSync(path.join(dir, 'systems_decl.c'), c.decls);
units.push(path.join(dir, 'systems_decl.c'));
}
// One C, as many machines as were asked for: the host's own by default, or one
// module per output (a cross-compiled package's ABIs), each in its own folder.
const jobs = opts.outputs
? opts.outputs.map((o) => ({ id: o.id, cc: o.cc, flags: [...o.flags], out: path.join(dir, o.id, o.module) }))
: [{ id: plan.module, cc: opts.cc!, flags: plan.linkFlags(c.symbols), out: path.join(dir, plan.module) }];
const modules: Record<string, string> = {};
for (const job of jobs) {
mkdirSync(path.dirname(job.out), { recursive: true });
const built = await opts.run(job.cc, [...CFLAGS, '-Wall', '-Wextra', ...job.flags, '-o', job.out, ...units], dir);
if (built.code !== 0 || !existsSync(job.out)) {
return { ok: false, modulePath: null, manifest: null, notes,
errors: [`${opts.outputs ? `${job.id}: ` : ''}${built.stderr.trim()}`] };
}
modules[job.id] = job.out;
}
const modulePath = jobs[0].out;
const manifest: CompiledSystemsManifest = {
engineAbi: c.handshake.engineAbi,
projectShapes: c.handshake.projectShapes,
moduleContract: c.handshake.moduleContract,
// The one mapping from plans to declarations, shared with whatever else
// installs a module it just compiled — this was a second copy of the C
// table the artifact carries.
systems: moduleDeclarations(lowered.module, inlined),
};
writeFileSync(path.join(dir, MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`);
return { ok: true, modulePath, ...(opts.outputs ? { modules } : {}), manifest, errors: [], notes };
}
/** Read back what a previous build wrote, or null. */
export function readCompiledManifest(root: string): CompiledSystemsManifest | null {
const at = path.join(root, CACHE_DIR, MANIFEST);
if (!existsSync(at)) return null;
return JSON.parse(readFileSync(at, 'utf8')) as CompiledSystemsManifest;
}