Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(aot): the same systems, built as a library a native host loads

yhhc8e99273c7b7
diff --git a/compiler/src/codegen.ts b/compiler/src/codegen.ts
index ac48285c7..edbc14f00 100644
--- a/compiler/src/codegen.ts
+++ b/compiler/src/codegen.ts
@@ -804,7 +804,16 @@ class Emitter {
* reaction to invalid EIR is to throw, which is the right reaction to a compiler
* bug and the wrong one to a source file outside the subset.
*/
-export function emitC(module: EirModule, layout: AbiLayout, systems: readonly EirSystem[]): CModule {
+export function emitC(
+ module: EirModule,
+ layout: AbiLayout,
+ systems: readonly EirSystem[],
+ /** `sizeof(es_addr_t)` where this artifact will run: 4 for a host owning one
+ * block it addresses by offset, 8 where an address is a pointer. The machine
+ * and not the contract, so it is mixed into the handshake and not into the
+ * offsets — the digest is what refuses a 32-bit module to a 64-bit host. */
+ addressBytes: 4 | 8 = 4,
+): CModule {
const offsets = new Map<string, number>();
const plans = systems.map((s) => planFor(s));
const bodies: string[] = [];
@@ -824,7 +833,7 @@ export function emitC(module: EirModule, layout: AbiLayout, systems: readonly Ei
});
const manifest = emitManifest(systems, plans);
- const handshake = abiHandshake(module.comps, plans);
+ const handshake = abiHandshake(module.comps, plans, addressBytes);
const macros = [...offsets].sort(([a], [b]) => (a < b ? -1 : 1)).map(([k, v]) => `#define ${k} ${v}u`);
const offsetsHeader = [
'/* GENERATED by @estella/compiler — do not edit. */',
diff --git a/compiler/src/hostCC.ts b/compiler/src/hostCC.ts
new file mode 100644
index 000000000..3b7d34d3f
--- /dev/null
+++ b/compiler/src/hostCC.ts
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file hostCC.ts
+ * @brief The C compiler that builds for the machine this is running on.
+ *
+ * @details Two callers want the same answer and must not each have their own:
+ * the differentials, which compile the runtime header and compare bits,
+ * and the AOT build step, which compiles a project's systems into a
+ * library a native host loads into its own process.
+ *
+ * clang first because it is what the wasm and aarch64 paths are; gcc is
+ * accepted, and is arguably the better witness — agreeing with a SECOND
+ * implementation of C says more than agreeing with the one the lowering
+ * was written against.
+ *
+ * The candidate has to BUILD something, not merely answer `--version`.
+ * emsdk's clang shadows the system one on PATH, reports `Target:
+ * unknown`, and answers `--version` with 0 while refusing every host
+ * compile — so the wrong probe reddens exactly the machines that can
+ * run AOT.
+ */
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+/** What a loadable module is called here. The host resolves it by name. */
+export function nativeModuleExt(platform: string = process.platform): string {
+ if (platform === 'win32') return '.dll';
+ return platform === 'darwin' ? '.dylib' : '.so';
+}
+
+/**
+ * Linking a compiled system into a module a native host loads.
+ *
+ * `-fPIC` because it is loaded, not linked; nothing else, because the C calls
+ * only what `estella_abi.h` defines `static inline` — the module has no
+ * undefined symbol to resolve and wants none of libc.
+ */
+export const NATIVE_LINK_FLAGS: readonly string[] = ['-shared', '-fPIC'];
+
+/**
+ * The host C compiler, or null where this machine has none.
+ *
+ * Never throws and never installs anything: a machine without one still builds
+ * every project that promised nothing, so the refusal belongs to the step that
+ * knows whether anything was promised.
+ */
+export function findHostCC(): string | null {
+ const dir = mkdtempSync(join(tmpdir(), 'estella-cc-'));
+ const src = join(dir, 'probe.c');
+ writeFileSync(src, 'int main(void) { return 0; }\n');
+ for (const cc of ['clang', 'gcc', 'cc']) {
+ if (spawnSync(cc, ['-std=c11', '-o', join(dir, `probe-${cc}`), src], { encoding: 'utf8' }).status === 0) {
+ return cc;
+ }
+ }
+ return null;
+}
diff --git a/compiler/tests/codegen.test.ts b/compiler/tests/codegen.test.ts
index b6cf08bed..d65b4f7c5 100644
--- a/compiler/tests/codegen.test.ts
+++ b/compiler/tests/codegen.test.ts
@@ -39,7 +39,7 @@ import { inlineSystem } from '../src/inline';
import { builtinShapes } from '../src/builtins';
import { AbiMemory, flushCommands, materialize, packLayout, planFor, runOnAbi } from '../src/abi';
import { CFLAGS, cSymbol, emitC, type CModule } from '../src/codegen';
-import { findCC } from './hostCC';
+import { findHostCC } from '../src/hostCC';
import { abiHashFor } from '../src/abi';
import type { AbiLayout } from '../src/abi';
import { F64, type EirSystem } from '../src/eir';
@@ -58,7 +58,7 @@ const FRAMES = 12;
/** Small enough that a frame is one cheap pipe, big enough for the world above. */
const IMAGE = 1 << 16;
-const CC = findCC();
+const CC = findHostCC();
/** The harness: linear memory in on stdin, linear memory out on stdout. */
const MAIN_C = `/* Test scaffolding, not compiler output. */
diff --git a/compiler/tests/exact-trig.test.ts b/compiler/tests/exact-trig.test.ts
index f2ec91fd2..1d479dfae 100644
--- a/compiler/tests/exact-trig.test.ts
+++ b/compiler/tests/exact-trig.test.ts
@@ -22,9 +22,9 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { CFLAGS, RUNTIME_H } from '../src/codegen';
import { exact } from '../../sdk/src/math/exact';
-import { findCC } from './hostCC';
+import { findHostCC } from '../src/hostCC';
-const CC = findCC();
+const CC = findHostCC();
/** Where a range reduction, a quadrant choice or a polynomial could differ. */
function arguments_(): number[] {
diff --git a/compiler/tests/hostCC.ts b/compiler/tests/hostCC.ts
deleted file mode 100644
index 6cacb9114..000000000
--- a/compiler/tests/hostCC.ts
+++ /dev/null
@@ -1,33 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
-/**
- * @file hostCC.ts
- * @brief The host C compiler a differential needs, or nothing.
- *
- * @details clang first because it is what Stage 2 ships with — emcc is clang and
- * the aarch64 path is clang. gcc is accepted, and is arguably the better
- * witness: agreeing with a SECOND independent implementation of C says
- * more than agreeing with the one the lowering was written against.
- *
- * The candidate has to BUILD something, not merely answer `--version`.
- * emsdk's clang shadows the system one on PATH, reports `Target:
- * unknown`, and answers `--version` with 0 while refusing every host
- * compile — so the wrong probe reddens these differentials on exactly
- * the machines that can run AOT.
- */
-import { spawnSync } from 'node:child_process';
-import { mkdtempSync, writeFileSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-
-export function findCC(): string | null {
- const dir = mkdtempSync(join(tmpdir(), 'estella-cc-'));
- const src = join(dir, 'probe.c');
- writeFileSync(src, 'int main(void) { return 0; }\n');
- for (const cc of ['clang', 'gcc', 'cc']) {
- if (spawnSync(cc, ['-std=c11', '-o', join(dir, `probe-${cc}`), src], { encoding: 'utf8' }).status === 0) {
- return cc;
- }
- }
- return null;
-}
diff --git a/compiler/tests/native-module.test.ts b/compiler/tests/native-module.test.ts
new file mode 100644
index 000000000..22cb422b3
--- /dev/null
+++ b/compiler/tests/native-module.test.ts
@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file native-module.test.ts
+ * @brief The same C, built as a library a native host can load.
+ *
+ * @details The wasm road is gated end to end and the C++ dispatch half is gated
+ * against a real Registry (tests/aot/test_aot_host.cpp). Between them
+ * sits the step nothing exercised: taking the emitted C to the HOST
+ * compiler with the flags a loadable module needs, at the width where
+ * an address is a pointer rather than an offset into one block.
+ *
+ * What it pins is the part that can silently be wrong: that the module
+ * links with no undefined symbol (the C calls only what the header
+ * defines `static inline`, and a libc call would appear here), that the
+ * promised systems are exported under the names the manifest hands a
+ * loader, and that the wider address changes the handshake — a 32-bit
+ * artifact loading into a 64-bit host reads every row at half a
+ * pointer.
+ */
+import { describe, it, expect } from 'vitest';
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, writeFileSync, existsSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { lowerProgram } from '../src/frontend';
+import { verifySystem } from '../src/verify';
+import { inlineSystem } from '../src/inline';
+import { builtinShapes } from '../src/builtins';
+import { packLayout } from '../src/abi';
+import { CFLAGS, cSymbol, emitC } from '../src/codegen';
+import { NATIVE_LINK_FLAGS, findHostCC, nativeModuleExt } from '../src/hostCC';
+
+const ROOT = resolve(fileURLToPath(new URL('../..', import.meta.url)));
+const CC = findHostCC();
+
+/** The example that carries the marker: a real system, not a fixture written
+ * to compile. */
+const SOURCES = [
+ join(ROOT, 'examples/ecs-basics/src/components.ts'),
+ join(ROOT, 'examples/ecs-basics/src/systems/move.ts'),
+];
+
+const lowered = lowerProgram(SOURCES, builtinShapes());
+const move = lowered.module.systems.find((s) => s.name === 'MoveSystem');
+
+describe('the compiled systems, as a module a native host loads', () => {
+ it('reports whether this gate could run at all', () => {
+ if (!CC) console.warn('[native-module] NO HOST C COMPILER — the build did NOT run.');
+ expect(move, 'MoveSystem never lowered').toBeDefined();
+ expect(verifySystem(move!, lowered.module.comps, lowered.module.fns)).toEqual([]);
+ });
+
+ it.skipIf(!CC)('links with no undefined symbol, and exports what the manifest names', () => {
+ const layout = packLayout(lowered.module.comps);
+ const c = emitC(lowered.module, layout, [inlineSystem(move!, lowered.module.fns)], 8);
+
+ const dir = mkdtempSync(join(tmpdir(), 'estella-native-'));
+ writeFileSync(join(dir, 'estella_abi.h'), c.header);
+ writeFileSync(join(dir, 'estella_offsets.h'), c.offsets);
+ writeFileSync(join(dir, 'systems.c'), c.source);
+ // The declarations are the DATA half. A wasm module must not carry them
+ // (its data section would sit at an address the engine owns); a library
+ // loaded into the host's own process is exactly who does.
+ writeFileSync(join(dir, 'systems_decl.c'), c.decls);
+
+ const out = join(dir, `systems${nativeModuleExt()}`);
+ const built = spawnSync(CC!, [
+ ...CFLAGS, '-Wall', '-Wextra', ...NATIVE_LINK_FLAGS,
+ '-o', out, join(dir, 'systems.c'), join(dir, 'systems_decl.c'),
+ ], { encoding: 'utf8', cwd: dir });
+
+ expect(built.status, built.stderr).toBe(0);
+ // A warning here is a contract problem, not style: every one this C can
+ // produce is about a type or a conversion the lowering chose.
+ expect(built.stderr.trim()).toBe('');
+ expect(existsSync(out)).toBe(true);
+
+ // The symbol a loader asks for by name. Read out of the built file rather
+ // than assumed, because "it compiled" and "it exports that" are two
+ // claims and only the second is what a host depends on.
+ const want = cSymbol(move!.name);
+ expect(symbolsOf(out)).toContain(want);
+ });
+
+ it('the wider address is in the handshake, so the two artifacts cannot be swapped', () => {
+ const layout = packLayout(lowered.module.comps);
+ const sys = [inlineSystem(move!, lowered.module.fns)];
+ const asOffsets = emitC(lowered.module, layout, sys, 4);
+ const asPointers = emitC(lowered.module, layout, sys, 8);
+
+ // The C is the same file: the width is a typedef the building compiler
+ // picks, which is why one source serves both hosts.
+ expect(asPointers.source).toBe(asOffsets.source);
+ // The digest is not, which is what refuses the wrong one at load.
+ expect(asPointers.handshake.engineAbi).not.toBe(asOffsets.handshake.engineAbi);
+ // And the project's own shapes are the contract, so they do NOT move.
+ expect(asPointers.handshake.projectShapes).toBe(asOffsets.handshake.projectShapes);
+ });
+});
+
+/**
+ * Exported function names in a built module.
+ *
+ * Three formats and one tool each; an unknown platform returns null rather than
+ * an empty list, so "no symbols" can never read as "the symbol is missing".
+ */
+function symbolsOf(file: string): string[] {
+ const run = (cmd: string, args: string[]): string | null => {
+ const r = spawnSync(cmd, args, { encoding: 'utf8' });
+ return r.status === 0 ? r.stdout : null;
+ };
+ // nm reads ELF, Mach-O and, with binutils on Windows, PE too.
+ const nm = run('nm', ['-g', '--defined-only', file]) ?? run('nm', ['-g', file]);
+ if (nm !== null) {
+ return nm.split(/\r?\n/)
+ .map((l) => l.trim().split(/\s+/).pop() ?? '')
+ .map((s) => (s.startsWith('_') ? s.slice(1) : s))
+ .filter(Boolean);
+ }
+ const dumpbin = run('dumpbin', ['/exports', file]);
+ if (dumpbin !== null) {
+ return dumpbin.split(/\r?\n/).map((l) => l.trim().split(/\s+/).pop() ?? '').filter(Boolean);
+ }
+ throw new Error('no symbol reader (nm or dumpbin) — this gate cannot say what the module exports');
+}