A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(native): build the app's compiled half once, as a runtime template
esengineeedc3bb2f250
diff --git a/build-tools/tasks/nativePackage.js b/build-tools/tasks/nativePackage.jsindex d26d3a175..2b4ea5561 100644--- a/build-tools/tasks/nativePackage.js+++ b/build-tools/tasks/nativePackage.js@@ -8,30 +8,20 @@import path from 'path';import { existsSync, readFileSync } from 'fs';-import { mkdir, copyFile, cp, rm, writeFile, readdir } from 'fs/promises';+import { mkdir, copyFile, cp, rm, writeFile } from 'fs/promises';import config from '../build.config.js';import * as logger from '../utils/logger.js';import { runCommand } from '../utils/emscripten.js';import { requireSdk, requireNdk, buildTool, platformJar, ndkTool, ndkLibcxxShared, javaHome, jdkTool } from '../utils/android.js';import { readAppConfig, androidScreenOrientation, fillTemplate } from '../utils/nativeApp.js';+import { BYTECODE_FILE } from '../utils/nativeTemplate.js';+import { compileJavaShim } from './nativeTemplateEmit.js';const HOST_LIBRARY = 'libestella_js_host.so';-/**- * The shipped bytecode's filename. NOT `.bc`: that is LLVM bitcode's extension, and- * aapt2 treats a `.bc` asset as RenderScript output — which silently drops the APK's- * native-code declaration, so the package installs nowhere. Cost a long hunt; the- * only symptom is INSTALL_FAILED_NO_MATCHING_ABIS on a device whose ABI matches.- */-export const BYTECODE_FILE = 'esengine.native.qjsbc';const MANIFEST_TEMPLATE = path.join('native', 'android', 'host', 'AndroidManifest.xml.in');-/** The Java shim's sources — the IME's own side of an editable field, which has- * to be Java because only a Java View gets an InputConnection to compose into- * (see native/android/java/.../TextEditor.java). */-const JAVA_DIR = path.join('native', 'android', 'java');-/** The APK is named after the app it contains, so packaging a second project does* not quietly replace the first one's file. */const apkName = (appId) => `${appId.split('.').pop() || 'estella'}.apk`;@@ -60,41 +50,6 @@ async function debugKeystore(jdk) {// a zip entry name must use forward slashes — so AAssetManager could not open// anything in a subdirectory. Invisible while the packaged content was a handful// of flat files; every real project export has directories.-/**- * Compile the Java shim to a classes.dex in `staging`. javac + d8 are already- * beside the tools packaging uses (the JDK apksigner needs, the build-tools aapt2- * comes from), so this stays a plain APK build with no Gradle. Returns false when- * there is nothing to compile, which leaves an APK whose host reports no editing- * surface rather than one that fails to build.- */-async function compileJavaShim(staging, sdk, platform, jdk, rootDir) {- const sourceDir = path.join(rootDir, JAVA_DIR);- if (!existsSync(sourceDir)) return false;- const sources = (await readdir(sourceDir, { recursive: true }))- .filter((f) => String(f).endsWith('.java'))- .map((f) => path.join(sourceDir, String(f)));- if (sources.length === 0) return false;-- const classes = path.join(staging, 'classes');- await mkdir(classes, { recursive: true });- // Java 8 bytecode: d8 accepts it everywhere, and the shim uses nothing newer.- await runCommand(jdkTool('javac', jdk), [- // The sources are UTF-8; javac otherwise reads them in the platform's- // default charset, which on a Windows build machine is not.- '-encoding', 'UTF-8', '-source', '8', '-target', '8', '-nowarn',- '-bootclasspath', platformJar(sdk, platform),- '-d', classes, ...sources,- ]);- const classFiles = (await readdir(classes, { recursive: true }))- .filter((f) => String(f).endsWith('.class'))- .map((f) => path.join(classes, String(f)));- await runCommand(buildTool(sdk, 'd8'), [- '--lib', platformJar(sdk, platform),- '--output', staging, ...classFiles,- ]);- return existsSync(path.join(staging, 'classes.dex'));-}-async function addToApk(apk, stagingDir, entries, jdk) {await runCommand(jdkTool('jar', jdk), ['ufM', apk, ...entries.flatMap((e) => ['-C', stagingDir, e])]);}@@ -168,7 +123,9 @@ export async function packageNativeApk(options = {}) {// OS must lock the window to. The manifest is a template because all four vary// per project — a static one is why every example installed over the last.logger.step('Compiling the Java shim (javac + d8)...');- const hasDex = await compileJavaShim(staging, sdk, platform, options.jdk, rootDir);+ const hasDex = await compileJavaShim(staging, {+ sdk, androidPlatform: platform, jdk: options.jdk, root: rootDir,+ });if (!hasDex) logger.warn('No classes.dex: the app will have no soft keyboard (typing is off).');const app = readAppConfig(content, (m) => logger.warn(m));diff --git a/build-tools/tasks/nativeTemplateEmit.js b/build-tools/tasks/nativeTemplateEmit.jsnew file mode 100644index 000000000..1a1c0343c--- /dev/null+++ b/build-tools/tasks/nativeTemplateEmit.js@@ -0,0 +1,160 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+//+// Emitting a runtime template — the ONLY step that knows about build trees.+//+// Everything downstream (the editor's iOS project, the APK assembler) reads a+// template and never a build directory, so the toolchain that produced these+// binaries has to exist on exactly one machine: this one. Shipping the result is+// what frees a user from cloning the engine, building Dawn and installing an NDK+// to put a game on a phone.++import path from 'path';+import { existsSync, readFileSync, writeFileSync } from 'fs';+import { mkdir, rm, cp, readdir } from 'fs/promises';+import config from '../build.config.js';+import * as logger from '../utils/logger.js';+import { runCommand } from '../utils/emscripten.js';+import { buildTool, platformJar, ndkTool, ndkLibcxxShared, jdkTool } from '../utils/android.js';+import { makeZip, zipTree } from '../utils/zip.js';+import {+ templateLayout, templateId, DEFAULT_ABI, writeTemplateManifest, missingTemplateFiles,+ installedTemplateDir, templateZipName,+} from '../utils/nativeTemplate.js';++/** The Java shim's sources — the IME's own side of an editable field, which has to+ * be Java because only a Java View gets an InputConnection to compose into. */+const JAVA_DIR = path.join('native', 'android', 'java');++/** The NDK triple an ABI's sysroot lives under. */+const ABI_TRIPLE = {+ 'arm64-v8a': 'aarch64-linux-android',+ 'x86_64': 'x86_64-linux-android',+};++/**+ * The version a template is stamped with — `desktop/package.json`, which is also+ * what `app.getVersion()` reports in the editor. The two must be the same number:+ * matching them is how the editor refuses a template built for another release.+ */+export function readEngineVersion(root = config.paths.root) {+ return JSON.parse(readFileSync(path.join(root, 'desktop', 'package.json'), 'utf8')).version;+}++/**+ * Compile the Java shim to a `classes.dex` in @p outDir. javac + d8 come from the+ * JDK and the build-tools this machine already needed to produce the binaries, so+ * the template carries the dex and no user needs a JDK to package.+ *+ * @returns Whether a dex was produced — false leaves an app whose host reports no+ * editing surface, rather than one that cannot be built at all.+ */+export async function compileJavaShim(outDir, { sdk, androidPlatform, jdk, root }) {+ const sourceDir = path.join(root, JAVA_DIR);+ if (!existsSync(sourceDir)) return false;+ const sources = (await readdir(sourceDir, { recursive: true }))+ .filter((f) => String(f).endsWith('.java'))+ .map((f) => path.join(sourceDir, String(f)));+ if (sources.length === 0) return false;++ const classes = path.join(outDir, 'classes');+ await mkdir(classes, { recursive: true });+ // Java 8 bytecode: d8 accepts it everywhere, and the shim uses nothing newer.+ await runCommand(jdkTool('javac', jdk), [+ // The sources are UTF-8; javac otherwise reads them in the platform's+ // default charset, which on a Windows build machine is not.+ '-encoding', 'UTF-8', '-source', '8', '-target', '8', '-nowarn',+ '-bootclasspath', platformJar(sdk, androidPlatform),+ '-d', classes, ...sources,+ ]);+ const classFiles = (await readdir(classes, { recursive: true }))+ .filter((f) => String(f).endsWith('.class'))+ .map((f) => path.join(classes, String(f)));+ await runCommand(buildTool(sdk, 'd8'), ['--lib', platformJar(sdk, androidPlatform), '--output', outDir, ...classFiles]);+ await rm(classes, { recursive: true, force: true });+ return existsSync(path.join(outDir, 'classes.dex'));+}++/**+ * Pack a finished native build into a runtime template.+ *+ * @param {object} options+ * @param {'android'|'ios'} options.platform+ * @param {string} [options.abi]+ * @param {string} [options.dawnBuild] Android: the Dawn build whose .so ships.+ * @param {string} [options.ndk] Android: for llvm-strip and libc++_shared.+ * @param {string} [options.sdk] Android: for d8 (the Java shim).+ * @param {string} [options.outDir] Defaults to this machine's template store.+ * @param {string} [options.zipTo] Also write the distributable archive here.+ * @returns {Promise<{dir: string, manifest: object, zip: string|null}>}+ */+export async function emitNativeTemplate(options) {+ const root = options.root || config.paths.root;+ const platform = options.platform;+ const abi = options.abi || DEFAULT_ABI[platform];+ const engineVersion = readEngineVersion(root);+ const dir = options.outDir || installedTemplateDir(engineVersion, platform, abi);++ const ctx = {+ root,+ dawnBuild: options.dawnBuild,+ libcxxShared: platform === 'android' && options.ndk+ ? ndkLibcxxShared(options.ndk, ABI_TRIPLE[abi] || ABI_TRIPLE['arm64-v8a'])+ : null,+ };++ // Rewritten wholesale: a template that inherited one file from the previous+ // release is the exact failure the version stamp exists to prevent.+ await rm(dir, { recursive: true, force: true });+ await mkdir(dir, { recursive: true });++ for (const entry of templateLayout(platform, { abi })) {+ if (entry.produced) continue;+ const src = entry.from(ctx);+ const dest = path.join(dir, entry.rel);+ if (!src || !existsSync(src)) {+ if (entry.optional) {+ logger.warn(`Template omits ${entry.rel} (not built): ${src ?? 'no source'}`);+ continue;+ }+ throw new Error(`Cannot emit the ${platform} template: ${entry.rel} is missing at ${src ?? '<unset>'}.`);+ }+ await mkdir(path.dirname(dest), { recursive: true });+ if (entry.strip) {+ await runCommand(ndkTool(options.ndk, 'llvm-strip'), ['--strip-all', '-o', dest, src]);+ } else {+ await cp(src, dest, { recursive: entry.kind === 'dir' });+ }+ }++ if (platform === 'android') {+ if (!await compileJavaShim(dir, { sdk: options.sdk, androidPlatform: options.androidPlatform, jdk: options.jdk, root })) {+ throw new Error('Cannot emit the android template: the Java shim produced no classes.dex '+ + '(the packaged app would have no soft keyboard).');+ }+ }++ const manifest = writeTemplateManifest(dir, {+ platform,+ abi,+ engineVersion,+ spineVersion: options.spineVersion || '4.2',+ ...(platform === 'ios' ? { deploymentTarget: options.deploymentTarget || '17.0' } : {}),+ ...(platform === 'android' ? { androidPlatform: options.androidPlatform || 'android-33' } : {}),+ });++ const missing = missingTemplateFiles(dir, platform, { abi });+ if (missing.length) throw new Error(`Emitted template is incomplete: ${missing.join(', ')}`);++ logger.success(`Runtime template: ${dir}`);++ let zip = null;+ if (options.zipTo) {+ await mkdir(options.zipTo, { recursive: true });+ zip = path.join(options.zipTo, templateZipName(platform, abi, engineVersion));+ writeFileSync(zip, makeZip(zipTree(dir)));+ logger.success(`Template archive: ${zip} (${(readFileSync(zip).length / 1048576).toFixed(1)} MB)`);+ }+ logger.info(`The editor picks it up as ${templateId(platform, abi)} for v${engineVersion}.`);+ return { dir, manifest, zip };+}diff --git a/build-tools/utils/nativeTemplate.d.ts b/build-tools/utils/nativeTemplate.d.tsnew file mode 100644index 000000000..51bf8c8cf--- /dev/null+++ b/build-tools/utils/nativeTemplate.d.ts@@ -0,0 +1,58 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+//+// Types for nativeTemplate.js — the module is plain ESM so the CLI can run it+// unbuilt, and this is how the editor's TypeScript sees it.++export type NativePlatform = 'android' | 'ios';++export interface NativeTemplateManifest {+ kind: 'estella-native-template';+ formatVersion: number;+ id: string;+ platform: NativePlatform;+ abi: string;+ /** Matched EXACTLY against the editor's version: the SDK bundle is compiled+ * into the host binary, so a near-miss fails on a device. */+ engineVersion: string;+ spineVersion: string;+ /** iOS only. */+ deploymentTarget?: string;+ /** Android only. */+ androidPlatform?: string;+}++export interface TemplateWant {+ platform: NativePlatform;+ engineVersion: string;+ abi?: string;+}++export interface FoundTemplate {+ dir: string;+ manifest: NativeTemplateManifest;+ /** Required files the directory does not actually hold. */+ missing: string[];+}++export const TEMPLATE_FORMAT: number;+export const TEMPLATE_MANIFEST: string;+export const BYTECODE_FILE: string;+export const DEFAULT_ABI: Readonly<Record<NativePlatform, string>>;++export function templateId(platform: NativePlatform, abi: string): string;+export function requiredTemplateFiles(platform: NativePlatform, options?: { abi?: string }): string[];+export function missingTemplateFiles(dir: string, platform: NativePlatform, options?: { abi?: string }): string[];+export function readTemplateManifest(dir: string): NativeTemplateManifest | null;+export function templateMatches(manifest: NativeTemplateManifest | null, want: TemplateWant): boolean;+export function templateZipName(platform: NativePlatform, abi: string, engineVersion: string): string;+export function estellaDataDir(): string;+export function templateStoreDir(): string;+export function installedTemplateDir(+ engineVersion: string, platform: NativePlatform, abi: string, storeDir?: string): string;+export function findTemplate(want: TemplateWant, storeDir?: string): FoundTemplate | null;+export function iosTemplateSources(dir: string): {+ xcframework: string;+ mainM: string;+ infoPlistIn: string;+};diff --git a/build-tools/utils/nativeTemplate.js b/build-tools/utils/nativeTemplate.jsnew file mode 100644index 000000000..12d5f144f--- /dev/null+++ b/build-tools/utils/nativeTemplate.js@@ -0,0 +1,213 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+//+// The runtime template contract — what a prebuilt native runtime IS, and where it+// lives once installed.+//+// A native app's binary carries no project data: the export is content only, and+// `native/CMakeLists.txt` takes no project input (every ES_ENABLE_* is on, and the+// optional runtimes are linked in). So the compiled half is the SAME for every+// game, and belongs in an artifact built once per release rather than on each+// user's machine. That artifact is a runtime template.+//+// This module is the ONE place that names the files in one: the emitter writes+// what the table says, the editor looks for what the table says, and a template+// that satisfies the table is usable by definition. Plain ESM because both halves+// read it — build-tools and the packaged editor.++import path from 'path';+import { existsSync, readFileSync, writeFileSync } from 'fs';++/** Bumped when the LAYOUT changes shape — an older template is then refused by+ * name instead of failing halfway through an assembly. */+export const TEMPLATE_FORMAT = 1;++export const TEMPLATE_MANIFEST = 'template.json';++/** The shipped bytecode's filename. NOT `.bc`: that is LLVM bitcode's extension,+ * and aapt2 treats a `.bc` asset as RenderScript output — which silently drops+ * the APK's native-code declaration, so the package installs nowhere. */+export const BYTECODE_FILE = 'esengine.native.qjsbc';++/** A template's id: one per thing that must be compiled separately. */+export function templateId(platform, abi) {+ return `${platform}-${abi}`;+}++/** The default ABI a platform's template carries. iOS ships both slices inside+ * one xcframework, so its id names the device architecture only. */+export const DEFAULT_ABI = { android: 'arm64-v8a', ios: 'arm64' };++/**+ * What a template holds, per platform.+ *+ * `from` resolves the file in a BUILD TREE (the emitter's input); `rel` is where+ * it sits inside the template (what every consumer sees). `strip` marks a binary+ * whose debug info an app does not need — the host's is most of its size, and+ * shipping it unstripped also broke installing onto a clean device.+ *+ * @param {'android'|'ios'} platform+ * @param {{abi?: string}} [options]+ */+export function templateLayout(platform, options = {}) {+ const abi = options.abi || DEFAULT_ABI[platform];+ if (platform === 'ios') {+ return [+ {+ rel: 'Estella.xcframework', kind: 'dir',+ from: (ctx) => path.join(ctx.root, 'build-native-ios', 'Estella.xcframework'),+ },+ { rel: 'App/main.m', from: (ctx) => path.join(ctx.root, 'native', 'ios', 'App', 'main.m') },+ { rel: 'App/Info.plist.in', from: (ctx) => path.join(ctx.root, 'native', 'ios', 'App', 'Info.plist.in') },+ ];+ }+ if (platform === 'android') {+ return [+ {+ rel: `lib/${abi}/libestella_js_host.so`, strip: true,+ from: (ctx) => path.join(ctx.root, 'build-native', 'libestella_js_host.so'),+ },+ {+ rel: `lib/${abi}/libwebgpu_dawn.so`, strip: true,+ from: (ctx) => path.join(ctx.dawnBuild, 'src', 'dawn', 'native', 'libwebgpu_dawn.so'),+ },+ { rel: `lib/${abi}/libc++_shared.so`, from: (ctx) => ctx.libcxxShared },+ // Compiled by the emitter (javac + d8), not copied: the IME's own side+ // of an editable field has to be Java, and a template exists so that no+ // user needs a JDK to package.+ { rel: 'classes.dex', produced: true },+ // Absent is valid — the host falls back to compiling the bundle, at the+ // cost of a ~14 s black screen on first launch.+ {+ rel: `assets/${BYTECODE_FILE}`, optional: true,+ from: (ctx) => path.join(ctx.root, 'build-native', 'gen', BYTECODE_FILE),+ },+ {+ rel: 'AndroidManifest.xml.in',+ from: (ctx) => path.join(ctx.root, 'native', 'android', 'host', 'AndroidManifest.xml.in'),+ },+ ];+ }+ throw new Error(`Unknown native platform "${platform}" (expected android or ios).`);+}++/** Template-relative paths that must be present for a template to be usable. */+export function requiredTemplateFiles(platform, options) {+ return templateLayout(platform, options).filter((e) => !e.optional).map((e) => e.rel);+}++/** What a template is missing, if anything — the one readiness test. */+export function missingTemplateFiles(dir, platform, options) {+ return requiredTemplateFiles(platform, options).filter((rel) => !existsSync(path.join(dir, rel)));+}++// =============================================================================+// The manifest+// =============================================================================++/**+ * Read a template's `template.json`. Null when the directory holds no template,+ * or one this build cannot read — callers treat both as "not installed", which is+ * the only honest answer either way.+ */+export function readTemplateManifest(dir) {+ const file = path.join(dir, TEMPLATE_MANIFEST);+ if (!existsSync(file)) return null;+ try {+ const parsed = JSON.parse(readFileSync(file, 'utf8'));+ if (parsed?.kind !== 'estella-native-template') return null;+ if (parsed.formatVersion !== TEMPLATE_FORMAT) return null;+ return parsed;+ } catch {+ return null;+ }+}++export function writeTemplateManifest(dir, meta) {+ const manifest = {+ kind: 'estella-native-template',+ formatVersion: TEMPLATE_FORMAT,+ id: templateId(meta.platform, meta.abi),+ ...meta,+ };+ // No timestamp: two builds of the same commit must produce byte-identical+ // artifacts, or a published checksum means nothing.+ writeFileSync(path.join(dir, TEMPLATE_MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');+ return manifest;+}++/**+ * Whether an installed template can run this editor's exports.+ *+ * The match is EXACT on version. The SDK bundle is compiled into the host binary,+ * and the cooked content, the scene format and the `es_*` surface all travel with+ * it — a near-miss is a crash on a device rather than a smaller problem.+ */+export function templateMatches(manifest, want) {+ return !!manifest+ && manifest.platform === want.platform+ && manifest.engineVersion === want.engineVersion+ && (!want.abi || manifest.abi === want.abi);+}++/** The distributed artifact's filename. */+export function templateZipName(platform, abi, engineVersion) {+ return `estella-native-${templateId(platform, abi)}-${engineVersion}.zip`;+}++// =============================================================================+// Where templates live+// =============================================================================++/**+ * Estella's own data directory. Deliberately not Electron's `userData`: the CLI+ * emits templates and the editor consumes them, so the path has to be one both+ * can compute the same way, without importing Electron.+ */+export function estellaDataDir() {+ const home = process.env.HOME || process.env.USERPROFILE || '';+ if (process.platform === 'win32') return path.join(process.env.APPDATA || path.join(home, 'AppData', 'Roaming'), 'Estella');+ if (process.platform === 'darwin') return path.join(home, 'Library', 'Application Support', 'Estella');+ return path.join(process.env.XDG_DATA_HOME || path.join(home, '.local', 'share'), 'Estella');+}++/** The template store. `ESTELLA_NATIVE_TEMPLATES` overrides it — for CI, for a+ * shared network location, and for tests. */+export function templateStoreDir() {+ return process.env.ESTELLA_NATIVE_TEMPLATES || path.join(estellaDataDir(), 'native-templates');+}++/** Where one template installs to. Versioned, so upgrading the editor never+ * silently reuses the previous release's binary. */+export function installedTemplateDir(engineVersion, platform, abi, storeDir = templateStoreDir()) {+ return path.join(storeDir, engineVersion, templateId(platform, abi));+}++/**+ * The installed template for this platform, or null.+ *+ * @returns {{dir: string, manifest: object, missing: string[]}|null}+ */+export function findTemplate(want, storeDir = templateStoreDir()) {+ const abi = want.abi || DEFAULT_ABI[want.platform];+ const dir = installedTemplateDir(want.engineVersion, want.platform, abi, storeDir);+ const manifest = readTemplateManifest(dir);+ if (!templateMatches(manifest, { ...want, abi })) return null;+ return { dir, manifest, missing: missingTemplateFiles(dir, want.platform, { abi }) };+}++// =============================================================================+// Per-platform consumers+// =============================================================================++/**+ * The three pieces an iOS Xcode project is built around. Named here rather than+ * at the call site so the layout table stays the only thing that knows filenames.+ */+export function iosTemplateSources(dir) {+ return {+ xcframework: path.join(dir, 'Estella.xcframework'),+ mainM: path.join(dir, 'App', 'main.m'),+ infoPlistIn: path.join(dir, 'App', 'Info.plist.in'),+ };+}