A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
perf(native): ship the SDK bundle's bytecode, so the first launch need not build it
esengine82c4bf28d3b9
diff --git a/build-tools/tasks/native.js b/build-tools/tasks/native.jsindex f1a64ace1..1ddfb5c43 100644--- a/build-tools/tasks/native.js+++ b/build-tools/tasks/native.js@@ -15,7 +15,7 @@ import config from '../build.config.js';import * as logger from '../utils/logger.js';import { runCommand, getCpuCount, resolvePython } from '../utils/emscripten.js';import { requireSdk, requireNdk, sdkCmake } from '../utils/android.js';-import { packageNativeApk } from './nativePackage.js';+import { packageNativeApk, BYTECODE_FILE } from './nativePackage.js';import { readAppConfig, iosInterfaceOrientations, fillTemplate } from '../utils/nativeApp.js';import { renderPbxproj, renderScheme } from '../utils/xcodeProject.js';@@ -108,18 +108,73 @@ async function generateSdkBundle(rootDir, genDir) {+ '(cd sdk && pnpm run build) — it produces the QuickJS-loadable index.native.bundled.js.');}const js = readFileSync(bundlePath, 'utf8');+ // The raw literal opens with a newline, so what the host compiles — and hashes+ // its bytecode cache against — is that newline plus the bundle. It is written+ // out as a file too, because the bytecode step must compile these exact bytes+ // for the host to accept the result (see precompileBundleBytecode).+ const embedded = '\n' + js;const header ='// The real esengine SDK, bundled (dist/index.native.bundled.js) — installs\n'+ '// globalThis.ESEngine. Embedded by `cli native --quickjs` — DO NOT EDIT.\n'+ '#pragma once\n'- + 'static const char* kSdkBundleJS = R"ESJS(\n'- + js+ + 'static const char* kSdkBundleJS = R"ESJS('+ + embedded+ ')ESJS";\n';const headerPath = path.join(genDir, 'esengine_bundle.h');writeFileSync(headerPath, header, 'utf8');+ writeFileSync(path.join(genDir, 'esengine_bundle.embedded.js'), embedded, 'utf8');return headerPath;}+/**+ * Build the SDK bundle's bytecode now, so the first launch does not have to.+ *+ * QuickJS parses the bundle in about fourteen seconds on a device. The host+ * caches that compile, but the cache does not exist until one launch has paid+ * for it — which is exactly the black screen after an install. Shipping the+ * bytecode makes the first launch as fast as the rest.+ *+ * Best effort by design: it needs a compiler for THIS machine (the NDK's targets+ * the device), and a build machine without one still produces a working app —+ * one that compiles the bundle on first run, as before. So a failure here is+ * logged and returns null rather than failing the build.+ *+ * @returns Path to the bytecode, or null if it could not be produced.+ */+async function precompileBundleBytecode(rootDir, genDir, quickjs) {+ const source = path.join(genDir, 'esengine_bundle.embedded.js');+ const out = path.join(genDir, BYTECODE_FILE);+ const exe = process.platform === 'win32' ? '.exe' : '';+ const tool = path.join(genDir, `mkbc${exe}`);++ // The very sources the host links, so the bytecode it writes is the bytecode+ // the host reads. Anything else is a format gamble.+ const qjsSources = ['quickjs.c', 'dtoa.c', 'libregexp.c', 'libunicode.c']+ .map((f) => path.join(quickjs, f));+ const missing = qjsSources.find((f) => !existsSync(f));+ if (missing) {+ logger.warn(`Bytecode precompile skipped: ${path.basename(missing)} not in ${quickjs}.`);+ return null;+ }++ const cc = process.platform === 'win32' ? 'gcc' : 'cc';+ try {+ await runCommand(cc, [+ '-std=c11', '-O2', '-w',+ '-I', quickjs,+ '-o', tool,+ path.join(rootDir, 'native', 'tools', 'mkbc.c'),+ ...qjsSources,+ ], { cwd: rootDir, silent: true });+ await runCommand(tool, [source, out], { cwd: rootDir, silent: true });+ } catch (err) {+ logger.warn('Bytecode precompile skipped (no host compiler?) — the app will '+ + `compile the bundle on first launch instead: ${err.message}`);+ return null;+ }+ return existsSync(out) ? out : null;+}+// Embed the host bootstrap (native/host/bootstrap.js) the same way. It is real// JavaScript — the bridge install and the default init/update — so it lives as a// .js file that can be linted and diffed, not as a C++ string literal.@@ -166,6 +221,8 @@ async function prepareGenerated(rootDir, buildDir, quickjs) {await generateSdkBundle(rootDir, genDir);logger.step('Embedding the host bootstrap (native/host/bootstrap.js)...');await generateBootstrap(rootDir, genDir);+ logger.step('Precompiling the bundle to bytecode (so the first launch need not)...');+ await precompileBundleBytecode(rootDir, genDir, quickjs);await stageEditorHeaders(rootDir, quickjs);return genDir;}diff --git a/build-tools/tasks/nativePackage.js b/build-tools/tasks/nativePackage.jsindex 0cc4088fc..8bd56c5fd 100644--- a/build-tools/tasks/nativePackage.js+++ b/build-tools/tasks/nativePackage.js@@ -15,7 +15,16 @@ import { runCommand } from '../utils/emscripten.js';import { requireSdk, requireNdk, buildTool, platformJar, ndkTool, ndkLibcxxShared, javaHome, jdkTool } from '../utils/android.js';import { readAppConfig, androidScreenOrientation, fillTemplate } from '../utils/nativeApp.js';+const HOST_LIBRARY = 'libestella_js_host.so';++/**+ * The shipped bytecode's filename. NOT `.bc`: that is LLVM bitcode's extension, and+ * aapt2 treats a `.bc` asset as RenderScript output — which silently drops the APK's+ * native-code declaration, so the package installs nowhere. Cost a long hunt; the+ * only symptom is INSTALL_FAILED_NO_MATCHING_ABIS on a device whose ABI matches.+ */+export const BYTECODE_FILE = 'esengine.native.qjsbc';const MANIFEST_TEMPLATE = path.join('native', 'android', 'host', 'AndroidManifest.xml.in');/** The APK is named after the app it contains, so packaging a second project does@@ -75,12 +84,16 @@ export async function packageNativeApk(options = {}) {await mkdir(libDir, { recursive: true });logger.step('Staging the native libraries...');- await copyFile(library, path.join(libDir, HOST_LIBRARY));await copyFile(ndkLibcxxShared(ndk), path.join(libDir, 'libc++_shared.so'));- // Dawn carries debug info an APK does not need — it dominates the payload.- await runCommand(ndkTool(ndk, 'llvm-strip'), [- '--strip-all', '-o', path.join(libDir, 'libwebgpu_dawn.so'), dawnLib,- ]);+ // Both carry debug info an APK does not need, and the host's is most of its+ // size. Shipping it unstripped also broke installing onto a clean device --+ // the extract step gave up on the payload -- which only reproduced on a first+ // install, since replacing an existing package never re-extracted it.+ for (const [src, dst] of [[library, HOST_LIBRARY], [dawnLib, 'libwebgpu_dawn.so']]) {+ await runCommand(ndkTool(ndk, 'llvm-strip'), [+ '--strip-all', '-o', path.join(libDir, dst), src,+ ]);+ }// assets/ is what the host's readAsset() sees, and what goes in is an editor// export — cooked assets + manifests + scenes + game.config.json. A whole-@@ -98,6 +111,19 @@ export async function packageNativeApk(options = {}) {await cp(content, path.join(staging, 'assets'), { recursive: true });logger.step(`Staging exported content from ${path.relative(rootDir, content)}...`);+ // The bundle's bytecode, if this machine could build it. Without it the first+ // launch spends ~14 s parsing the bundle before anything is drawn; with it the+ // host reads the compile straight off and the first launch matches the rest.+ // Absent is a valid state — the host falls back to compiling (see Runtime.cpp).+ const bytecode = path.join(rootDir, 'build-native', 'gen', BYTECODE_FILE);+ if (existsSync(bytecode)) {+ await cp(bytecode, path.join(staging, 'assets', BYTECODE_FILE));+ logger.step('Staging the precompiled SDK bytecode...');+ } else {+ logger.warn('No precompiled bytecode: the first launch will compile the bundle '+ + '(~14 s of black screen). Install a host C compiler to avoid it.');+ }+// The app the content asks to be: identity, version, and the orientation the// OS must lock the window to. The manifest is a template because all four vary// per project — a static one is why every example installed over the last.diff --git a/native/host/Runtime.cpp b/native/host/Runtime.cppindex a9f9d512c..f7869519f 100644--- a/native/host/Runtime.cpp+++ b/native/host/Runtime.cpp@@ -202,39 +202,70 @@ uint64_t hashBytes(const char* p, size_t n) {}/**- * Evaluate a global script, reusing a cached bytecode compile. QuickJS is an- * interpreter: parsing the ~700 KB SDK bundle costs ~8 s every launch. So compile- * once (JS_WriteObject the bytecode, tagged with the bundle hash), and on later- * launches JS_ReadObject it — skipping the parse. The same QuickJS build writes and- * reads it, so the format always matches; the hash guards against a stale bundle,- * and a bad read just recompiles. Cache file: [8-byte hash][bytecode].+ * Read a [8-byte hash][bytecode] blob, if its hash is the one we want. A blob that+ * does not match, or does not load, is simply not used — the caller falls through+ * to the next source and ultimately to compiling.+ */+bool tryBytecode(HostState& h, const u8* blob, size_t size, uint64_t want,+ const char* origin, JSValue& out) {+ if (size <= sizeof(uint64_t)) return false;+ uint64_t got = 0;+ std::memcpy(&got, blob, sizeof(got));+ if (got != want) return false; // built against a different bundle++ JSValue fn = JS_ReadObject(h.js, blob + sizeof(got), size - sizeof(got), JS_READ_OBJ_BYTECODE);+ if (JS_IsException(fn)) {+ JSValue e = JS_GetException(h.js);+ JS_FreeValue(h.js, e);+ return false;+ }+ ESHOST_LOGI("SDK bundle: loaded from %s", origin);+ out = JS_EvalFunction(h.js, fn);+ return true;+}++/**+ * Evaluate a global script, skipping the parse whenever bytecode for it exists.+ * QuickJS is an interpreter: parsing the ~700 KB SDK bundle costs ~14 s on a+ * device, and that parse is the black screen after an install. Three sources, in+ * order of how early they are available:+ *+ * 1. Bytecode built with the app (`assets/esengine.native.bc`), so the FIRST+ * launch is already fast. Present only if the build machine could produce it.+ * 2. Bytecode this device compiled on an earlier launch.+ * 3. The source, parsed — and cached on the way out, so step 2 works next time.+ *+ * Every source is tagged with a hash of the bundle it was built from, so a stale+ * one is skipped rather than trusted. Cache file: [8-byte hash][bytecode].*/JSValue evalCachedScript(HostState& h, const char* src, size_t srcLen, const char* filename,- const std::string& cachePath) {+ const std::string& cachePath, const char* shippedAsset) {const uint64_t want = hashBytes(src, srcLen);+ JSValue out;++ if (shippedAsset) {+ const std::vector<u8> blob = readAsset(h, shippedAsset);+ if (!blob.empty() && tryBytecode(h, blob.data(), blob.size(), want,+ "bytecode shipped with the app", out)) {+ return out;+ }+ }if (!cachePath.empty()) {FILE* f = fopen(cachePath.c_str(), "rb");if (f) {- uint64_t got = 0;- std::vector<u8> bc;- if (fread(&got, sizeof(got), 1, f) == 1 && got == want) {- fseek(f, 0, SEEK_END);- long sz = ftell(f);- fseek(f, (long)sizeof(got), SEEK_SET);- if (sz > (long)sizeof(got)) {- bc.resize((size_t)sz - sizeof(got));- if (fread(bc.data(), 1, bc.size(), f) != bc.size()) bc.clear();- }+ std::vector<u8> blob;+ fseek(f, 0, SEEK_END);+ const long sz = ftell(f);+ fseek(f, 0, SEEK_SET);+ if (sz > 0) {+ blob.resize((size_t)sz);+ if (fread(blob.data(), 1, blob.size(), f) != blob.size()) blob.clear();}fclose(f);- if (!bc.empty()) {- JSValue obj = JS_ReadObject(h.js, bc.data(), bc.size(), JS_READ_OBJ_BYTECODE);- if (!JS_IsException(obj)) {- ESHOST_LOGI("SDK bundle: loaded from bytecode cache");- return JS_EvalFunction(h.js, obj);- }- JSValue e = JS_GetException(h.js); JS_FreeValue(h.js, e); // stale/corrupt -> recompile+ if (!blob.empty() && tryBytecode(h, blob.data(), blob.size(), want,+ "bytecode cache", out)) {+ return out;}}}@@ -358,7 +389,8 @@ void initRuntime(HostState& h) {// Layer 1: the real SDK bundle, embedded — installs `ESEngine`.const double tCtx = nowMs();std::string bcPath = h.cacheDir.empty() ? std::string() : (h.cacheDir + "/esengine.native.bc");- JSValue br = evalCachedScript(h, kSdkBundleJS, strlen(kSdkBundleJS), "esengine.native.js", bcPath);+ JSValue br = evalCachedScript(h, kSdkBundleJS, strlen(kSdkBundleJS), "esengine.native.js",+ bcPath, "esengine.native.qjsbc");if (JS_IsException(br)) logJsError(h.js, "SDK bundle");JS_FreeValue(h.js, br);const double tBundle = nowMs();diff --git a/native/tools/mkbc.c b/native/tools/mkbc.cnew file mode 100644index 000000000..07e269406--- /dev/null+++ b/native/tools/mkbc.c@@ -0,0 +1,90 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file mkbc.c+ * @brief Compiles a script to QuickJS bytecode, in the cache format the host reads.+ *+ * @details QuickJS is an interpreter, so the host pays to parse the SDK bundle+ * the first time it ever runs — measured at ~14 s on a device, which is+ * the whole of the black screen a user sees after installing. The host+ * already caches the compile (see Runtime.cpp), but the cache does not+ * exist until that first launch has paid for it. Producing it at build+ * time instead means the first launch is as fast as every later one.+ *+ * This is a host tool: it runs on the machine doing the build, not on+ * the device. The bytecode is portable between them because both use the+ * same QuickJS — the build links the very sources the host does — and+ * the format depends on that, not on the CPU. Verified by building the+ * bytecode on one machine and loading it on a device of another+ * architecture.+ *+ * Output is exactly what the host's cache file holds:+ * [8-byte FNV-1a of the source][bytecode]. The hash is what lets the+ * host reject bytecode that does not match the bundle it was built+ * with, so this must hash the same bytes the host embeds — the caller+ * is responsible for handing over that exact form.+ */+#include "quickjs.h"++#include <stdint.h>+#include <stdio.h>+#include <stdlib.h>++/** Must match hashBytes() in native/host/Runtime.cpp. */+static uint64_t fnv1a(const char* p, size_t n) {+ uint64_t h = 1469598103934665603ull;+ for (size_t i = 0; i < n; i++) {+ h ^= (unsigned char)p[i];+ h *= 1099511628211ull;+ }+ return h;+}++int main(int argc, char** argv) {+ if (argc < 3) {+ fprintf(stderr, "usage: mkbc <input.js> <output.bc>\n");+ return 2;+ }++ FILE* in = fopen(argv[1], "rb");+ if (!in) { fprintf(stderr, "mkbc: cannot open %s\n", argv[1]); return 1; }+ fseek(in, 0, SEEK_END);+ long size = ftell(in);+ fseek(in, 0, SEEK_SET);+ if (size <= 0) { fprintf(stderr, "mkbc: %s is empty\n", argv[1]); fclose(in); return 1; }+ char* source = (char*)malloc((size_t)size + 1);+ if (!source || fread(source, 1, (size_t)size, in) != (size_t)size) {+ fprintf(stderr, "mkbc: cannot read %s\n", argv[1]);+ fclose(in);+ return 1;+ }+ source[size] = '\0';+ fclose(in);++ JSRuntime* runtime = JS_NewRuntime();+ JSContext* ctx = JS_NewContext(runtime);+ // The filename is part of what the bytecode carries, so it has to match what+ // the host would have compiled under or stack traces name a different file.+ JSValue fn = JS_Eval(ctx, source, (size_t)size, "esengine.native.js",+ JS_EVAL_TYPE_GLOBAL | JS_EVAL_FLAG_COMPILE_ONLY);+ if (JS_IsException(fn)) {+ JSValue err = JS_GetException(ctx);+ const char* msg = JS_ToCString(ctx, err);+ fprintf(stderr, "mkbc: compile failed: %s\n", msg ? msg : "(no message)");+ return 1;+ }++ size_t bcLen = 0;+ uint8_t* bc = JS_WriteObject(ctx, &bcLen, fn, JS_WRITE_OBJ_BYTECODE);+ if (!bc || bcLen == 0) { fprintf(stderr, "mkbc: JS_WriteObject failed\n"); return 1; }++ FILE* out = fopen(argv[2], "wb");+ if (!out) { fprintf(stderr, "mkbc: cannot write %s\n", argv[2]); return 1; }+ const uint64_t hash = fnv1a(source, (size_t)size);+ fwrite(&hash, sizeof(hash), 1, out);+ fwrite(bc, 1, bcLen, out);+ fclose(out);++ printf("mkbc: %ld source bytes -> %zu bytecode bytes\n", size, bcLen);+ return 0;+}