A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(native): pin Dawn + QuickJS, and build them from the recipe rather than a README
esengine10b4eaa18965
diff --git a/build-tools/cli.js b/build-tools/cli.jsindex ecd310f9d..56f1d1419 100644--- a/build-tools/cli.js+++ b/build-tools/cli.js@@ -161,9 +161,10 @@ program.command('native').description('Build the native (embedded-Dawn) host for Android or iOS arm64').option('--target <target>', 'android or ios', 'android')- .option('--dawn <dir>', 'Dawn source dir (or ESTELLA_DAWN_DIR); see native/README.md')- .option('--dawn-build <dir>', 'Dawn arm64 build dir for this target (or ESTELLA_DAWN_BUILD)')- .option('--quickjs <dir>', 'QuickJS-ng source dir (or ESTELLA_QUICKJS_DIR)')+ .option('--fetch-deps', 'Check out Dawn + QuickJS-ng at the pinned commits, then stop', false)+ .option('--dawn <dir>', 'Dawn source dir (default: the pinned checkout; or ESTELLA_DAWN_DIR)')+ .option('--dawn-build <dir>', 'Dawn build dir for this target (default: <dawn>/out-<target>, built if absent)')+ .option('--quickjs <dir>', 'QuickJS-ng source dir (default: the pinned checkout; or ESTELLA_QUICKJS_DIR)').option('--abi <abi>', 'Android ABI', 'arm64-v8a').option('--platform <platform>', 'Android platform', 'android-33').option('--ios-min <version>', 'iOS deployment target', '17.0')@@ -177,15 +178,18 @@ program.option('--no-template', 'Skip refreshing this machine\'s runtime template with the build').option('--template-only', 'Emit the runtime template from an existing build, without rebuilding', false).option('--template-out <dir>', 'Also write the distributable template archive here (for a release)')+ .option('--template-index <dir>', 'Write native-templates.json describing the archives in this dir, then stop').option('-v, --verbose', 'Verbose output', false).action(async (options) => {logger.setVerbose(options.verbose);try {await buildNative({target: options.target,+ fetchDeps: options.fetchDeps,template: options.template,templateOnly: options.templateOnly,templateOut: options.templateOut,+ templateIndex: options.templateIndex,dawn: options.dawn,dawnBuild: options.dawnBuild,quickjs: options.quickjs,diff --git a/build-tools/tasks/native.js b/build-tools/tasks/native.jsindex 8717f2afd..a3cbd2497 100644--- a/build-tools/tasks/native.js+++ b/build-tools/tasks/native.js@@ -15,7 +15,8 @@ import config from '../build.config.js';import * as logger from '../utils/logger.js';import { runCommand, getCpuCount, resolvePython } from '../utils/emscripten.js';import { requireSdk, requireNdk, sdkCmake } from '../utils/android.js';-import { emitNativeTemplate, readEngineVersion } from './nativeTemplateEmit.js';+import { emitNativeTemplate, writeTemplateIndex, readEngineVersion } from './nativeTemplateEmit.js';+import { fetchNativeDeps, pinnedDep, ensureDawnBuild } from './nativeDeps.js';import {BYTECODE_FILE, DEFAULT_ABI, findTemplate, iosTemplateSources, templateStoreDir,} from '../utils/nativeTemplate.js';@@ -232,16 +233,40 @@ async function prepareGenerated(rootDir, buildDir, quickjs) {return genDir;}-function dawnPaths(options) {- const dawnDir = options.dawn || process.env.ESTELLA_DAWN_DIR;- const dawnBuild = options.dawnBuild || process.env.ESTELLA_DAWN_BUILD;- if (!dawnDir || !dawnBuild) {- throw new Error('Dawn not configured. Pass --dawn <src> --dawn-build <arm64 build>, or set '- + 'ESTELLA_DAWN_DIR / ESTELLA_DAWN_BUILD. Build Dawn per native/README.md.');+/**+ * Dawn, for one target: where its source is and where its build is — building it+ * if it is not built yet.+ *+ * Nothing has to be passed. The pinned checkout (`--fetch-deps`) is the default+ * source, its `out-<target>` the default build, and a missing build is produced+ * rather than reported — so `cli native --target ios` works from a clean tree+ * once the dependencies are fetched, and CI runs the same two commands a+ * contributor does.+ */+async function dawnPaths(options, target, toolchain = {}) {+ const dawnDir = options.dawn || process.env.ESTELLA_DAWN_DIR || pinnedDep('dawn');+ if (!dawnDir) {+ throw new Error('Dawn not configured. Run `node build-tools/cli.js native --fetch-deps` '+ + '(it checks out the pinned commit), or pass --dawn <src> / set ESTELLA_DAWN_DIR.');}+ const dawnBuild = await ensureDawnBuild({+ dawn: dawnDir,+ target,+ buildDir: options.dawnBuild || process.env.ESTELLA_DAWN_BUILD,+ abi: options.abi,+ androidPlatform: options.platform,+ iosMin: options.iosMin,+ ...toolchain,+ });return { dawnDir, dawnBuild };}+/** The QuickJS source the host is built against. Same rule as Dawn: the pinned+ * checkout unless something says otherwise. */+function quickjsDir(options) {+ return options.quickjs || process.env.ESTELLA_QUICKJS_DIR || pinnedDep('quickjs');+}+// CMake parses backslashes in -D string values as escapes (F:\estella -> \e), so// hand it forward-slash paths (valid on Windows too).const fwd = (p) => p.replace(/\\/g, '/');@@ -253,17 +278,16 @@ async function buildAndroidHost(options) {const sdk = requireSdk();const ndk = requireNdk(sdk);const toolchain = path.join(ndk, 'build', 'cmake', 'android.toolchain.cmake');-- const { dawnDir, dawnBuild } = dawnPaths(options);-const { cmake, ninja } = sdkCmake(sdk);++ const { dawnDir, dawnBuild } = await dawnPaths(options, 'android', { ndk, cmake, ninja });const buildDir = path.join(rootDir, 'build-native');await mkdir(buildDir, { recursive: true });// JS host (opt-in): pass --quickjs <dir> (or ESTELLA_QUICKJS_DIR) to also build// the QuickJS host — a game script driving the engine through the generated// bindings + the real SDK ptrAccessors.- const quickjs = options.quickjs || process.env.ESTELLA_QUICKJS_DIR;+ const quickjs = quickjsDir(options);const genDir = quickjs ? await prepareGenerated(rootDir, buildDir, quickjs) : null;logger.step(`Configuring native host (${abi}, ${platform})...`);@@ -404,20 +428,23 @@ async function writeIosAppIdentity(rootDir, contentDir) {async function buildIosHost(options) {if (process.platform !== 'darwin') throw new Error('The iOS host builds on macOS only.');const rootDir = config.paths.root;- const { dawnDir, dawnBuild } = dawnPaths(options);const slice = options.simulator ? IOS_SLICES.simulator : IOS_SLICES.device;// Unlike Android there is no pure-C++ reference host for iOS: the app IS the// JS host, so QuickJS is required rather than opt-in.- const quickjs = options.quickjs || process.env.ESTELLA_QUICKJS_DIR;+ const quickjs = quickjsDir(options);if (!quickjs) {- throw new Error('The iOS host needs QuickJS: pass --quickjs <dir> (or set ESTELLA_QUICKJS_DIR).');+ throw new Error('The iOS host needs QuickJS: run `node build-tools/cli.js native --fetch-deps`, '+ + 'or pass --quickjs <dir> (or set ESTELLA_QUICKJS_DIR).');}const developerDir = await iosDeveloperDir();const env = developerDir ? { DEVELOPER_DIR: developerDir } : undefined;if (developerDir) logger.info(`Using DEVELOPER_DIR=${developerDir} (the active one has no iPhoneOS SDK)`);+ const { dawnDir, dawnBuild } = await dawnPaths(+ options, options.simulator ? 'ios-sim' : 'ios', { env });+const buildDir = path.join(rootDir, slice.dir);await mkdir(buildDir, { recursive: true });const genDir = await prepareGenerated(rootDir, buildDir, quickjs);@@ -549,6 +576,11 @@ async function emitFromExistingBuild(target, options) {}export async function buildNative(options = {}) {+ if (options.fetchDeps) return fetchNativeDeps(options);+ if (options.templateIndex) {+ return writeTemplateIndex(path.isAbsolute(options.templateIndex)+ ? options.templateIndex : path.join(config.paths.root, options.templateIndex));+ }const target = (options.target || 'android').toLowerCase();if (target !== 'ios' && target !== 'android') {throw new Error(`Unknown --target ${target} (expected android or ios).`);diff --git a/build-tools/tasks/nativeDeps.js b/build-tools/tasks/nativeDeps.jsnew file mode 100644index 000000000..96e837173--- /dev/null+++ b/build-tools/tasks/nativeDeps.js@@ -0,0 +1,160 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+//+// Dawn and QuickJS, checked out at the commits this release builds against.+//+// A runtime template is a binary other people install, so it has to be+// reproducible: `git clone --depth 1` of a branch gives whatever landed today,+// and Dawn's webgpu.h is a moving target. The pins live in+// toolchain.manifest.json; this fetches them, and the build finds them here+// without a flag.++import path from 'path';+import { existsSync, readFileSync } from 'fs';+import { mkdir } from 'fs/promises';+import config from '../build.config.js';+import * as logger from '../utils/logger.js';+import { runCommand, resolvePython } from '../utils/emscripten.js';++/** Where a fetched checkout lands. Inside the repo's build dir (gitignored), so a+ * CI cache can hold it and `rm -rf build` throws it away like everything else. */+export function nativeDepsDir(rootDir = config.paths.root) {+ return process.env.ESTELLA_NATIVE_DEPS || path.join(rootDir, 'build', 'native-deps');+}++export function nativePins(rootDir = config.paths.root) {+ const manifest = JSON.parse(readFileSync(path.join(rootDir, 'toolchain.manifest.json'), 'utf8'));+ if (!manifest.native?.dawn?.commit || !manifest.native?.quickjs?.commit) {+ throw new Error('toolchain.manifest.json has no native.dawn / native.quickjs pin.');+ }+ return manifest.native;+}++/**+ * Check `repo` out at `commit` in `dir`, fetching only that commit.+ *+ * `git fetch --depth 1 <sha>` rather than a clone: a full clone of Dawn is+ * gigabytes of history nobody here reads, and naming the commit is what makes the+ * result the same for everyone.+ */+async function fetchAt(dir, repo, commit, label) {+ if (!existsSync(path.join(dir, '.git'))) {+ await mkdir(dir, { recursive: true });+ await runCommand('git', ['init', '-q'], { cwd: dir });+ await runCommand('git', ['remote', 'add', 'origin', repo], { cwd: dir });+ }+ const head = await runCommand('git', ['rev-parse', 'HEAD'], { cwd: dir, silent: true }).catch(() => null);+ if (head?.stdout?.trim() === commit) {+ logger.info(`${label} already at ${commit.slice(0, 12)}`);+ return dir;+ }+ logger.step(`Fetching ${label} at ${commit.slice(0, 12)}...`);+ await runCommand('git', ['fetch', '--depth', '1', 'origin', commit], { cwd: dir });+ await runCommand('git', ['checkout', '-q', 'FETCH_HEAD'], { cwd: dir });+ return dir;+}++/**+ * Fetch both dependencies at their pinned commits.+ *+ * @returns {Promise<{dawn: string, quickjs: string}>} the two checkouts.+ */+export async function fetchNativeDeps(options = {}) {+ const rootDir = options.root || config.paths.root;+ const pins = nativePins(rootDir);+ const deps = options.depsDir || nativeDepsDir(rootDir);++ const dawn = await fetchAt(path.join(deps, 'dawn'), pins.dawn.repo, pins.dawn.commit, 'Dawn');+ // Dawn's own dependencies are fetched by a script, not by submodules — which+ // is the whole reason tracking Dawn as one would save nobody anything.+ logger.step('Fetching Dawn\'s dependencies (shallow)...');+ await runCommand(await resolvePython(), [+ path.join(dawn, 'tools', 'fetch_dawn_dependencies.py'), '--shallow', '--directory', dawn,+ ], { cwd: dawn });++ const quickjs = await fetchAt(path.join(deps, 'quickjs'), pins.quickjs.repo, pins.quickjs.commit, 'QuickJS-ng');++ logger.success(`Native dependencies ready in ${path.relative(rootDir, deps) || deps}`);+ logger.info('Build with: node build-tools/cli.js native (Android)');+ logger.info(' node build-tools/cli.js native --target ios');+ return { dawn, quickjs };+}++/** The pinned checkout of a dependency, if it is there — what the build falls back+ * to when no --dawn / --quickjs is passed. */+export function pinnedDep(name, rootDir = config.paths.root) {+ const dir = path.join(nativeDepsDir(rootDir), name);+ return existsSync(dir) ? dir : null;+}++/**+ * The Dawn build for a target, and how to produce it.+ *+ * The recipe used to live in native/README.md as a block to paste. It is here+ * because CI has to run exactly what a contributor runs — a second copy in a+ * workflow file is a second thing to keep right — and because a build that+ * fetches its own dependencies should be able to build them too.+ */+export const DAWN_TARGETS = {+ android: { out: 'out-android', shared: true },+ ios: { out: 'out-ios', sysroot: 'iphoneos' },+ 'ios-sim': { out: 'out-ios-sim', sysroot: 'iphonesimulator' },+};++/** Where a target's monolithic Dawn library lands, once built. */+export function dawnLibrary(dawnBuild, target) {+ const name = DAWN_TARGETS[target].shared ? 'libwebgpu_dawn.so' : 'libwebgpu_dawn.a';+ return path.join(dawnBuild, 'src', 'dawn', 'native', name);+}++/**+ * Build Dawn for @p target if it is not built already.+ *+ * @param {object} options `{ dawn, target, buildDir, ndk, cmake, ninja, iosMin, env }`+ * @returns {Promise<string>} the build directory.+ */+export async function ensureDawnBuild(options) {+ const target = DAWN_TARGETS[options.target];+ if (!target) throw new Error(`Unknown Dawn target ${options.target}.`);+ const buildDir = options.buildDir || path.join(options.dawn, target.out);+ if (existsSync(dawnLibrary(buildDir, options.target))) return buildDir;++ const cmake = options.cmake || 'cmake';+ const shared = [+ '-S', options.dawn, '-B', buildDir, '-G', 'Ninja',+ '-DCMAKE_BUILD_TYPE=Release',+ '-DDAWN_ENABLE_D3D12=OFF', '-DDAWN_ENABLE_NULL=OFF',+ '-DDAWN_ENABLE_OPENGLES=OFF', '-DDAWN_ENABLE_DESKTOP_GL=OFF',+ '-DDAWN_BUILD_SAMPLES=OFF', '-DDAWN_BUILD_TESTS=OFF', '-DTINT_BUILD_TESTS=OFF',+ // Cross-compiling Dawn otherwise wants a host protoc.+ '-DDAWN_BUILD_PROTOBUF=OFF', '-DTINT_BUILD_IR_BINARY=OFF',+ // Its dependencies are already fetched (fetchNativeDeps); letting CMake do+ // it again would undo the pin.+ '-DDAWN_FETCH_DEPENDENCIES=OFF',+ ...(options.ninja ? [`-DCMAKE_MAKE_PROGRAM=${options.ninja}`] : []),+ ];+ const perTarget = options.target === 'android'+ ? [+ `-DCMAKE_TOOLCHAIN_FILE=${path.join(options.ndk, 'build', 'cmake', 'android.toolchain.cmake')}`,+ `-DANDROID_ABI=${options.abi || 'arm64-v8a'}`,+ `-DANDROID_PLATFORM=${options.androidPlatform || 'android-33'}`,+ '-DANDROID_STL=c++_shared',+ '-DDAWN_ENABLE_VULKAN=ON', '-DDAWN_ENABLE_METAL=OFF',+ // Shared on Android (the APK ships the .so); static on iOS (an app+ // embeds it).+ '-DDAWN_BUILD_MONOLITHIC_LIBRARY=SHARED',+ ]+ : [+ '-DCMAKE_SYSTEM_NAME=iOS', '-DCMAKE_OSX_ARCHITECTURES=arm64',+ `-DCMAKE_OSX_DEPLOYMENT_TARGET=${options.iosMin || '17.0'}`,+ `-DCMAKE_OSX_SYSROOT=${target.sysroot}`,+ '-DDAWN_ENABLE_METAL=ON', '-DDAWN_ENABLE_VULKAN=OFF',+ '-DDAWN_BUILD_MONOLITHIC_LIBRARY=STATIC',+ ];++ logger.step(`Building Dawn for ${options.target} (once per pin; this takes a few minutes)...`);+ await runCommand(cmake, [...shared, ...perTarget], { cwd: options.dawn, env: options.env });+ await runCommand(cmake, ['--build', buildDir, '--target', 'webgpu_dawn'], { cwd: options.dawn, env: options.env });+ logger.success(`Dawn: ${dawnLibrary(buildDir, options.target)}`);+ return buildDir;+}diff --git a/build-tools/utils/android.js b/build-tools/utils/android.jsindex fe1b0b8a6..a8f87342a 100644--- a/build-tools/utils/android.js+++ b/build-tools/utils/android.js@@ -31,8 +31,16 @@ export function requireSdk() {}export function requireNdk(sdk) {+ // An NDK installed outside the SDK (a CI action, a manual unzip) exports one+ // of these; Android Studio's lives under the SDK.+ const explicit = [process.env.ANDROID_NDK_HOME, process.env.ANDROID_NDK_ROOT].find(+ (dir) => dir && existsSync(dir));+ if (explicit) return explicit;const ndk = newest(path.join(sdk, 'ndk'));- if (!ndk) throw new Error(`No NDK under ${sdk}/ndk. Install one via Android Studio's SDK Manager.`);+ if (!ndk) {+ throw new Error(`No NDK under ${sdk}/ndk and no ANDROID_NDK_HOME. `+ + "Install one via Android Studio's SDK Manager.");+ }return ndk;}diff --git a/toolchain.manifest.json b/toolchain.manifest.jsonindex 6e3e8ee22..f2553e173 100644--- a/toolchain.manifest.json+++ b/toolchain.manifest.json@@ -26,6 +26,24 @@]}},+ "native": {+ "$comment": [+ "The two projects the native host links that are too large to vendor. PINNED, not",+ "tracked: `--depth 1` of a moving branch makes a runtime template unreproducible, and",+ "webgpu.h is a moving target. Not submodules either — Dawn fetches its own dependencies",+ "with a script rather than submodules, so tracking it that way saves nobody the fetch",+ "while putting a multi-GB clone in front of everyone who only builds for the web.",+ "`node build-tools/cli.js native --fetch-deps` checks both out at these commits."+ ],+ "dawn": {+ "repo": "https://dawn.googlesource.com/dawn",+ "commit": "296a00d20c4324d0a0363096ab23da4bf4f26d35"+ },+ "quickjs": {+ "repo": "https://github.com/quickjs-ng/quickjs",+ "commit": "3d3cf3d86b66551403745a5b06acbed2b0f31dfd"+ }+ },"cmake": {"version": "3.31.7","platforms": {