Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(native): pin Dawn + QuickJS, and build them from the recipe rather than a README

esengine10b4eaa18965
diff --git a/build-tools/cli.js b/build-tools/cli.js
index ecd310f9d..56f1d1419 100644
--- a/build-tools/cli.js
+++ b/build-tools/cli.js
@@ -161,9 +161,10 @@ program
.command('native')
.description('Build the native (embedded-Dawn) host for Android or iOS arm64')
.option('--target <target>', 'android or ios', 'android')
- .option('--dawn <dir>', 'Dawn source dir (or ESTELLA_DAWN_DIR); see native/README.md')
- .option('--dawn-build <dir>', 'Dawn arm64 build dir for this target (or ESTELLA_DAWN_BUILD)')
- .option('--quickjs <dir>', 'QuickJS-ng source dir (or ESTELLA_QUICKJS_DIR)')
+ .option('--fetch-deps', 'Check out Dawn + QuickJS-ng at the pinned commits, then stop', false)
+ .option('--dawn <dir>', 'Dawn source dir (default: the pinned checkout; or ESTELLA_DAWN_DIR)')
+ .option('--dawn-build <dir>', 'Dawn build dir for this target (default: <dawn>/out-<target>, built if absent)')
+ .option('--quickjs <dir>', 'QuickJS-ng source dir (default: the pinned checkout; or ESTELLA_QUICKJS_DIR)')
.option('--abi <abi>', 'Android ABI', 'arm64-v8a')
.option('--platform <platform>', 'Android platform', 'android-33')
.option('--ios-min <version>', 'iOS deployment target', '17.0')
@@ -177,15 +178,18 @@ program
.option('--no-template', 'Skip refreshing this machine\'s runtime template with the build')
.option('--template-only', 'Emit the runtime template from an existing build, without rebuilding', false)
.option('--template-out <dir>', 'Also write the distributable template archive here (for a release)')
+ .option('--template-index <dir>', 'Write native-templates.json describing the archives in this dir, then stop')
.option('-v, --verbose', 'Verbose output', false)
.action(async (options) => {
logger.setVerbose(options.verbose);
try {
await buildNative({
target: options.target,
+ fetchDeps: options.fetchDeps,
template: options.template,
templateOnly: options.templateOnly,
templateOut: options.templateOut,
+ templateIndex: options.templateIndex,
dawn: options.dawn,
dawnBuild: options.dawnBuild,
quickjs: options.quickjs,
diff --git a/build-tools/tasks/native.js b/build-tools/tasks/native.js
index 8717f2afd..a3cbd2497 100644
--- a/build-tools/tasks/native.js
+++ b/build-tools/tasks/native.js
@@ -15,7 +15,8 @@ import config from '../build.config.js';
import * as logger from '../utils/logger.js';
import { runCommand, getCpuCount, resolvePython } from '../utils/emscripten.js';
import { requireSdk, requireNdk, sdkCmake } from '../utils/android.js';
-import { emitNativeTemplate, readEngineVersion } from './nativeTemplateEmit.js';
+import { emitNativeTemplate, writeTemplateIndex, readEngineVersion } from './nativeTemplateEmit.js';
+import { fetchNativeDeps, pinnedDep, ensureDawnBuild } from './nativeDeps.js';
import {
BYTECODE_FILE, DEFAULT_ABI, findTemplate, iosTemplateSources, templateStoreDir,
} from '../utils/nativeTemplate.js';
@@ -232,16 +233,40 @@ async function prepareGenerated(rootDir, buildDir, quickjs) {
return genDir;
}
-function dawnPaths(options) {
- const dawnDir = options.dawn || process.env.ESTELLA_DAWN_DIR;
- const dawnBuild = options.dawnBuild || process.env.ESTELLA_DAWN_BUILD;
- if (!dawnDir || !dawnBuild) {
- throw new Error('Dawn not configured. Pass --dawn <src> --dawn-build <arm64 build>, or set '
- + 'ESTELLA_DAWN_DIR / ESTELLA_DAWN_BUILD. Build Dawn per native/README.md.');
+/**
+ * Dawn, for one target: where its source is and where its build is — building it
+ * if it is not built yet.
+ *
+ * Nothing has to be passed. The pinned checkout (`--fetch-deps`) is the default
+ * source, its `out-<target>` the default build, and a missing build is produced
+ * rather than reported — so `cli native --target ios` works from a clean tree
+ * once the dependencies are fetched, and CI runs the same two commands a
+ * contributor does.
+ */
+async function dawnPaths(options, target, toolchain = {}) {
+ const dawnDir = options.dawn || process.env.ESTELLA_DAWN_DIR || pinnedDep('dawn');
+ if (!dawnDir) {
+ throw new Error('Dawn not configured. Run `node build-tools/cli.js native --fetch-deps` '
+ + '(it checks out the pinned commit), or pass --dawn <src> / set ESTELLA_DAWN_DIR.');
}
+ const dawnBuild = await ensureDawnBuild({
+ dawn: dawnDir,
+ target,
+ buildDir: options.dawnBuild || process.env.ESTELLA_DAWN_BUILD,
+ abi: options.abi,
+ androidPlatform: options.platform,
+ iosMin: options.iosMin,
+ ...toolchain,
+ });
return { dawnDir, dawnBuild };
}
+/** The QuickJS source the host is built against. Same rule as Dawn: the pinned
+ * checkout unless something says otherwise. */
+function quickjsDir(options) {
+ return options.quickjs || process.env.ESTELLA_QUICKJS_DIR || pinnedDep('quickjs');
+}
+
// CMake parses backslashes in -D string values as escapes (F:\estella -> \e), so
// hand it forward-slash paths (valid on Windows too).
const fwd = (p) => p.replace(/\\/g, '/');
@@ -253,17 +278,16 @@ async function buildAndroidHost(options) {
const sdk = requireSdk();
const ndk = requireNdk(sdk);
const toolchain = path.join(ndk, 'build', 'cmake', 'android.toolchain.cmake');
-
- const { dawnDir, dawnBuild } = dawnPaths(options);
-
const { cmake, ninja } = sdkCmake(sdk);
+
+ const { dawnDir, dawnBuild } = await dawnPaths(options, 'android', { ndk, cmake, ninja });
const buildDir = path.join(rootDir, 'build-native');
await mkdir(buildDir, { recursive: true });
// JS host (opt-in): pass --quickjs <dir> (or ESTELLA_QUICKJS_DIR) to also build
// the QuickJS host — a game script driving the engine through the generated
// bindings + the real SDK ptrAccessors.
- const quickjs = options.quickjs || process.env.ESTELLA_QUICKJS_DIR;
+ const quickjs = quickjsDir(options);
const genDir = quickjs ? await prepareGenerated(rootDir, buildDir, quickjs) : null;
logger.step(`Configuring native host (${abi}, ${platform})...`);
@@ -404,20 +428,23 @@ async function writeIosAppIdentity(rootDir, contentDir) {
async function buildIosHost(options) {
if (process.platform !== 'darwin') throw new Error('The iOS host builds on macOS only.');
const rootDir = config.paths.root;
- const { dawnDir, dawnBuild } = dawnPaths(options);
const slice = options.simulator ? IOS_SLICES.simulator : IOS_SLICES.device;
// Unlike Android there is no pure-C++ reference host for iOS: the app IS the
// JS host, so QuickJS is required rather than opt-in.
- const quickjs = options.quickjs || process.env.ESTELLA_QUICKJS_DIR;
+ const quickjs = quickjsDir(options);
if (!quickjs) {
- throw new Error('The iOS host needs QuickJS: pass --quickjs <dir> (or set ESTELLA_QUICKJS_DIR).');
+ throw new Error('The iOS host needs QuickJS: run `node build-tools/cli.js native --fetch-deps`, '
+ + 'or pass --quickjs <dir> (or set ESTELLA_QUICKJS_DIR).');
}
const developerDir = await iosDeveloperDir();
const env = developerDir ? { DEVELOPER_DIR: developerDir } : undefined;
if (developerDir) logger.info(`Using DEVELOPER_DIR=${developerDir} (the active one has no iPhoneOS SDK)`);
+ const { dawnDir, dawnBuild } = await dawnPaths(
+ options, options.simulator ? 'ios-sim' : 'ios', { env });
+
const buildDir = path.join(rootDir, slice.dir);
await mkdir(buildDir, { recursive: true });
const genDir = await prepareGenerated(rootDir, buildDir, quickjs);
@@ -549,6 +576,11 @@ async function emitFromExistingBuild(target, options) {
}
export async function buildNative(options = {}) {
+ if (options.fetchDeps) return fetchNativeDeps(options);
+ if (options.templateIndex) {
+ return writeTemplateIndex(path.isAbsolute(options.templateIndex)
+ ? options.templateIndex : path.join(config.paths.root, options.templateIndex));
+ }
const target = (options.target || 'android').toLowerCase();
if (target !== 'ios' && target !== 'android') {
throw new Error(`Unknown --target ${target} (expected android or ios).`);
diff --git a/build-tools/tasks/nativeDeps.js b/build-tools/tasks/nativeDeps.js
new file mode 100644
index 000000000..96e837173
--- /dev/null
+++ b/build-tools/tasks/nativeDeps.js
@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+//
+// Dawn and QuickJS, checked out at the commits this release builds against.
+//
+// A runtime template is a binary other people install, so it has to be
+// reproducible: `git clone --depth 1` of a branch gives whatever landed today,
+// and Dawn's webgpu.h is a moving target. The pins live in
+// toolchain.manifest.json; this fetches them, and the build finds them here
+// without a flag.
+
+import path from 'path';
+import { existsSync, readFileSync } from 'fs';
+import { mkdir } from 'fs/promises';
+import config from '../build.config.js';
+import * as logger from '../utils/logger.js';
+import { runCommand, resolvePython } from '../utils/emscripten.js';
+
+/** Where a fetched checkout lands. Inside the repo's build dir (gitignored), so a
+ * CI cache can hold it and `rm -rf build` throws it away like everything else. */
+export function nativeDepsDir(rootDir = config.paths.root) {
+ return process.env.ESTELLA_NATIVE_DEPS || path.join(rootDir, 'build', 'native-deps');
+}
+
+export function nativePins(rootDir = config.paths.root) {
+ const manifest = JSON.parse(readFileSync(path.join(rootDir, 'toolchain.manifest.json'), 'utf8'));
+ if (!manifest.native?.dawn?.commit || !manifest.native?.quickjs?.commit) {
+ throw new Error('toolchain.manifest.json has no native.dawn / native.quickjs pin.');
+ }
+ return manifest.native;
+}
+
+/**
+ * Check `repo` out at `commit` in `dir`, fetching only that commit.
+ *
+ * `git fetch --depth 1 <sha>` rather than a clone: a full clone of Dawn is
+ * gigabytes of history nobody here reads, and naming the commit is what makes the
+ * result the same for everyone.
+ */
+async function fetchAt(dir, repo, commit, label) {
+ if (!existsSync(path.join(dir, '.git'))) {
+ await mkdir(dir, { recursive: true });
+ await runCommand('git', ['init', '-q'], { cwd: dir });
+ await runCommand('git', ['remote', 'add', 'origin', repo], { cwd: dir });
+ }
+ const head = await runCommand('git', ['rev-parse', 'HEAD'], { cwd: dir, silent: true }).catch(() => null);
+ if (head?.stdout?.trim() === commit) {
+ logger.info(`${label} already at ${commit.slice(0, 12)}`);
+ return dir;
+ }
+ logger.step(`Fetching ${label} at ${commit.slice(0, 12)}...`);
+ await runCommand('git', ['fetch', '--depth', '1', 'origin', commit], { cwd: dir });
+ await runCommand('git', ['checkout', '-q', 'FETCH_HEAD'], { cwd: dir });
+ return dir;
+}
+
+/**
+ * Fetch both dependencies at their pinned commits.
+ *
+ * @returns {Promise<{dawn: string, quickjs: string}>} the two checkouts.
+ */
+export async function fetchNativeDeps(options = {}) {
+ const rootDir = options.root || config.paths.root;
+ const pins = nativePins(rootDir);
+ const deps = options.depsDir || nativeDepsDir(rootDir);
+
+ const dawn = await fetchAt(path.join(deps, 'dawn'), pins.dawn.repo, pins.dawn.commit, 'Dawn');
+ // Dawn's own dependencies are fetched by a script, not by submodules — which
+ // is the whole reason tracking Dawn as one would save nobody anything.
+ logger.step('Fetching Dawn\'s dependencies (shallow)...');
+ await runCommand(await resolvePython(), [
+ path.join(dawn, 'tools', 'fetch_dawn_dependencies.py'), '--shallow', '--directory', dawn,
+ ], { cwd: dawn });
+
+ const quickjs = await fetchAt(path.join(deps, 'quickjs'), pins.quickjs.repo, pins.quickjs.commit, 'QuickJS-ng');
+
+ logger.success(`Native dependencies ready in ${path.relative(rootDir, deps) || deps}`);
+ logger.info('Build with: node build-tools/cli.js native (Android)');
+ logger.info(' node build-tools/cli.js native --target ios');
+ return { dawn, quickjs };
+}
+
+/** The pinned checkout of a dependency, if it is there — what the build falls back
+ * to when no --dawn / --quickjs is passed. */
+export function pinnedDep(name, rootDir = config.paths.root) {
+ const dir = path.join(nativeDepsDir(rootDir), name);
+ return existsSync(dir) ? dir : null;
+}
+
+/**
+ * The Dawn build for a target, and how to produce it.
+ *
+ * The recipe used to live in native/README.md as a block to paste. It is here
+ * because CI has to run exactly what a contributor runs — a second copy in a
+ * workflow file is a second thing to keep right — and because a build that
+ * fetches its own dependencies should be able to build them too.
+ */
+export const DAWN_TARGETS = {
+ android: { out: 'out-android', shared: true },
+ ios: { out: 'out-ios', sysroot: 'iphoneos' },
+ 'ios-sim': { out: 'out-ios-sim', sysroot: 'iphonesimulator' },
+};
+
+/** Where a target's monolithic Dawn library lands, once built. */
+export function dawnLibrary(dawnBuild, target) {
+ const name = DAWN_TARGETS[target].shared ? 'libwebgpu_dawn.so' : 'libwebgpu_dawn.a';
+ return path.join(dawnBuild, 'src', 'dawn', 'native', name);
+}
+
+/**
+ * Build Dawn for @p target if it is not built already.
+ *
+ * @param {object} options `{ dawn, target, buildDir, ndk, cmake, ninja, iosMin, env }`
+ * @returns {Promise<string>} the build directory.
+ */
+export async function ensureDawnBuild(options) {
+ const target = DAWN_TARGETS[options.target];
+ if (!target) throw new Error(`Unknown Dawn target ${options.target}.`);
+ const buildDir = options.buildDir || path.join(options.dawn, target.out);
+ if (existsSync(dawnLibrary(buildDir, options.target))) return buildDir;
+
+ const cmake = options.cmake || 'cmake';
+ const shared = [
+ '-S', options.dawn, '-B', buildDir, '-G', 'Ninja',
+ '-DCMAKE_BUILD_TYPE=Release',
+ '-DDAWN_ENABLE_D3D12=OFF', '-DDAWN_ENABLE_NULL=OFF',
+ '-DDAWN_ENABLE_OPENGLES=OFF', '-DDAWN_ENABLE_DESKTOP_GL=OFF',
+ '-DDAWN_BUILD_SAMPLES=OFF', '-DDAWN_BUILD_TESTS=OFF', '-DTINT_BUILD_TESTS=OFF',
+ // Cross-compiling Dawn otherwise wants a host protoc.
+ '-DDAWN_BUILD_PROTOBUF=OFF', '-DTINT_BUILD_IR_BINARY=OFF',
+ // Its dependencies are already fetched (fetchNativeDeps); letting CMake do
+ // it again would undo the pin.
+ '-DDAWN_FETCH_DEPENDENCIES=OFF',
+ ...(options.ninja ? [`-DCMAKE_MAKE_PROGRAM=${options.ninja}`] : []),
+ ];
+ const perTarget = options.target === 'android'
+ ? [
+ `-DCMAKE_TOOLCHAIN_FILE=${path.join(options.ndk, 'build', 'cmake', 'android.toolchain.cmake')}`,
+ `-DANDROID_ABI=${options.abi || 'arm64-v8a'}`,
+ `-DANDROID_PLATFORM=${options.androidPlatform || 'android-33'}`,
+ '-DANDROID_STL=c++_shared',
+ '-DDAWN_ENABLE_VULKAN=ON', '-DDAWN_ENABLE_METAL=OFF',
+ // Shared on Android (the APK ships the .so); static on iOS (an app
+ // embeds it).
+ '-DDAWN_BUILD_MONOLITHIC_LIBRARY=SHARED',
+ ]
+ : [
+ '-DCMAKE_SYSTEM_NAME=iOS', '-DCMAKE_OSX_ARCHITECTURES=arm64',
+ `-DCMAKE_OSX_DEPLOYMENT_TARGET=${options.iosMin || '17.0'}`,
+ `-DCMAKE_OSX_SYSROOT=${target.sysroot}`,
+ '-DDAWN_ENABLE_METAL=ON', '-DDAWN_ENABLE_VULKAN=OFF',
+ '-DDAWN_BUILD_MONOLITHIC_LIBRARY=STATIC',
+ ];
+
+ logger.step(`Building Dawn for ${options.target} (once per pin; this takes a few minutes)...`);
+ await runCommand(cmake, [...shared, ...perTarget], { cwd: options.dawn, env: options.env });
+ await runCommand(cmake, ['--build', buildDir, '--target', 'webgpu_dawn'], { cwd: options.dawn, env: options.env });
+ logger.success(`Dawn: ${dawnLibrary(buildDir, options.target)}`);
+ return buildDir;
+}
diff --git a/build-tools/utils/android.js b/build-tools/utils/android.js
index fe1b0b8a6..a8f87342a 100644
--- a/build-tools/utils/android.js
+++ b/build-tools/utils/android.js
@@ -31,8 +31,16 @@ export function requireSdk() {
}
export function requireNdk(sdk) {
+ // An NDK installed outside the SDK (a CI action, a manual unzip) exports one
+ // of these; Android Studio's lives under the SDK.
+ const explicit = [process.env.ANDROID_NDK_HOME, process.env.ANDROID_NDK_ROOT].find(
+ (dir) => dir && existsSync(dir));
+ if (explicit) return explicit;
const ndk = newest(path.join(sdk, 'ndk'));
- if (!ndk) throw new Error(`No NDK under ${sdk}/ndk. Install one via Android Studio's SDK Manager.`);
+ if (!ndk) {
+ throw new Error(`No NDK under ${sdk}/ndk and no ANDROID_NDK_HOME. `
+ + "Install one via Android Studio's SDK Manager.");
+ }
return ndk;
}
diff --git a/toolchain.manifest.json b/toolchain.manifest.json
index 6e3e8ee22..f2553e173 100644
--- a/toolchain.manifest.json
+++ b/toolchain.manifest.json
@@ -26,6 +26,24 @@
]
}
},
+ "native": {
+ "$comment": [
+ "The two projects the native host links that are too large to vendor. PINNED, not",
+ "tracked: `--depth 1` of a moving branch makes a runtime template unreproducible, and",
+ "webgpu.h is a moving target. Not submodules either — Dawn fetches its own dependencies",
+ "with a script rather than submodules, so tracking it that way saves nobody the fetch",
+ "while putting a multi-GB clone in front of everyone who only builds for the web.",
+ "`node build-tools/cli.js native --fetch-deps` checks both out at these commits."
+ ],
+ "dawn": {
+ "repo": "https://dawn.googlesource.com/dawn",
+ "commit": "296a00d20c4324d0a0363096ab23da4bf4f26d35"
+ },
+ "quickjs": {
+ "repo": "https://github.com/quickjs-ng/quickjs",
+ "commit": "3d3cf3d86b66551403745a5b06acbed2b0f31dfd"
+ }
+ },
"cmake": {
"version": "3.31.7",
"platforms": {