Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

14.0 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
import path from 'path';
import { mkdir, cp, rm, stat, readFile, rename } from 'fs/promises';
import { existsSync } from 'fs';
import { fileURLToPath } from 'node:url';
import { randomUUID } from 'node:crypto';
import config from '../build.config.js';
import * as logger from '../utils/logger.js';
import { runCommand, getCpuCount } from '../utils/emscripten.js';
import { hashFiles, hashDirectory, HashCache } from '../utils/hash.js';
import { generateShaderEmbeds } from './shader-embeds.js';
import { GLX_EMSDK, glxEmsdkEnv, glxLibrary } from '../toolchain/glx.js';

/**
 * The one flag a probe build adds, and it adds ONLY this one.
 *
 * Build type, optimisation, backend and every feature define stay where the
 * target put them — a probe build is the shipping renderer answering one more
 * call. It joins the cache key, so one is never served for a release ask.
 */
function testProbeFlags() {
    // ALWAYS spelled, never omitted: a CMake option is sticky in the cache, so
    // leaving it out of a release configure keeps whatever the last probe build
    // set — and the release wasm then still exports the test adapter.
    return [`-DES_ENABLE_TEST_PROBES=${process.env.ESTELLA_TEST_PROBES === '1' ? 'ON' : 'OFF'}`];
}

/**
 * Extra `-D` flags for one build, from `ESTELLA_CMAKE_FLAGS`.
 *
 * Measuring what a subsystem costs in the core means building without it, and
 * every switch for that is already a CMake option — what was missing was a way
 * to reach one. Space-separated, as on a cmake line.
 */
function extraCmakeFlags() {
    return (process.env.ESTELLA_CMAKE_FLAGS ?? '').split(/\s+/).filter(Boolean);
}

async function computeWasmHash(target, targetConfig, debug) {
    const rootDir = config.paths.root;
    const srcDir = path.join(rootDir, 'src/esengine');
    const cmakeLists = path.join(rootDir, 'CMakeLists.txt');

    const sourceHash = await hashDirectory(srcDir, /\.(hpp|cpp|h|esshader)$/);

    const buildType = debug ? 'Debug' : 'Release';
    const flagsKey = [...targetConfig.cmakeFlags, ...testProbeFlags(),
                      `CMAKE_BUILD_TYPE=${buildType}`].join('|');

    // The toolchain files too: link flags live in cmake/Emscripten.cmake, and a
    // hash that only covered CMakeLists.txt served a cached build after they
    // changed — which is a wasm that does not match the flags it was asked for.
    const toolchainHash = await hashDirectory(path.join(rootDir, 'cmake'), /\.cmake$/);
    const configHash = await hashFiles([cmakeLists]);
    // The post-link optimizer is part of what a cached build promises. A change
    // to its transaction or optimization level must not reuse yesterday's output.
    const builderHash = await hashFiles([fileURLToPath(import.meta.url)]);

    const { createHash } = await import('crypto');
    return createHash('md5')
        .update(sourceHash)
        .update(configHash)
        .update(toolchainHash)
        .update(builderHash)
        .update(JSON.stringify(config.optimization[target] ?? {}))
        .update(flagsKey)
        .digest('hex');
}

export async function buildWasm(target, options = {}) {
    const { debug = false, clean = false, manifest = null, noCache = false } = options;

    const targetConfig = config.wasm[target];
    if (!targetConfig) {
        throw new Error(`Unknown target: ${target}. Available: ${Object.keys(config.wasm).join(', ')}`);
    }

    if (manifest) {
        manifest.startTarget(`wasm:${target}`);
    }

    try {
        logger.step(`Building WASM for ${target}...`);

        const rootDir = config.paths.root;
        const buildDir = path.join(rootDir, targetConfig.buildDir);
        const outputDir = config.paths.output;

        if (!noCache && !clean) {
            const cache = new HashCache(config.paths.cache);
            await cache.load();

            const currentHash = await computeWasmHash(target, targetConfig, debug);
            const cacheKey = `wasm:${target}`;

            if (!await cache.isChanged(cacheKey, currentHash)) {
                logger.success(`WASM ${target}: No changes detected (cached)`);
                const result = {
                    target,
                    buildDir,
                    outputDir,
                    outputs: Object.values(targetConfig.outputs),
                    skipped: true,
                };
                if (manifest) {
                    await manifest.endTarget(`wasm:${target}`, result);
                }
                return result;
            }

            await executeWasmBuild(target, targetConfig, { debug, clean, buildDir, rootDir, outputDir });

            cache.set(cacheKey, currentHash);
            await cache.save();
        } else {
            await executeWasmBuild(target, targetConfig, { debug, clean, buildDir, rootDir, outputDir });
        }

        logger.success(`WASM ${target}: Build complete`);

        const result = {
            target,
            buildDir,
            outputDir,
            outputs: Object.values(targetConfig.outputs),
        };

        if (manifest) {
            await manifest.endTarget(`wasm:${target}`, result);
        }

        return result;
    } catch (error) {
        if (manifest) {
            manifest.markTargetFailed(`wasm:${target}`, error);
        }
        throw error;
    }
}

/**
 * The generator a build directory was already configured with, or null for a fresh
 * one. CMake refuses a reconfigure that names a different generator, and `emcmake`
 * appends one of its own choosing when the caller names none — which on Windows is
 * MinGW Makefiles, so a tree configured with Ninja could never be rebuilt again.
 */
async function configuredGenerator(buildDir) {
    const cache = path.join(buildDir, 'CMakeCache.txt');
    if (!existsSync(cache)) return null;
    const line = /^CMAKE_GENERATOR:INTERNAL=(.+)$/m.exec(await readFile(cache, 'utf8'));
    return line ? line[1].trim() : null;
}

async function executeWasmBuild(target, targetConfig, { debug, clean, buildDir, rootDir, outputDir }) {
    await generateShaderEmbeds();

    if (clean && existsSync(buildDir)) {
        logger.debug(`Cleaning ${buildDir}`);
        await rm(buildDir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
    }

    await mkdir(buildDir, { recursive: true });

    const buildType = debug ? 'Debug' : 'Release';
    const optConfig = config.optimization[target];
    // A target pinned to another emsdk builds in that emsdk's environment, and
    // only its own children see it.
    const env = targetConfig.emsdk === GLX_EMSDK ? glxEmsdkEnv((m) => logger.info(m)) : undefined;
    const glxFlags = targetConfig.glx ? [`-DES_WXGAME_GLX_LIB=${await glxLibrary()}`] : [];
    const cmakeArgs = [
        'cmake',
        ...targetConfig.cmakeFlags,
        ...glxFlags,
        ...testProbeFlags(),
        ...extraCmakeFlags(),
        `-DCMAKE_BUILD_TYPE=${buildType}`,
    ];

    if (process.platform === 'win32') {
        cmakeArgs.push('-DES_ENABLE_CCACHE=OFF');
    }

    if (!debug && optConfig?.cmakeOpt) {
        cmakeArgs.push(`-DCMAKE_C_FLAGS=${optConfig.cmakeOpt}`);
        cmakeArgs.push(`-DCMAKE_CXX_FLAGS=${optConfig.cmakeOpt}`);
        cmakeArgs.push('-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON');
    }

    const generator = await configuredGenerator(buildDir);
    if (generator) {
        cmakeArgs.push('-G', generator);
    }

    cmakeArgs.push(rootDir);

    logger.debug(`CMake configure: emcmake ${cmakeArgs.join(' ')}`);
    await runCommand('emcmake', cmakeArgs, { cwd: buildDir, env });

    const cpuCount = getCpuCount();
    const buildArgs = ['--build', '.', '-j', String(cpuCount)];

    if (targetConfig.targets && targetConfig.targets.length > 0) {
        buildArgs.push('--target', ...targetConfig.targets);
    }

    logger.debug(`CMake build: cmake ${buildArgs.join(' ')}`);
    await runCommand('cmake', buildArgs, { cwd: buildDir, env });

    const wasmOptLevel = config.optimization[target]?.wasmOpt || '-O2';
    await optimizeWasmFiles(buildDir, targetConfig.outputs, wasmOptLevel, env);
    await copyOutputs(buildDir, outputDir, targetConfig.outputs);
}

export async function optimizeWasmFiles(buildDir, outputs, optLevel = '-O2', env = undefined) {
    const wasmFiles = Object.keys(outputs).filter(src => src.endsWith('.wasm'));
    if (wasmFiles.length === 0) return;

    for (const src of wasmFiles) {
        const wasmPath = path.join(buildDir, src);
        if (!existsSync(wasmPath)) continue;

        const optimizedPath = `${wasmPath}.opt-${randomUUID()}.wasm`;
        try {
            const before = (await stat(wasmPath)).size;
            // The optimizer drops the name section unless told to keep it; a build made to find where the bytes are (a size report by subsystem) asks.
            const keepNames = process.env.ESTELLA_WASM_NAMES === '1' ? ['-g'] : [];
            await runCommand('wasm-opt', [optLevel, ...keepNames, '--enable-bulk-memory', '--enable-nontrapping-float-to-int', '--enable-simd', '-o', optimizedPath, wasmPath], { silent: true, env });
            const bytes = await readFile(optimizedPath);
            if (!WebAssembly.validate(bytes)) throw new Error('optimizer produced invalid WASM');
            const after = bytes.byteLength;
            await rename(optimizedPath, wasmPath);
            const reduction = ((1 - after / before) * 100).toFixed(1);
            logger.debug(`wasm-opt ${optLevel}: ${src} ${(before / 1024).toFixed(0)}KB → ${(after / 1024).toFixed(0)}KB (-${reduction}%)`);
        } catch (error) {
            // Keep the original artifact intact; never label a failed release
            // stage successful or copy a partially overwritten file to the SDK.
            throw new Error(`wasm-opt ${optLevel} failed for ${src}: ${error.stderr?.trim() || error.message}`, { cause: error });
        } finally {
            await rm(optimizedPath, { force: true, maxRetries: 10, retryDelay: 50 });
        }
    }

}

async function copyOutputs(buildDir, outputDir, outputs) {
    const missing = [];

    for (const [src, dest] of Object.entries(outputs)) {
        const srcPath = path.join(buildDir, src);
        const destPath = path.join(outputDir, dest);

        if (!existsSync(srcPath)) {
            if (src.endsWith('.wasm') || src.endsWith('.js')) {
                missing.push(src);
            } else {
                logger.warn(`Output file not found: ${srcPath}`);
            }
            continue;
        }

        await mkdir(path.dirname(destPath), { recursive: true });
        await cp(srcPath, destPath);
        logger.debug(`Copied ${src} → ${dest}`);
    }

    if (missing.length > 0) {
        throw new Error(`Critical output files missing after build: ${missing.join(', ')}`);
    }
}

export async function buildWasmParallel(targets, options = {}) {
    const { manifest = null, continueOnError = false } = options;

    const groups = new Map();
    for (const target of targets) {
        const targetConfig = config.wasm[target];
        if (!targetConfig) {
            throw new Error(`Unknown target: ${target}. Available: ${Object.keys(config.wasm).join(', ')}`);
        }
        const dir = targetConfig.buildDir;
        if (!groups.has(dir)) {
            groups.set(dir, []);
        }
        groups.get(dir).push(target);
    }

    const groupPromises = [];
    for (const groupTargets of groups.values()) {
        groupPromises.push((async () => {
            const results = [];
            for (const target of groupTargets) {
                try {
                    const result = await buildWasm(target, { ...options, manifest });
                    results.push({ target, status: 'success', result });
                } catch (error) {
                    results.push({ target, status: 'failed', error });
                    logger.error(`Build failed for ${target}: ${error.message}`);
                }
            }
            return results;
        })());
    }

    const allResults = await Promise.allSettled(groupPromises);

    const failures = [];
    const successes = [];

    for (const groupResult of allResults) {
        if (groupResult.status === 'fulfilled') {
            for (const targetResult of groupResult.value) {
                if (targetResult.status === 'success') {
                    successes.push(targetResult.target);
                } else {
                    failures.push({
                        target: targetResult.target,
                        error: targetResult.error,
                    });
                }
            }
        } else {
            logger.error(`Group build failed: ${groupResult.reason.message}`);
            failures.push({
                target: 'unknown',
                error: groupResult.reason,
            });
        }
    }

    if (successes.length > 0) {
        logger.success(`Successfully built: ${successes.join(', ')}`);
    }

    if (failures.length > 0) {
        logger.error(`\nBuild failures (${failures.length}):`);
        for (const failure of failures) {
            logger.error(`  - ${failure.target}: ${failure.error.message}`);
        }

        if (!continueOnError) {
            throw new Error(`${failures.length} target(s) failed to build`);
        } else {
            logger.warn(`\n⚠️  ${failures.length} target(s) failed, continuing due to --continue-on-error`);
        }
    }
}

export async function buildAllWasm(options = {}) {
    await buildWasmParallel(Object.keys(config.wasm), options);
}

export async function cleanWasm(target = null) {
    const rootDir = config.paths.root;

    if (target) {
        const targetConfig = config.wasm[target];
        if (!targetConfig) {
            throw new Error(`Unknown target: ${target}`);
        }
        const buildDir = path.join(rootDir, targetConfig.buildDir);
        if (existsSync(buildDir)) {
            await rm(buildDir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
            logger.debug(`Removed ${buildDir}`);
        }
    } else {
        for (const t of Object.keys(config.wasm)) {
            await cleanWasm(t);
        }
    }
}