A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(renderer): readying a cell's shader variants early removes the first-visibility hitch
esengine9e2fdf9e0260
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 5c578a359..fdf6fa5ad 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -47,6 +47,11 @@ if(MSVC)endif()option(ES_SANITIZE "Build with AddressSanitizer + UndefinedBehaviorSanitizer" OFF)+# Adds test-only ADAPTERS to the exported surface and changes nothing else. It+# must stay orthogonal to build type, optimisation, backend and every feature+# define: a probe build has to be a shipping renderer that answers one more call,+# or the counterfactual it exists for measures a different engine.+option(ES_ENABLE_TEST_PROBES "Export test-only adapters (never in a release ABI)" OFF)if(ES_SANITIZE)if(MSVC)message(FATAL_ERROR "ES_SANITIZE requires clang or gcc (MSVC has no UBSan).")@@ -253,6 +258,10 @@ if(ES_ENABLE_POSTPROCESS)target_compile_definitions(esengine PUBLIC ES_ENABLE_POSTPROCESS)endif()+if(ES_ENABLE_TEST_PROBES)+ target_compile_definitions(esengine PUBLIC ES_ENABLE_TEST_PROBES)+endif()+if(ES_ENABLE_WEBGPU)target_compile_definitions(esengine PUBLIC ES_ENABLE_WEBGPU)# PUBLIC: the port supplies webgpu.h at compile time and its JS glue atdiff --git a/bench/residency/heavy.mjs b/bench/residency/heavy.mjsindex 905eab513..d338d7036 100755--- a/bench/residency/heavy.mjs+++ b/bench/residency/heavy.mjs@@ -36,6 +36,8 @@ const flag = (name, fallback) => {};const CELL = 600;+/** The one cell this fixture cooks, as residency registers it. */+const CELL_NAME = 'main.cell_1_0';const PROPS = Number(flag('props', '40'));const BODIES = Number(flag('bodies', '200'));@@ -220,6 +222,13 @@ function main() {// bookkeeping shows in the first, first visibility in the second.['blind', [{ do: 'step', frames: 30 }, { do: 'tap', key: 'Digit3', frames: 10 },profile('published', 'Digit1'), profile('revealed', 'Digit4')]],+ // The same walk, with the programs readied while the camera is still+ // away. If first visibility then costs what a warm frame costs, the+ // hitch was the compile and readying it early is where it belongs.+ ['prewarmed', [{ do: 'step', frames: 30 }, { do: 'tap', key: 'Digit3', frames: 10 },+ profile('published', 'Digit1'),+ { do: 'prewarm', as: 'ready', cell: CELL_NAME },+ profile('revealed', 'Digit4')]],];for (const [arm, steps] of arms) {@@ -227,12 +236,21 @@ function main() {for (const [as, { frames, delivery }] of runs) {report(`${arm}${runs.size > 1 ? `/${as}` : ''}`, frames, delivery);}+ const ready = prewarmed.get(arm);+ if (ready) {+ console.log(`\n readying ${arm}'s programs early took ${ready.ms.toFixed(2)} ms`+ + ` — ${ready.compiles} compile(s) of ${ready.uniqueKeys} unique key(s)`+ + ` over ${ready.asks} ask(s) for ${ready.entities} entit(ies);`+ + ` ${ready.materialCompiles} material compile(s) of ${ready.materialAsks} ask(s)`);+ }}console.log('');process.exit(failedTotal === 0 ? 0 : 1);}let failedTotal = 0;+/** What readying an arm's programs early cost, by arm. */+const prewarmed = new Map();/** Run one arm in its own Electron, so each pays for its own cold cache. */function drive(out, name, steps) {@@ -249,6 +267,11 @@ function drive(out, name, steps) {// experiment that separates them stops separating them.const runs = new Map();for (const line of (run.stdout || '').split('\n')) {+ const ready = /^prewarm ([^:]+): (.*)$/.exec(line);+ if (ready) {+ try { prewarmed.set(name, JSON.parse(ready[2])); } catch { /* partial */ }+ continue;+ }const match = /^(profile|delivery) ([^:]+): /.exec(line);if (!match) continue;const [, kind, as] = match;diff --git a/build-tools/tasks/wasm.js b/build-tools/tasks/wasm.jsindex bb4be4f5e..e0fa47c4d 100644--- a/build-tools/tasks/wasm.js+++ b/build-tools/tasks/wasm.js@@ -9,6 +9,20 @@ import { runCommand, getCpuCount } from '../utils/emscripten.js';import { hashFiles, hashDirectory, HashCache } from '../utils/hash.js';import { generateShaderEmbeds } from './shader-embeds.js';+/**+ * The one flag a probe build adds, and it adds ONLY this one.+ *+ * Build type, optimisation, backend and every feature define stay where the+ * target put them — a probe build is the shipping renderer answering one more+ * call. It joins the cache key, so one is never served for a release ask.+ */+function testProbeFlags() {+ // ALWAYS spelled, never omitted: a CMake option is sticky in the cache, so+ // leaving it out of a release configure keeps whatever the last probe build+ // set — and the release wasm then still exports the test adapter.+ return [`-DES_ENABLE_TEST_PROBES=${process.env.ESTELLA_TEST_PROBES === '1' ? 'ON' : 'OFF'}`];+}+async function computeWasmHash(target, targetConfig, debug) {const rootDir = config.paths.root;const srcDir = path.join(rootDir, 'src/esengine');@@ -17,7 +31,8 @@ async function computeWasmHash(target, targetConfig, debug) {const sourceHash = await hashDirectory(srcDir, /\.(hpp|cpp|h|esshader)$/);const buildType = debug ? 'Debug' : 'Release';- const flagsKey = [...targetConfig.cmakeFlags, `CMAKE_BUILD_TYPE=${buildType}`].join('|');+ const flagsKey = [...targetConfig.cmakeFlags, ...testProbeFlags(),+ `CMAKE_BUILD_TYPE=${buildType}`].join('|');// The toolchain files too: link flags live in cmake/Emscripten.cmake, and a// hash that only covered CMakeLists.txt served a cached build after they@@ -133,6 +148,7 @@ async function executeWasmBuild(target, targetConfig, { debug, clean, buildDir,const cmakeArgs = ['cmake',...targetConfig.cmakeFlags,+ ...testProbeFlags(),`-DCMAKE_BUILD_TYPE=${buildType}`,];diff --git a/pipeline/src/runtime/gameHost.ts b/pipeline/src/runtime/gameHost.tsindex a73c008be..ee5eb3b3d 100644--- a/pipeline/src/runtime/gameHost.ts+++ b/pipeline/src/runtime/gameHost.ts@@ -141,7 +141,47 @@ async function boot(): Promise<void> {if (headless) {let statsOn = false;- (window as unknown as { __estellaCooked?: unknown }).__estellaCooked = {+ /**+ * Ready the shader programs a resident cell's meshes will be asked for,+ * drawing none of them. Attached only where the engine answers it.+ *+ * Turning the camera would ready them too, and would bring every other+ * first-sight effect with it — which is the isolation this exists to keep.+ */+ const prewarmMeshVariants = (cell: string): Record<string, number> => {+ const rows = (worldResidencyReport(app).cellRows ?? {})[cell];+ if (!rows || rows.length === 0) throw new Error(`[estella] no resident cell "${cell}"`);+ const entities = new Uint32Array(rows.map((r) => r.entity));+ const m = module as unknown as {+ _malloc(n: number): number;+ _free(p: number): void;+ HEAPU32: Uint32Array;+ engine_prewarmMeshVariants(registry: unknown, entitiesPtr: number, count: number,+ outPtr: number): void;+ };+ const entityPtr = m._malloc(entities.length * 4);+ const outPtr = m._malloc(5 * 4);+ try {+ m.HEAPU32.set(entities, entityPtr >> 2);+ // Around the call ALONE: reading the counts back is this probe's cost,+ // not the readiness cost the experiment is about.+ const began = performance.now();+ m.engine_prewarmMeshVariants(app.world.getCppRegistry(), entityPtr, entities.length,+ outPtr);+ const ms = performance.now() - began;+ const out = m.HEAPU32.subarray(outPtr >> 2, (outPtr >> 2) + 5);+ return {+ ms, entities: entities.length,+ asks: out[0], compiles: out[1], uniqueKeys: out[2],+ materialAsks: out[3], materialCompiles: out[4],+ };+ } finally {+ m._free(entityPtr);+ m._free(outPtr);+ }+ };++ const cooked: Record<string, unknown> = {capture(): { width: number; height: number; rgba: Uint8Array } {const w = canvas.width, h = canvas.height;const rgba = new Uint8Array(w * h * 4);@@ -535,6 +575,14 @@ async function boot(): Promise<void> {return { changed: plan.changedAssets.length, applied: result.ok, failed: result.failed.length };},};+ // Registered only where the engine answers it. A release build exports no+ // test adapter, and a probe that existed there and threw would turn "this+ // build cannot do that" into a runtime error about something else.+ if (typeof (module as unknown as Record<string, unknown>).engine_prewarmMeshVariants+ === 'function') {+ cooked.prewarmMeshVariants = prewarmMeshVariants;+ }+ (window as unknown as { __estellaCooked?: unknown }).__estellaCooked = cooked;}// physics.wasm sits next to esengine.wasm; the runtime loads it when a scene// uses physics, or when the project declared physics on for bodies it spawnsdiff --git a/src/esengine/bindings/RendererBindings.cpp b/src/esengine/bindings/RendererBindings.cppindex 298e8f37f..83ff6df8b 100644--- a/src/esengine/bindings/RendererBindings.cpp+++ b/src/esengine/bindings/RendererBindings.cpp@@ -738,6 +738,21 @@ std::string engine_getCpuScopes() {return FrameProfiler::get().lastJson();}+#ifdef ES_ENABLE_TEST_PROBES+void engine_prewarmMeshVariants(ecs::Registry& registry, u32 entitiesPtr, u32 count, u32 outPtr) {+ auto* out = reinterpret_cast<u32*>(static_cast<uintptr_t>(outPtr));+ for (u32 i = 0; i < 5; ++i) out[i] = 0;+ if (!g_initialized || !g_renderFrame) return;+ const auto* entities = reinterpret_cast<const Entity*>(static_cast<uintptr_t>(entitiesPtr));+ const RenderPrewarmResult r = g_renderFrame->prewarmPrograms(registry, entities, count);+ out[0] = r.asks;+ out[1] = r.compiles;+ out[2] = r.uniqueKeys;+ out[3] = r.materialAsks;+ out[4] = r.materialCompiles;+}+#endif+std::string engine_getCounters() {return FrameProfiler::get().lastCountersJson();}diff --git a/src/esengine/bindings/RendererBindings.hpp b/src/esengine/bindings/RendererBindings.hppindex dd38ebba4..a9411f536 100644--- a/src/esengine/bindings/RendererBindings.hpp+++ b/src/esengine/bindings/RendererBindings.hpp@@ -136,6 +136,18 @@ void engine_setCpuProfiling(bool on);* Unset, every run differs — which is what a player wants of particles. */void engine_setRandomSeed(u32 seed);std::string engine_getCpuScopes();++#ifdef ES_ENABLE_TEST_PROBES+/**+ * @brief Ready the shader programs a set of entities will need, drawing none.+ *+ * @details A disposable adapter over `RenderFrame::prewarmPrograms` — the only+ * gated part of shader readiness, the mechanism being ordinary renderer+ * code a production lifecycle would call directly. Writes five counts+ * to @p outPtr; JSON here would put an allocation inside the measurement.+ */+void engine_prewarmMeshVariants(ecs::Registry& registry, u32 entitiesPtr, u32 count, u32 outPtr);+#endifstd::string engine_getCounters();std::string engine_getGpuScopes();f64 renderer_getTextureBytes();diff --git a/src/esengine/bindings/WebSDKEntry.cpp b/src/esengine/bindings/WebSDKEntry.cppindex c74e785a7..8670a3d5f 100644--- a/src/esengine/bindings/WebSDKEntry.cpp+++ b/src/esengine/bindings/WebSDKEntry.cpp@@ -718,6 +718,9 @@ EMSCRIPTEN_BINDINGS(esengine_renderer) {emscripten::function("engine_setCpuProfiling", &esengine::engine_setCpuProfiling);emscripten::function("engine_setRandomSeed", &esengine::engine_setRandomSeed);emscripten::function("engine_getCpuScopes", &esengine::engine_getCpuScopes);+#ifdef ES_ENABLE_TEST_PROBES+ emscripten::function("engine_prewarmMeshVariants", &esengine::engine_prewarmMeshVariants);+#endifemscripten::function("engine_getCounters", &esengine::engine_getCounters);emscripten::function("engine_getGpuScopes", &esengine::engine_getGpuScopes);emscripten::function("renderer_getTextureBytes", &esengine::renderer_getTextureBytes);diff --git a/src/esengine/renderer/RenderTypePlugin.hpp b/src/esengine/renderer/RenderTypePlugin.hppindex db0107819..7c9c0dc32 100644--- a/src/esengine/renderer/RenderTypePlugin.hpp+++ b/src/esengine/renderer/RenderTypePlugin.hpp@@ -105,6 +105,26 @@ inline CameraView computeCameraView(const glm::mat4& viewProjection) {return view;}+/** @brief What readying a set of renderables' programs found. Counts, not JSON:+ * this crosses no boundary and pays for no encoding. */+struct RenderPrewarmResult {+ u32 asks = 0;+ u32 compiles = 0;+ /** Distinct variant keys the content needed — the dimension of the answer. */+ u32 uniqueKeys = 0;+ u32 materialAsks = 0;+ u32 materialCompiles = 0;++ RenderPrewarmResult& operator+=(const RenderPrewarmResult& other) {+ asks += other.asks;+ compiles += other.compiles;+ uniqueKeys += other.uniqueKeys;+ materialAsks += other.materialAsks;+ materialCompiles += other.materialCompiles;+ return *this;+ }+};+struct RenderCollectContext {ecs::Registry& registry;const Frustum& frustum;@@ -293,6 +313,20 @@ public:virtual void collect(RenderCollectContext& ctx) = 0;+ /**+ * @brief Make ready the programs @p entities will be asked for, drawing none.+ *+ * @details What it readies is derived by the SAME rule the collect derives it+ * by, so a caller cannot ready a variant the draw will not ask for.+ * `shadowPasses` says whether this scene also renders depth, which+ * needs its own variant. The default readies nothing.+ */+ virtual RenderPrewarmResult prewarm(RenderFrameContext& ctx, ecs::Registry& registry,+ const Entity* entities, u32 count, bool shadowPasses) {+ (void)ctx; (void)registry; (void)entities; (void)count; (void)shadowPasses;+ return {};+ }+/*** @brief What this plugin's collect is called in a frame profile.*diff --git a/src/esengine/renderer/frame/RenderFrame.cpp b/src/esengine/renderer/frame/RenderFrame.cppindex 243fd3a8f..6d2a7a006 100644--- a/src/esengine/renderer/frame/RenderFrame.cpp+++ b/src/esengine/renderer/frame/RenderFrame.cpp@@ -1576,6 +1576,20 @@ bool RenderFrame::visibleToCamera(i32 layer, const glm::vec3& center,return frustum_.intersectsAABB(center, halfExtents);}+RenderPrewarmResult RenderFrame::prewarmPrograms(ecs::Registry& registry,+ const Entity* entities, u32 count) {+ auto ctx = makeContext();+ // What the SCENE renders, read from the plan the last frame built rather than+ // assumed: a world with no shadow caster needs no depth variant, and readying+ // one would be six milliseconds spent on a program nothing asks for.+ const bool shadowPasses = !shadow_casters_.empty();+ RenderPrewarmResult out;+ for (auto& plugin : plugins_) {+ out += plugin->prewarm(ctx, registry, entities, count, shadowPasses);+ }+ return out;+}+void RenderFrame::collectAll(ecs::Registry& registry) {ES_PROFILE_SCOPE("render.collect");// Split because "collect is expensive" names no mechanism: planning shadows,diff --git a/src/esengine/renderer/frame/RenderFrame.hpp b/src/esengine/renderer/frame/RenderFrame.hppindex 1e9146e06..2c747fa58 100644--- a/src/esengine/renderer/frame/RenderFrame.hpp+++ b/src/esengine/renderer/frame/RenderFrame.hpp@@ -404,6 +404,16 @@ public:void destroySkeletalPreview(SkeletalPreviewId id);void addPlugin(std::unique_ptr<RenderTypePlugin> plugin);++ /**+ * @brief Make ready the programs @p entities will need, drawing nothing.+ *+ * @details Ordinary renderer code, reachable by anything inside the engine —+ * a residency lifecycle that wants a cell render-ready before it is+ * seen would call THIS. The only thing gated to test builds is the+ * WASM adapter that lets a driver call it from outside.+ */+ RenderPrewarmResult prewarmPrograms(ecs::Registry& registry, const Entity* entities, u32 count);void collectAll(ecs::Registry& registry);/**diff --git a/src/esengine/renderer/plugins/MeshPlugin.cpp b/src/esengine/renderer/plugins/MeshPlugin.cppindex 18dee12e3..a7949be7b 100644--- a/src/esengine/renderer/plugins/MeshPlugin.cpp+++ b/src/esengine/renderer/plugins/MeshPlugin.cpp@@ -287,6 +287,47 @@ u32 MeshPlugin::meshProgram(RenderFrameContext& ctx, u32 variant) {return mesh_programs_[variant];}+RenderPrewarmResult MeshPlugin::prewarm(RenderFrameContext& ctx, ecs::Registry& registry,+ const Entity* entities, u32 count, bool shadowPasses) {+ RenderPrewarmResult out;+ u64 seen = 0;+ const u32 materialsBefore = ctx.materials ? ctx.materials->builtVariantCount() : 0;+ for (u32 i = 0; i < count; ++i) {+ const Entity entity = entities[i];+ const auto* mesh = registry.tryGet<ecs::MeshRenderer>(entity);+ if (!mesh || !mesh->enabled) continue;+ if (mesh->indices.empty() && !mesh->mesh.isValid()) continue;+ const Mesh* resident = mesh->mesh.isValid() ? ctx.resources.getMesh(mesh->mesh) : nullptr;+ // Both purposes when the scene draws depth too: the same renderable is+ // asked for twice with different keys, and readying only the camera's+ // leaves the first shadow map to pay for the other.+ for (u32 pass = 0; pass < 2; ++pass) {+ const bool shadowDepth = pass == 1;+ if (shadowDepth && (!shadowPasses || !resident)) continue;+ const u32 variant =+ meshDrawFor(ctx, registry, entity, *mesh, resident, shadowDepth,+ warned_bones_).variant;+ ++out.asks;+ if ((seen & (1ull << variant)) == 0) { seen |= 1ull << variant; ++out.uniqueKeys; }+ if (!mesh_compiled_[variant]) ++out.compiles;+ meshProgram(ctx, variant);+ }+ // The second lazily-compiled path. A fixture without materials would let+ // readiness look complete while a material-shaded world still hitched.+ if (mesh->material != 0 && ctx.materials && resident) {+ ++out.materialAsks;+ ctx.materials->meshProgram(mesh->material, ctx.resources, resident->hasNormals,+ skinJointCount(registry, entity, *resident, warned_bones_) > 0,+ ctx.environment_texture_id != 0);+ }+ }+ if (ctx.materials) {+ out.materialCompiles = ctx.materials->builtVariantCount() - materialsBefore;+ }+ compiled_this_frame_ = 0;+ return out;+}+void MeshPlugin::collect(RenderCollectContext& collect_ctx) {auto& registry = collect_ctx.registry;auto& frustum = collect_ctx.frustum;diff --git a/src/esengine/renderer/plugins/MeshPlugin.hpp b/src/esengine/renderer/plugins/MeshPlugin.hppindex 88e8bcd82..142fcf7a1 100644--- a/src/esengine/renderer/plugins/MeshPlugin.hpp+++ b/src/esengine/renderer/plugins/MeshPlugin.hpp@@ -25,6 +25,8 @@ public:/// renderable's shadow is ShadowCaster2D's to cast, not the map's.bool castsShadows() const override { return true; }void collect(RenderCollectContext& ctx) override;+ RenderPrewarmResult prewarm(RenderFrameContext& ctx, ecs::Registry& registry,+ const Entity* entities, u32 count, bool shadowPasses) override;const char* collectScope() const override { return "render.collect.mesh"; }private:diff --git a/src/esengine/renderer/store/MaterialStore.hpp b/src/esengine/renderer/store/MaterialStore.hppindex 79b7708d8..fd0d74bac 100644--- a/src/esengine/renderer/store/MaterialStore.hpp+++ b/src/esengine/renderer/store/MaterialStore.hpp@@ -300,6 +300,13 @@ private:/// asks a read-only store, and the answer is the same every time.mutable std::unordered_map<u64, u32> mesh_programs_;+public:+ /** @brief How many material program variants have been built. A prewarm counts+ * what it caused by the difference, rather than by asking again. */+ u32 builtVariantCount() const { return static_cast<u32>(mesh_programs_.size()); }++private:+/// One material compiled for one vertex source: @p shape distinguishes the/// cache entries, @p extraFeatures the engine vertex stage, @p what the log.u32 variantProgram(u32 materialId, resource::ResourceManager& resources,diff --git a/tools/launchers/residency-run.mjs b/tools/launchers/residency-run.mjsindex a6579c6af..8d01d2625 100644--- a/tools/launchers/residency-run.mjs+++ b/tools/launchers/residency-run.mjs@@ -239,6 +239,16 @@ async function main() {continue;}if (step.do === 'tap') { await exec(tapScript(step.key, step.frames ?? 10)); continue; }+ if (step.do === 'prewarm') {+ // Absent capability is a FAILED run, never a quiet pass: a probe that was+ // not built reads as "nothing needed compiling", which is the answer the+ // experiment is trying to earn.+ const has = await exec('typeof window.__estellaCooked.prewarmMeshVariants === "function"');+ if (!has) { stop(); server.close(); return fail('this build has no prewarm probe', 2); }+ const r = await exec(`window.__estellaCooked.prewarmMeshVariants(${JSON.stringify(step.cell)})`);+ console.log(`prewarm ${step.as}: ${JSON.stringify(r)}`);+ continue;+ }if (step.do === 'frames') {// One frame at a time, timed INSIDE the page: a spike is a property of a// frame, and measuring from outside adds a round trip to every sample.