A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
#pragma once
#include "../store/ProbeConstants.hpp"
#include "../store/PlanarConstants.hpp"
#include "../../core/Types.hpp"
#include "../frame/RenderStage.hpp"
#include "./RenderItem.hpp"
#include "./BlendMode.hpp"
#include "../rhi/TransientBufferPool.hpp"
#include "../rhi/PipelineState.hpp"
#include <algorithm>
#include <bit>
#include <cstring>
namespace esengine {
static constexpr u32 MAX_CMD_TEXTURE_SLOTS = 8;
static constexpr u16 CMD_STATE_SCISSOR = 0x01;
static constexpr u16 CMD_STATE_STENCIL_WRITE = 0x02;
static constexpr u16 CMD_STATE_UI_STENCIL = 0x20;
static constexpr u16 CMD_STATE_STENCIL_INCREMENT = 0x40;
static constexpr u16 CMD_STATE_STENCIL_DECREMENT = 0x80;
static constexpr u16 CMD_STATE_STENCIL_ZERO = 0x100;
static constexpr u16 CMD_STATE_STENCIL_TEST = 0x04;
static constexpr u16 CMD_STATE_CUSTOM_DRAW = 0x08;
// Tests the reference the other way: drawn only where the stencil does NOT match.
// A separate flag rather than a sign on the ref, so a draw inside a mask and one
// outside the same mask never merge — they are different pipelines.
static constexpr u16 CMD_STATE_STENCIL_OUT = 0x10;
/**
* @brief The stencil mode a draw's state flags mean.
* @details One reading of these bits. Done at the point of use it would be done in each
* backend's own words, and a mode nobody mapped is a draw that quietly tests the
* wrong way — which looks like a mask that works everywhere but one path.
*/
inline GfxStencilMode stencilModeOf(u16 stateFlags) {
if (stateFlags & CMD_STATE_STENCIL_INCREMENT) return GfxStencilMode::Increment;
if (stateFlags & CMD_STATE_STENCIL_DECREMENT) return GfxStencilMode::Decrement;
if (stateFlags & CMD_STATE_STENCIL_ZERO) return GfxStencilMode::Zero;
if (stateFlags & CMD_STATE_STENCIL_WRITE) return GfxStencilMode::Write;
if (stateFlags & CMD_STATE_STENCIL_TEST) {
return (stateFlags & CMD_STATE_STENCIL_OUT) ? GfxStencilMode::TestOutside
: GfxStencilMode::Test;
}
return GfxStencilMode::Off;
}
/**
* Why a draw call had to start rather than join the one before it. Every member
* is a condition the merge itself tests, so the list cannot drift from the rule.
*/
enum class BatchBreak : u8 {
None = 0,
RunStart,
Instanced,
Shader,
Blend,
Layout,
Material,
Depth,
Cull,
State,
Scissor,
Stencil,
IndexGap,
TextureSlots,
Count,
};
/**
* The profiler counter each reason publishes under. String LITERALS: FrameProfiler
* keys entries by the pointer's contents and never copies, so a built-up buffer
* would dangle the moment the frame ended.
*/
inline const char* batchBreakCounter(BatchBreak r) {
switch (r) {
case BatchBreak::RunStart: return "batch.break.runStart";
case BatchBreak::Instanced: return "batch.break.instanced";
case BatchBreak::Shader: return "batch.break.shader";
case BatchBreak::Blend: return "batch.break.blend";
case BatchBreak::Layout: return "batch.break.layout";
case BatchBreak::Material: return "batch.break.material";
case BatchBreak::Depth: return "batch.break.depth";
case BatchBreak::Cull: return "batch.break.cull";
case BatchBreak::State: return "batch.break.state";
case BatchBreak::Scissor: return "batch.break.scissor";
case BatchBreak::Stencil: return "batch.break.stencil";
case BatchBreak::IndexGap: return "batch.break.indexGap";
case BatchBreak::TextureSlots: return "batch.break.textureSlots";
default: return "batch.break.none";
}
}
struct DrawCommand {
u64 sort_key = 0;
u32 index_offset = 0;
u32 index_count = 0;
u32 vertex_byte_offset = 0;
u32 shader_id = 0;
/// The program that writes only this draw's depth, for the scene's depth prepass (0 = the draw takes no part in one). Its vertex stage is
/// the draw's own, so the depth it writes is the depth the draw's colour pass lands on.
u32 depth_shader_id = 0;
BlendMode blend_mode = BlendMode::Normal;
LayoutId layout_id = LayoutId::Batch;
// The material handle this draw resolved from (0 = none). Identity for sort + merge so
// distinct materials never coalesce, and the lookup key for per-material GPU state.
u32 material_id = 0;
// Pipeline depth/cull state, resolved from the material (or defaults). depth_write stays
// on with the test off — the engine's 2D state — unless a material overrides it.
bool depth_test = false;
bool depth_write = true;
u8 cull = 0; ///< CullMode: 0 = none, 1 = back, 2 = front.
i16 depth_bias = 0; ///< Depth-buffer units toward the eye. @see PipelineDesc.
GfxTopology topology = GfxTopology::Triangles; ///< Triangles or lines. @see PipelineDesc.
BatchBreak break_reason = BatchBreak::RunStart; ///< Why this draw call started.
u8 texture_count = 0;
u32 texture_ids[MAX_CMD_TEXTURE_SLOTS] = {};
u16 state_flags = 0;
ScissorRect scissor;
i32 stencil_ref = 0;
u16 stencil_depth = 0;
Entity entity = INVALID_ENTITY;
RenderType type = RenderType::Sprite;
// Kept beside the sort key rather than unpacked from it: the bit layout is
// buildSortKey's business alone, and a second place that knows it is a second
// place to forget when it moves. `layer` is here for the same reason.
RenderStage stage = RenderStage::Transparent;
i32 layer = 0;
u32 entity_count = 1;
/// This draw's instance records have been moved to the end of the stream,
/// where the run it heads is growing. Set by the merge, read by it alone.
bool instances_relocated = false;
// > 0 selects an instanced draw: index_count indices drawn instance_count times,
// with per-instance attributes based at vertex_byte_offset (see LayoutId::ParticleInstance).
u32 instance_count = 0;
// Bytes one of those records occupies; 0 = the stream's own stride, which the
// merge would have to guess. Set by whoever wrote them.
u32 instance_stride = 0;
// The instance records already sit in a GPU buffer (a compute kernel wrote them): bound whole in place of the
// stream's slice. Never merged; instance_count says how many to draw.
BufferHandle instance_buffer = BufferHandle::Invalid;
// Geometry on the GPU rather than in this frame's pool. Set together or not
// at all; index_offset/index_count then index INTO these. Sorting, material and
// depth state are unchanged — where vertices live is not what is drawn.
BufferHandle vertex_buffer = BufferHandle::Invalid;
BufferHandle index_buffer = BufferHandle::Invalid;
VertexLayoutHandle vertex_layout = VertexLayoutHandle::Invalid;
// A skinned draw's pose: where its bone matrices start in the frame's pool
// and how many there are. Zero count = the geometry is not deformed.
u32 skin_offset = 0;
u32 skin_count = 0;
// A morphed draw's shapes: 1 + where its block sits in the frame's pool, so
// 0 means the geometry is in the shape it was authored in.
u32 morph_index = 0;
// The indirect light where this draw stands: 1 + where its block sits in the
// frame's pool, so 0 means no volume holds it and the frame's environment
// answers instead.
u32 probe_index = 0;
u32 planar_index = 0;
// Where this draw's instances start in the frame's probe texture. A merged
// run reads slot `probe_base + gl_InstanceID`, which is why the merge
// records every instance's probe in draw order rather than dropping it.
u32 probe_base = 0;
bool hasPersistentGeometry() const { return vertex_buffer != BufferHandle::Invalid; }
// Vertices owned by this command (from vertex_byte_offset). Needed so the merge pass
// can rewrite their texIndex when coalescing into a multi-texture batch.
u32 vertex_count = 0;
// Texture is not in the key: draws differing only by texture then sort adjacent and
// coalesce into one multi-texture batch (up to 8, selected per-vertex in the shader).
// Layer, then order, then stage — the author's two promises outrank the physical
// grouping. Stage first would let an opaque draw in layer 5 precede a painter draw in
// layer 3: a sorting layer's whole meaning inverted, and an order's one level finer.
// Under both, the 3D order. A blended draw composites onto what is there, so its
// depth IS the result and outranks batching; an opaque one is order-independent, so
// material groups first and depth breaks ties. Interleaved orders cost early-z.
// [63:48] layer | [47:40] order | [39:38] stage, then 38 bits the stage decides:
// opaque [37:30] shader | [29:27] blend | [26:25] flags | [24:14] material | [13:0] depth (or [13:6] geometry | [5:0] depth)
// blended [37:18] depth | [17:10] shader | [9:7] blend | [6:5] flags | [4:0] material
// A SortingGroup owns [63:40] for its members; [37:30] then carries the member's own
// order. See withGroupOrder.
// Material pays for the order field, being the only one that can afford it: a
// batching HINT, not identity (canMergeWith compares in full), so a collision costs
// a merge and never a wrong draw.
// Stage now has exactly its four values and no spare. A fifth would land in `order`
// and reorder the frame silently, so it stops the build here instead.
static_assert(static_cast<u32>(RenderStage::Overlay) <= 3,
"RenderStage outgrew the sort key's 2-bit field — re-lay the key out");
// `geometry`, when nonzero, is a hash of the mesh an instanced opaque draw repeats. It takes the top bits of the opaque depth field, so draws
// of one mesh that share a material sort together and merge into one instanced draw rather than alternating with another mesh's by depth. A
// batching hint like the material: a collision costs a merge, never a wrong draw. Opaque draws are order-independent, so only the coarse
// front-to-back order inside a mesh's run is spent.
static u64 buildSortKey(RenderStage stage, i32 layer, u32 shaderId,
BlendMode blend, u16 stateFlags, f32 depth, u32 materialId = 0,
i32 order = 0, u32 geometry = 0) {
i32 normalizedLayer = std::clamp(layer + 32768, 0, 65535);
u64 layerKey = static_cast<u64>(normalizedLayer & 0xFFFF) << 48;
u64 stageKey = static_cast<u64>(stage) << 38;
// Order-preserving float mapping — monotonic over the FULL float range, so
// any real-world depth sorts. `depth` is CameraView::viewDepth, larger =
// nearer: blended back-to-front, opaque front-to-back for early-z.
const bool blended = stage == RenderStage::Transparent || stage == RenderStage::Overlay;
u32 orderedDepth = blended ? orderedFloatBits(depth) : ~orderedFloatBits(depth);
// Blended spends its bits on depth (20: sign, exponent, 11 mantissa — a step of
// ~0.004 around z=10) and leaves material 5; opaque keeps depth at 14 and material
// 11. One trade twice: depth is what the eye sees, material only what batches well.
if (blended) {
return layerKey | orderBits(order) | stageKey
| (static_cast<u64>(orderedDepth >> 12) << 18)
| (static_cast<u64>(shaderId & 0xFF) << 10)
| (blendBits(blend) << 7)
| (static_cast<u64>(stateFlags & 0x03) << 5)
| (static_cast<u64>(materialId) & 0x1F);
}
const u64 depthField = geometry != 0
? (static_cast<u64>(geometry & 0xFF) << 6) | static_cast<u64>(orderedDepth >> 26)
: static_cast<u64>(orderedDepth >> 18);
return layerKey | orderBits(order) | stageKey
| (static_cast<u64>(shaderId & 0xFF) << 30)
| (blendBits(blend) << 27)
| (static_cast<u64>(stateFlags & 0x03) << 25)
| ((static_cast<u64>(materialId) & 0x7FF) << 14)
| depthField;
}
/** @brief The blend field, 3 bits wide wherever it appears.
* @details Masked, because the enum outgrew the field: BlendMode::None is 9, and
* unmasked it set a bit in the SHADER field above it. Aliasing None onto
* Additive costs a merge; the spill cost a wrong order. */
static u64 blendBits(BlendMode blend) {
return static_cast<u64>(blend) & 0x7;
}
/** @brief The order field at [47:40], biased so a negative order sorts below zero.
* @details CLAMPED, not wrapped: an order past the range pins to the extreme it ran
* past, where a wrap would land it on the far side — "very much on top"
* turning into "behind everything" is the one failure a clamp cannot make. */
static u64 orderBits(i32 order) {
return static_cast<u64>(std::clamp(order + 128, 0, 255)) << 40;
}
/**
* @brief Re-stamps a member's own order into a key whose top fields a SortingGroup
* has taken over.
*
* @details [37:30] is the only field directly under stage in ALL three packings, and
* it must be one field or two members of a group are incomparable. Costs
* those bits' projection (depth, shader, worldY) inside a group only.
*/
static u64 withGroupOrder(u64 key, i32 memberOrder) {
constexpr u64 field = 0xFFull << 30;
return (key & ~field) | (static_cast<u64>(std::clamp(memberOrder + 128, 0, 255)) << 30);
}
// Order-preserving float → u32: flip the sign bit for positives, all bits for
// negatives, so unsigned compare matches float compare (finite values).
static u32 orderedFloatBits(f32 v) {
u32 bits = std::bit_cast<u32>(v);
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
// Y-sorted variant (top-down occlusion): higher Y (further "back" under Y-up) draws
// first, so lower-on-screen entities land on top. Material and depth leave the key;
// shader/blend/flags stay as tie-breaks so equal-Y draws group for the merge.
// Same top three fields as buildSortKey, so a y-sorted layer and a plain one order
// correctly against each other; below them this spends its bits on worldY instead.
// Order outranks worldY because y-sort INFERS an order from position and an author
// stating one is overriding that inference. Without it a y-sorted layer has no
// override at all: z never reaches this key, so a shadow cannot be pinned under its owner.
// [63:48] layer | [47:40] order | [39:38] stage | [37:14] worldY
// [13:6] shader | [5:3] blend | [2:1] flags
static u64 buildSortKeyYSorted(RenderStage stage, i32 layer, f32 worldY,
u32 shaderId, BlendMode blend, u16 stateFlags,
i32 order = 0) {
i32 normalizedLayer = std::clamp(layer + 32768, 0, 65535);
u64 layerKey = static_cast<u64>(normalizedLayer & 0xFFFF) << 48;
u64 stageKey = static_cast<u64>(stage) << 38;
u32 yDescending = (~orderedFloatBits(worldY)) >> 8; // 24 bits, larger Y → smaller key
u64 yKey = static_cast<u64>(yDescending & 0xFFFFFF) << 14;
u64 shaderKey = static_cast<u64>(shaderId & 0xFF) << 6;
u64 blendKey = blendBits(blend) << 3;
u64 flagsKey = static_cast<u64>(stateFlags & 0x03) << 1;
return stageKey | layerKey | orderBits(order) | yKey | shaderKey | blendKey | flagsKey;
}
// Tree-ordered variant: the entity's place in the hierarchy walk takes the field worldY has in a y-sorted layer, so draws of every kind
// stack as their nodes do, and shader/blend/flags only separate what shares a node.
static u64 buildSortKeyTree(RenderStage stage, i32 layer, u32 sequence,
u32 shaderId, BlendMode blend, u16 stateFlags,
i32 order = 0) {
i32 normalizedLayer = std::clamp(layer + 32768, 0, 65535);
u64 layerKey = static_cast<u64>(normalizedLayer & 0xFFFF) << 48;
u64 stageKey = static_cast<u64>(stage) << 38;
u64 sequenceKey = static_cast<u64>(sequence & 0xFFFFFF) << 14;
u64 shaderKey = static_cast<u64>(shaderId & 0xFF) << 6;
u64 blendKey = blendBits(blend) << 3;
u64 flagsKey = static_cast<u64>(stateFlags & 0x03) << 1;
return stageKey | layerKey | orderBits(order) | sequenceKey | shaderKey | blendKey | flagsKey;
}
/** @brief Finds @p texId in this command's texture set, adds it (returns its slot), or
* -1 if the set is full. Used by the merge to assign per-vertex sampler slots.
* @p limit is how many slots the merge may spend: a frame with a 2D shadow
* mask keeps the top one for it, and a merge that took it would bind a
* sprite where every Lit fragment reads its shadow. */
/**
* How many sprite textures a merged run headed by this draw may carry.
*
* The Batch stream merges draws of different textures by giving each its own
* sampler slot and stamping the slot on its vertices — which only the engine's
* own batch program reads. A material's shader samples the sprite at slot 0
* and nothing else (`u_textures[0]` / `t0`), so a run of material draws that
* merged across textures drew every sprite with the first one's picture.
* Such a run keeps to one texture; a second one starts a new draw.
*/
u8 textureSlotBudget(u8 reservedSlots) const {
if (material_id != 0) return 1;
return static_cast<u8>(MAX_CMD_TEXTURE_SLOTS - reservedSlots);
}
i32 addTextureSlot(u32 texId, u8 limit = MAX_CMD_TEXTURE_SLOTS) {
for (u8 i = 0; i < texture_count; ++i) {
if (texture_ids[i] == texId) return static_cast<i32>(i);
}
if (texture_count >= limit) return -1;
texture_ids[texture_count] = texId;
return static_cast<i32>(texture_count++);
}
/**
* Why `next` cannot join this draw, or None if it can.
*
* The predicate answers with the reason rather than a bare no: a frame is
* some number of draw calls, and the only actionable thing about that number
* is what kept them apart. canMergeWith is this, read as a bool.
*/
/**
* Whether this draw's result is the same wherever it happens among its
* peers: it replaces what it covers rather than reading it, and depth
* decides what covers what. Only such draws can be merged into one — a
* merge is exactly the loss of their relative order.
*/
bool orderIndependent() const {
return blend_mode == BlendMode::None && depth_test && depth_write;
}
/**
* Whether two instanced draws are the same geometry drawn again — what a
* second instance record answers, rather than a second draw. Both must be
* order-independent, since merging is the loss of their relative order; a
* skinned draw is out because one draw can only be in one pose.
*/
bool sameInstancedGeometry(const DrawCommand& next) const {
if (!hasPersistentGeometry() || !next.hasPersistentGeometry()) return false;
if (instance_stride == 0 || instance_stride != next.instance_stride) return false;
if (skin_count != 0 || next.skin_count != 0) return false;
if (planar_index != next.planar_index) return false;
// Same reason: one draw can only be in one shape.
if (morph_index != 0 || next.morph_index != 0) return false;
// Indirect light is NOT in this list: it is per instance, so two draws
// standing in different places are one geometry drawn twice.
// BOTH, not just this one: an opaque head asked about a draw whose
// result depends on when it happens would otherwise swallow it and
// paint it with the head's own state.
if (!orderIndependent() || !next.orderIndependent()) return false;
if (vertex_buffer != next.vertex_buffer || index_buffer != next.index_buffer) return false;
if (vertex_layout != next.vertex_layout) return false;
// The same range of the same buffer: a draw is one instance of THIS.
if (index_offset != next.index_offset || index_count != next.index_count) return false;
// Textures are per draw here, not selected per vertex the way the Batch
// stream selects them, so instances sharing one draw share its samplers.
if (texture_count != next.texture_count) return false;
for (u8 s = 0; s < texture_count; ++s) {
if (texture_ids[s] != next.texture_ids[s]) return false;
}
return true;
}
BatchBreak mergeBlocker(const DrawCommand& next) const {
if (planar_index != next.planar_index) return BatchBreak::Material;
// UI mask vertices are replayed to restore their parent's stencil.
// Changing sampler indices while merging the entering draw would also
// change its later replay, so both must keep the same binding plan.
if ((state_flags & (CMD_STATE_UI_STENCIL | CMD_STATE_STENCIL_WRITE))
== (CMD_STATE_UI_STENCIL | CMD_STATE_STENCIL_WRITE)) {
if (texture_count != next.texture_count
|| !std::equal(texture_ids, texture_ids + texture_count, next.texture_ids)) {
return BatchBreak::TextureSlots;
}
}
// An instanced draw is one command per emitter, except where a second
// command is the same geometry again — see sameInstancedGeometry, which
// the merge answers by relocating the records rather than by a draw.
if (instance_count != 0 || next.instance_count != 0) {
// A run carrying irradiance is bounded by what one block holds; one
// standing in no volume carries none and is not bounded.
if ((probe_index != 0 || next.probe_index != 0)
&& instance_count + next.instance_count > PROBE_MAX_INSTANCES) {
return BatchBreak::Instanced;
}
if (!sameInstancedGeometry(next)) return BatchBreak::Instanced;
if (shader_id != next.shader_id || depth_shader_id != next.depth_shader_id) return BatchBreak::Shader;
if (material_id != next.material_id) return BatchBreak::Material;
if (cull != next.cull) return BatchBreak::Cull;
if (depth_bias != next.depth_bias) return BatchBreak::Cull;
if (topology != next.topology) return BatchBreak::Layout;
if (state_flags != next.state_flags) return BatchBreak::State;
if (state_flags & CMD_STATE_SCISSOR) {
if (scissor != next.scissor) return BatchBreak::Scissor;
}
if (state_flags & (CMD_STATE_STENCIL_WRITE | CMD_STATE_STENCIL_TEST)) {
if (stencil_ref != next.stencil_ref) return BatchBreak::Stencil;
}
return BatchBreak::None;
}
if (shader_id != next.shader_id || depth_shader_id != next.depth_shader_id) return BatchBreak::Shader;
if (blend_mode != next.blend_mode) return BatchBreak::Blend;
if (layout_id != next.layout_id || vertex_buffer != next.vertex_buffer
|| index_buffer != next.index_buffer || vertex_layout != next.vertex_layout) return BatchBreak::Layout;
// Same material handle => same uniforms/textures; different ones must not
// coalesce. material_id 0 (no material) shares the path's defaults, so they still merge.
if (material_id != next.material_id) return BatchBreak::Material;
// Depth state is checked on its own rather than trusted to follow from the
// material. It used to: every draw resolved its depth from a material, so equal
// material_id implied equal state, and material_id 0 meant one shared set of
// defaults. Once a layer derives depth from the stage instead, two material-0
// draws in the same layer can differ — one opaque and writing depth, one blended
// and not — and merging them would silently give one of them the other's state.
// The symptom would be a blended sprite occasionally clipping what is behind it,
// depending on whether the two happened to land adjacent. Compared here as render
// state, not as stage: the stage is the reason, these three are the effect.
if (depth_test != next.depth_test) return BatchBreak::Depth;
if (depth_write != next.depth_write) return BatchBreak::Depth;
if (cull != next.cull) return BatchBreak::Cull;
if (depth_bias != next.depth_bias) return BatchBreak::Cull;
if (topology != next.topology) return BatchBreak::Layout;
if (state_flags != next.state_flags) return BatchBreak::State;
if (state_flags & CMD_STATE_SCISSOR) {
if (scissor != next.scissor) return BatchBreak::Scissor;
}
if (state_flags & (CMD_STATE_STENCIL_WRITE | CMD_STATE_STENCIL_TEST)) {
if (stencil_ref != next.stencil_ref) return BatchBreak::Stencil;
}
// Texture compatibility is decided by the merge (the combined set must fit in 8
// slots), not here, so different-texture draws can coalesce.
if (index_offset + index_count != next.index_offset) return BatchBreak::IndexGap;
return BatchBreak::None;
}
bool canMergeWith(const DrawCommand& next) const {
if (planar_index != next.planar_index) return false;
return mergeBlocker(next) == BatchBreak::None;
}
};
} // namespace esengine