Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

refactor(render): a sorting layer outranks the stage inside it

esengine8deaa065d0f1
diff --git a/src/esengine/renderer/draw/BatchBuilder.cpp b/src/esengine/renderer/draw/BatchBuilder.cpp
index 33b468b4e..64e6656e1 100644
--- a/src/esengine/renderer/draw/BatchBuilder.cpp
+++ b/src/esengine/renderer/draw/BatchBuilder.cpp
@@ -56,6 +56,7 @@ void pushBatchDraw(DrawList& drawList, const ClipState& clips,
key.blend, 0)
: DrawCommand::buildSortKey(key.stage, key.layer, key.shaderId,
key.blend, 0, key.depth, key.materialId);
+ cmd.stage = key.stage;
cmd.index_offset = indexOffset;
cmd.index_count = indexCount;
cmd.vertex_byte_offset = vertexByteOffset;
diff --git a/src/esengine/renderer/draw/DrawCommand.hpp b/src/esengine/renderer/draw/DrawCommand.hpp
index 613222eb9..a33912534 100644
--- a/src/esengine/renderer/draw/DrawCommand.hpp
+++ b/src/esengine/renderer/draw/DrawCommand.hpp
@@ -50,6 +50,10 @@ struct DrawCommand {
Entity entity = INVALID_ENTITY;
RenderType type = RenderType::Sprite;
+ // Kept beside the sort key rather than unpacked from it: the bit layout is
+ // buildSortKey's business alone, and a second place that knows it is a second
+ // place to forget when it moves. `layer` is here for the same reason.
+ RenderStage stage = RenderStage::Transparent;
i32 layer = 0;
u32 entity_count = 1;
bool merged = false;
@@ -67,12 +71,23 @@ struct DrawCommand {
// selected per-vertex in the shader). Order within a layer is otherwise unchanged.
// Material identity sorts above depth (like shader does) so same-material draws group
// adjacent for the merge, at the usual batching-over-cross-material-depth tradeoff.
+ //
+ // Layer outranks stage, and that order is load-bearing. A sorting layer is a promise
+ // the user made about what draws on top of what; a stage is how one layer resolves
+ // its own contents. Ranking stage first — the classic 3D pipeline order, where every
+ // opaque draw precedes every transparent one — would let an opaque draw in layer 5
+ // jump ahead of a painter draw in layer 3, which is the sorting layer's entire
+ // meaning inverted. Within a layer the 3D order is the right one: opaque first
+ // (front-to-back, early-z), then blended (back-to-front).
+ //
+ // [63:48] layer | [47:44] stage | [43:36] shader | [35:33] blend
+ // [32:31] flags | [30:14] material | [13:0] depth
static u64 buildSortKey(RenderStage stage, i32 layer, u32 shaderId,
BlendMode blend, u16 stateFlags, f32 depth, u32 materialId = 0) {
- u64 stageKey = static_cast<u64>(stage) << 60;
-
i32 normalizedLayer = std::clamp(layer + 32768, 0, 65535);
- u64 layerKey = static_cast<u64>(normalizedLayer & 0xFFFF) << 44;
+ u64 layerKey = static_cast<u64>(normalizedLayer & 0xFFFF) << 48;
+
+ u64 stageKey = static_cast<u64>(stage) << 44;
u64 shaderKey = static_cast<u64>(shaderId & 0xFF) << 36;
u64 blendKey = static_cast<u64>(blend) << 33;
@@ -108,12 +123,17 @@ struct DrawCommand {
// first, so lower-on-screen entities land on top. Material/depth leave the key
// (Y-order beats batching by design; adjacent same-state runs still merge in
// finalize), shader/blend/flags remain as tie-breaks so equal-Y draws group.
+ // Same top two fields as buildSortKey, so a y-sorted layer and a plain one order
+ // correctly against each other; below them this spends its bits on worldY instead.
+ //
+ // [63:48] layer | [47:44] stage | [43:20] worldY | [19:12] shader
+ // [11:9] blend | [8:7] flags
static u64 buildSortKeyYSorted(RenderStage stage, i32 layer, f32 worldY,
u32 shaderId, BlendMode blend, u16 stateFlags) {
- u64 stageKey = static_cast<u64>(stage) << 60;
-
i32 normalizedLayer = std::clamp(layer + 32768, 0, 65535);
- u64 layerKey = static_cast<u64>(normalizedLayer & 0xFFFF) << 44;
+ u64 layerKey = static_cast<u64>(normalizedLayer & 0xFFFF) << 48;
+
+ u64 stageKey = static_cast<u64>(stage) << 44;
u32 yDescending = (~orderedFloatBits(worldY)) >> 8; // 24 bits, larger Y → smaller key
u64 yKey = static_cast<u64>(yDescending & 0xFFFFFF) << 20;
diff --git a/src/esengine/renderer/draw/DrawList.cpp b/src/esengine/renderer/draw/DrawList.cpp
index 67fc69aab..eaad4c905 100644
--- a/src/esengine/renderer/draw/DrawList.cpp
+++ b/src/esengine/renderer/draw/DrawList.cpp
@@ -181,7 +181,7 @@ void DrawList::execute(GfxDevice& device, TransientBufferPool& buffers,
if (capture && capture->isCapturing()) {
capture->recordDrawCall(
- static_cast<RenderStage>(cmd.sort_key >> 60),
+ cmd.stage,
cmd.type, cmd.blend_mode,
cmd.texture_count > 0 ? cmd.texture_ids[0] : 0,
0, cmd.shader_id,
diff --git a/tests/renderer/test_batch_builder.cpp b/tests/renderer/test_batch_builder.cpp
index 36aaad9e9..678de562e 100644
--- a/tests/renderer/test_batch_builder.cpp
+++ b/tests/renderer/test_batch_builder.cpp
@@ -212,3 +212,44 @@ TEST_CASE("execute: indexed and instanced commands dispatch through the device")
CHECK(h.device.lastDrawInstanceCount == 5);
CHECK(h.device.setPipelineCalls == 2);
}
+
+// The sort key ranks layer above stage, and both of these pin that down. Swapping the
+// two fields back (stage on top, the classic 3D pipeline order) flips the first check
+// and leaves the second passing — which is exactly the failure the old layout had:
+// correct inside a layer, and silently wrong across layers. Every other case in this
+// file submits Transparent only, so neither would have been caught here.
+TEST_CASE("sort order: a lower sorting layer draws first even when a higher one is opaque") {
+ Harness h;
+ BatchVertex quad[4] = {};
+
+ BatchDrawKey opaqueAbove = quadKey(50, /*layer=*/5);
+ opaqueAbove.stage = RenderStage::Opaque;
+ // A different shader only keeps the two from coalescing, so their order stays
+ // observable; shader ranks below both fields under test.
+ opaqueAbove.shaderId = 8;
+ appendQuad(h.pool, h.list, h.clips, quad, opaqueAbove);
+ appendQuad(h.pool, h.list, h.clips, quad, quadKey(51, /*layer=*/3));
+
+ h.list.finalize(h.pool);
+
+ REQUIRE(h.list.mergedDrawCallCount() == 2);
+ CHECK(h.list.command(0).texture_ids[0] == 51);
+ CHECK(h.list.command(1).texture_ids[0] == 50);
+}
+
+TEST_CASE("sort order: inside one layer, opaque draws before blended") {
+ Harness h;
+ BatchVertex quad[4] = {};
+
+ appendQuad(h.pool, h.list, h.clips, quad, quadKey(60, /*layer=*/4));
+ BatchDrawKey opaque = quadKey(61, /*layer=*/4);
+ opaque.stage = RenderStage::Opaque;
+ opaque.shaderId = 8;
+ appendQuad(h.pool, h.list, h.clips, quad, opaque);
+
+ h.list.finalize(h.pool);
+
+ REQUIRE(h.list.mergedDrawCallCount() == 2);
+ CHECK(h.list.command(0).texture_ids[0] == 61);
+ CHECK(h.list.command(1).texture_ids[0] == 60);
+}