Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

fix(editor): a click selects the sprite you can see, not the one behind it

yhh2056626bde48
diff --git a/desktop/src/engine/ViewportController.ts b/desktop/src/engine/ViewportController.ts
index 70196de58..753acdbf3 100644
--- a/desktop/src/engine/ViewportController.ts
+++ b/desktop/src/engine/ViewportController.ts
@@ -11,11 +11,12 @@ import {
TilemapLayer, TilemapAPI, decodeTilemapChunks, CHUNK_SIZE, tileCollisionOutlines,
tileCellCenter, tileCellOutline, isNonOrthogonal,
readColliderShapes, colliderShapeOutline, shapeCenter,
+ layerOrderOf,
type TilesetModel, type TileCollisionPiece, type TileGridParams,
} from 'esengine';
import type { EntityId } from '@/types';
import { EngineHost } from './EngineHost';
-import { projectDesignSeed, projectCameraFit } from './projectSeams';
+import { projectDesignSeed, projectCameraFit, projectSortingLayerModes } from './projectSeams';
import { SceneModel } from './SceneModel';
import {
type OBB,
@@ -27,6 +28,8 @@ import {
screenAABB,
clamp,
worldToLocal2D,
+ rankPickCandidates,
+ type PickCandidate,
} from './viewportMath';
// World half-size of the pick/outline box for entities without renderable bounds
@@ -258,15 +261,24 @@ export const ViewportController = {
return this.pickUIEntities(clientX, clientY)[0] ?? null;
},
- /** Selectable entities under the pointer, topmost-first (UI, then world by
- * descending layer, later-drawn winning ties). `pickEntity` is its head;
- * click-through cycling walks the rest. */
+ /**
+ * Selectable entities under the pointer, topmost-first: UI, then world entities
+ * ranked the way the RENDERER stacked them — descending sorting layer, then
+ * whatever is in front inside that layer, then later-drawn winning ties.
+ *
+ * "In front" is the layer's own rule (y-sort / depth / painter), read from the
+ * same two project masks the renderer sorts by, because a click that ranks
+ * overlapping entities any other way selects something the person cannot see.
+ * It used to rank on list order alone, so a sprite drawn on top by its z lost
+ * the click to whatever happened to be created after it.
+ */
pickEntitiesAt(clientX: number, clientY: number): EntityId[] {
const out: EntityId[] = [...this.pickUIEntities(clientX, clientY)];
const world = EngineHost.world;
if (world) {
- const hits: { e: EntityId; layer: number; i: number }[] = [];
+ const masks = projectSortingLayerModes();
+ const hits: PickCandidate<EntityId>[] = [];
// The cursor is tested against each candidate ON ITS OWN PLANE. One shared
// world point would be the 2D projection of the cursor, which under a
// perspective view is where a sprite's shadow falls rather than where the
@@ -282,10 +294,19 @@ export const ViewportController = {
const b = this.entityBounds(e);
if (!wp || !b || !pointInOBB(wp.x, wp.y, b)) continue;
const layer = world.has(e, Sprite) ? world.get(e, Sprite).layer : ICON_PICK_LAYER;
- hits.push({ e, layer, i: hits.length });
+ const t = world.get(e, Transform);
+ hits.push({
+ entity: e,
+ index: hits.length,
+ rank: {
+ layer,
+ order: layerOrderOf(layer, masks.ySort, masks.depth),
+ worldY: t.worldPosition.y,
+ worldZ: t.worldPosition.z,
+ },
+ });
}
- hits.sort((a, b) => b.layer - a.layer || b.i - a.i);
- for (const h of hits) out.push(h.e);
+ for (const e of rankPickCandidates(hits)) out.push(e);
}
return out;
},
diff --git a/desktop/src/engine/projectSeams.ts b/desktop/src/engine/projectSeams.ts
index 201b977c6..9f9a13bd5 100644
--- a/desktop/src/engine/projectSeams.ts
+++ b/desktop/src/engine/projectSeams.ts
@@ -46,6 +46,23 @@ export function projectCameraFit(): { scaleMode: number; matchWidthOrHeight: num
return projectCameraFitSeed?.() ?? { scaleMode: -1, matchWidthOrHeight: 0.5 };
}
+// -- How each sorting layer resolves its contents ------------------------------
+// The two bitmasks the renderer sorts by (y-sort, depth). Picking has to rank
+// overlapping entities the way the frame was drawn — a click that ranks them any
+// other way selects something the person cannot see — and the ranking runs in
+// the viewport, below the store that owns the project setting.
+
+let sortingLayerModesSeed: (() => { ySort: number; depth: number }) | null = null;
+
+export function setProjectSortingLayerModes(fn: () => { ySort: number; depth: number }): void {
+ sortingLayerModesSeed = fn;
+}
+
+/** Both masks; all-painter (0, 0) with no project — the pre-2.5D default. */
+export function projectSortingLayerModes(): { ySort: number; depth: number } {
+ return sortingLayerModesSeed?.() ?? { ySort: 0, depth: 0 };
+}
+
// -- The project's own prefabs, as entity sources ------------------------------
// Injected by the store, which owns the asset registry; they join the one list
// every "create an entity" surface offers (see allEntitySources).
diff --git a/desktop/src/engine/viewportMath.ts b/desktop/src/engine/viewportMath.ts
index 9d798659e..517da984c 100644
--- a/desktop/src/engine/viewportMath.ts
+++ b/desktop/src/engine/viewportMath.ts
@@ -2,10 +2,13 @@
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
* @file viewportMath.ts
- * @brief Pure 2D geometry for viewport picking, marquee, and gizmos — no engine /
- * DOM coupling, so it unit-tests in isolation. The imperative shells
+ * @brief Pure 2D geometry for viewport picking, marquee, and gizmos — no engine
+ * state or DOM, so it unit-tests in isolation. The imperative shells
* (ViewportController picking, the gizmo tools) layer on top of these.
+ * The one import is the SDK's pure mirror of the renderer's layer rules,
+ * which picking has to rank by rather than restate.
*/
+import { compareDrawRank, type DrawRank } from 'esengine';
/** Oriented bounding box in world space: center, half-extents, Z rotation (radians). */
export interface OBB {
@@ -93,3 +96,27 @@ export const clamp = (v: number, lo: number, hi: number): number => Math.max(lo,
/** Round to the nearest multiple of `step` (snap). `step <= 0` returns `v` unchanged. */
export const snapTo = (v: number, step: number): number => (step > 0 ? Math.round(v / step) * step : v);
+
+/** One entity under the pointer, with everything its rank depends on. */
+export interface PickCandidate<T> {
+ entity: T;
+ /** Where the frame put it: layer, that layer's rule, and the world coords it uses. */
+ rank: DrawRank;
+ /** Position in the World's iteration order: the paint order for equal depth. */
+ index: number;
+}
+
+/**
+ * Candidates ranked topmost-first, the way the RENDERER stacked them — the layer
+ * rules via {@link compareDrawRank}, then later-drawn winning ties.
+ *
+ * Split out as a pure function because it is the part that can be wrong while
+ * every hit test is right — and it ranks what a person SEES, so getting it
+ * backwards means the click selects the thing hidden behind the thing they
+ * aimed at.
+ */
+export function rankPickCandidates<T>(candidates: ReadonlyArray<PickCandidate<T>>): T[] {
+ return [...candidates]
+ .sort((a, b) => compareDrawRank(b.rank, a.rank) || b.index - a.index)
+ .map((c) => c.entity);
+}
diff --git a/desktop/tests/viewport-math.test.ts b/desktop/tests/viewport-math.test.ts
index 93dcedc45..f952d7057 100644
--- a/desktop/tests/viewport-math.test.ts
+++ b/desktop/tests/viewport-math.test.ts
@@ -16,7 +16,10 @@ import {
snapTo,
clamp,
worldToLocal2D,
+ rankPickCandidates,
+ type PickCandidate,
} from '@/engine/viewportMath';
+import { LayerOrder, layerOrderOf } from 'esengine';
const box = (cx: number, cy: number, hw: number, hh: number, rot = 0): OBB => ({ cx, cy, hw, hh, rot });
@@ -124,3 +127,60 @@ describe('worldToLocal2D', () => {
expect(p.y).toBe(3);
});
});
+
+describe('rankPickCandidates', () => {
+ // Sprites listed near → far, so list order and depth order disagree on purpose.
+ // This is the exact shape that used to select the sprite BEHIND the cursor.
+ const cand = (entity: string, layer: number, z: number, index: number, ySort = 0, depth = 0, y = 0)
+ : PickCandidate<string> => ({
+ entity, index,
+ rank: { layer, order: layerOrderOf(layer, ySort, depth), worldY: y, worldZ: z },
+ });
+ const DEPTH12 = (1 << 1) | (1 << 2);
+
+ it('ranks the nearest first in a depth layer, whatever the list order', () => {
+ const near = cand('near', 1, 150, 0, 0, DEPTH12);
+ const mid = cand('mid', 1, 0, 1, 0, DEPTH12);
+ const far = cand('far', 1, -150, 2, 0, DEPTH12);
+ expect(rankPickCandidates([near, mid, far])).toEqual(['near', 'mid', 'far']);
+ expect(rankPickCandidates([far, mid, near])).toEqual(['near', 'mid', 'far']);
+ });
+
+ // The depth buffer resolves per pixel, so it beats the sorting layer — the very
+ // case the paired render check pins (near sprite in the LOWER layer wins).
+ it('lets a nearer sprite in a LOWER depth layer win the click', () => {
+ const near = cand('near', 1, 150, 0, 0, DEPTH12);
+ const far = cand('far', 2, -150, 1, 0, DEPTH12);
+ expect(rankPickCandidates([near, far])).toEqual(['near', 'far']);
+ });
+
+ // ... but only between two depth layers. A painter layer does not depth-test, so
+ // it is simply painted later and covers what is under it.
+ it('keeps the sorting layer in charge when a layer is not depth-resolved', () => {
+ const nearDepth = cand('nearDepth', 1, 150, 0, 0, 1 << 1);
+ const painterAbove = cand('painterAbove', 5, -900, 1);
+ expect(rankPickCandidates([nearDepth, painterAbove])).toEqual(['painterAbove', 'nearDepth']);
+ });
+
+ it('ranks by z inside a plain painter layer too — the renderer sorts on it', () => {
+ const near = cand('near', 1, 150, 0);
+ const mid = cand('mid', 1, 0, 1);
+ const far = cand('far', 1, -150, 2);
+ expect(rankPickCandidates([far, near, mid])).toEqual(['near', 'mid', 'far']);
+ });
+
+ it('ranks lower world Y first in a y-sorted layer, ignoring z', () => {
+ const top = cand('top', 1, 900, 0, 1 << 1, 0, 100);
+ const bottom = cand('bottom', 1, -900, 1, 1 << 1, 0, -100);
+ expect(rankPickCandidates([top, bottom])).toEqual(['bottom', 'top']);
+ });
+
+ it('falls back to later-drawn at equal depth, as paint order does', () => {
+ expect(rankPickCandidates([cand('a', 0, 0, 0), cand('b', 0, 0, 1)])).toEqual(['b', 'a']);
+ });
+
+ it('leaves an empty hit list empty', () => {
+ expect(rankPickCandidates([])).toEqual([]);
+ expect(LayerOrder.Painter).toBe(0);
+ });
+});
diff --git a/sdk/src/core-render.ts b/sdk/src/core-render.ts
index e1a69b850..e1317b275 100644
--- a/sdk/src/core-render.ts
+++ b/sdk/src/core-render.ts
@@ -40,7 +40,7 @@ export { TextureFilter, TextureWrap, setTextureFilter, setTextureWrap, setTextur
// =============================================================================
export { CameraView, CameraViewAPI } from './camera/Camera';
-export { EditorView, DEFAULT_EDITOR_VIEW } from './camera/EditorView';
+export { EditorView, DEFAULT_EDITOR_VIEW, editorViewHalfHeight, editorViewHalfExtent, setEditorViewHalfHeight } from './camera/EditorView';
export type { EditorViewData } from './camera/EditorView';
export { ScreenScaling, DEFAULT_SCREEN_SCALING, SCREEN_FIT_OFF } from './camera/ScreenScaling';
export type { ScreenScalingData } from './camera/ScreenScaling';
@@ -150,6 +150,14 @@ export {
type RenderStats,
} from './render/renderer';
+export {
+ LayerOrder,
+ layerOrderOf,
+ layerFrontness,
+ compareDrawRank,
+ type DrawRank,
+} from './render/layerOrder';
+
export {
FlushReason,
RenderType,
diff --git a/sdk/src/render/layerOrder.ts b/sdk/src/render/layerOrder.ts
new file mode 100644
index 000000000..ef7417be4
--- /dev/null
+++ b/sdk/src/render/layerOrder.ts
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file layerOrder.ts
+ * @brief How a sorting layer resolves the draws inside it — the JS mirror of
+ * the engine's `DrawList::layerOrder` (src/esengine/renderer/draw).
+ *
+ * The renderer decides this from two bitmasks and nothing else. Anything that has
+ * to AGREE with what was drawn — the editor's picking, a gizmo that ranks
+ * overlapping entities, a future runtime hit-test — has to ask the same question
+ * of the same two masks, or it answers about a different image than the one on
+ * screen. One rule, mirrored once, rather than each caller's idea of "on top".
+ */
+
+/** How one sorting layer resolves the draws inside it. */
+export enum LayerOrder {
+ /** Submission order, tie-broken by depth: the 2D default. */
+ Painter = 0,
+ /** World Y within the layer — lower on screen draws on top. */
+ YSort = 1,
+ /** The depth buffer, from each entity's z — the 2.5D opt-in. */
+ Depth = 2,
+}
+
+/**
+ * The one answer for @p layer, from two masks that are two spellings of the same
+ * question. Y-sort is a depth PROJECTED from world Y and depth is the real thing,
+ * so a layer declaring both has said two contradictory things: y-sort wins, which
+ * is what the engine does, so a project that had it renders as it did. Layers
+ * outside 0..31 have no bit and are therefore painter-ordered.
+ */
+export function layerOrderOf(layer: number, ySortMask: number, depthMask: number): LayerOrder {
+ if (!Number.isInteger(layer) || layer < 0 || layer >= 32) return LayerOrder.Painter;
+ const bit = 1 << layer;
+ if ((ySortMask & bit) !== 0) return LayerOrder.YSort;
+ if ((depthMask & bit) !== 0) return LayerOrder.Depth;
+ return LayerOrder.Painter;
+}
+
+/**
+ * How far in FRONT a draw sits within its layer — larger is nearer the viewer,
+ * so ranking by this descending is "topmost first".
+ *
+ * The camera looks down -z, so a larger z is nearer; under y-sort the projected
+ * depth is world Y inverted (lower on screen = nearer). Painter layers still
+ * answer by z because the engine's sort key tie-breaks on it, and a scene that
+ * puts one sprite in front of another with z means it — this is exactly the
+ * ordering a person sees.
+ */
+export function layerFrontness(order: LayerOrder, worldY: number, worldZ: number): number {
+ return order === LayerOrder.YSort ? -worldY : worldZ;
+}
+
+/** What a draw's place in the frame depends on: its layer, that layer's rule, and
+ * the two world coordinates either rule can be about. */
+export interface DrawRank {
+ layer: number;
+ order: LayerOrder;
+ worldY: number;
+ worldZ: number;
+}
+
+/**
+ * Positive when @p a ended up in FRONT of @p b — the order the frame resolved,
+ * for anything that has to answer "which of these is on top" outside the renderer.
+ *
+ * The sorting layer normally decides, because a higher layer is simply painted
+ * later. Two DEPTH layers are the exception, and not a special case bolted on: a
+ * depth layer resolves per PIXEL, so an opaque draw at a nearer z occludes what a
+ * later layer puts over it — that is the whole point of the 2.5D opt-in, and it is
+ * what a paired pixel check proves (near sprite in the LOWER layer, still in
+ * front). Ranking those two by layer would answer with the sprite that the depth
+ * buffer rejected.
+ */
+export function compareDrawRank(a: DrawRank, b: DrawRank): number {
+ if (a.order === LayerOrder.Depth && b.order === LayerOrder.Depth) return a.worldZ - b.worldZ;
+ if (a.layer !== b.layer) return a.layer - b.layer;
+ return layerFrontness(a.order, a.worldY, a.worldZ) - layerFrontness(b.order, b.worldY, b.worldZ);
+}
diff --git a/sdk/tests/layer-order.test.ts b/sdk/tests/layer-order.test.ts
new file mode 100644
index 000000000..b0f2128b6
--- /dev/null
+++ b/sdk/tests/layer-order.test.ts
@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file layer-order.test.ts — the JS mirror of DrawList::layerOrder.
+ *
+ * These pin the two things a mirror can get wrong and look fine: the precedence
+ * when a layer declares both masks, and which DIRECTION counts as "in front".
+ * A frontness with the sign flipped ranks a pick exactly backwards, and every
+ * individual answer still looks plausible.
+ */
+import { describe, it, expect } from 'vitest';
+import { LayerOrder, layerOrderOf, layerFrontness, compareDrawRank } from '../src/render/layerOrder';
+
+describe('layerOrderOf', () => {
+ it('is painter order when neither mask claims the layer', () => {
+ expect(layerOrderOf(0, 0, 0)).toBe(LayerOrder.Painter);
+ expect(layerOrderOf(5, 1 << 4, 1 << 6)).toBe(LayerOrder.Painter);
+ });
+
+ it('reads the bit for the layer, not the layer number', () => {
+ expect(layerOrderOf(3, 1 << 3, 0)).toBe(LayerOrder.YSort);
+ expect(layerOrderOf(3, 0, 1 << 3)).toBe(LayerOrder.Depth);
+ expect(layerOrderOf(31, 0, 1 << 31 >>> 0)).toBe(LayerOrder.Depth);
+ });
+
+ // The engine resolves the contradiction one way and the editor has to resolve
+ // it the same way, or a doubly-declared layer picks against what it drew.
+ it('gives y-sort precedence when a layer declares both', () => {
+ expect(layerOrderOf(2, 1 << 2, 1 << 2)).toBe(LayerOrder.YSort);
+ });
+
+ it('treats layers outside 0..31 as painter-ordered', () => {
+ expect(layerOrderOf(-1, 0xffffffff, 0xffffffff)).toBe(LayerOrder.Painter);
+ expect(layerOrderOf(32, 0xffffffff, 0xffffffff)).toBe(LayerOrder.Painter);
+ expect(layerOrderOf(1e6, 0xffffffff, 0xffffffff)).toBe(LayerOrder.Painter);
+ });
+});
+
+describe('layerFrontness', () => {
+ // Camera looks down -z, so larger z is nearer — the same statement the engine's
+ // sort key makes when it orders the transparent stage back-to-front.
+ it('ranks larger z in front for painter and depth layers', () => {
+ for (const order of [LayerOrder.Painter, LayerOrder.Depth]) {
+ expect(layerFrontness(order, 0, 150)).toBeGreaterThan(layerFrontness(order, 0, -150));
+ expect(layerFrontness(order, 999, 5)).toBe(5); // y is not consulted
+ }
+ });
+
+ // Y-up world, top-down game: lower on screen is nearer the viewer.
+ it('ranks lower world Y in front for y-sorted layers', () => {
+ expect(layerFrontness(LayerOrder.YSort, -100, 0))
+ .toBeGreaterThan(layerFrontness(LayerOrder.YSort, 100, 0));
+ expect(layerFrontness(LayerOrder.YSort, 10, 999)).toBe(-10); // z is not consulted
+ });
+});
+
+describe('compareDrawRank', () => {
+ const rank = (layer: number, order: LayerOrder, worldZ = 0, worldY = 0) =>
+ ({ layer, order, worldY, worldZ });
+
+ it('gives the higher sorting layer the front, as paint order does', () => {
+ expect(compareDrawRank(rank(5, LayerOrder.Painter), rank(1, LayerOrder.Painter))).toBeGreaterThan(0);
+ expect(compareDrawRank(rank(0, LayerOrder.YSort), rank(3, LayerOrder.YSort))).toBeLessThan(0);
+ });
+
+ // The 2.5D claim itself: a depth layer resolves per pixel, so a nearer opaque
+ // draw occludes what a LATER layer puts over it. Ranking these by layer would
+ // name the sprite the depth buffer just rejected.
+ it('lets depth beat the sorting layer — but only between two depth layers', () => {
+ const nearLow = rank(1, LayerOrder.Depth, 150);
+ const farHigh = rank(2, LayerOrder.Depth, -150);
+ expect(compareDrawRank(nearLow, farHigh)).toBeGreaterThan(0);
+
+ // One side painter-ordered: no depth test happens, so the later layer covers.
+ expect(compareDrawRank(rank(1, LayerOrder.Depth, 150), rank(2, LayerOrder.Painter, -900)))
+ .toBeLessThan(0);
+ });
+
+ it('falls back to the layer rule inside one layer', () => {
+ expect(compareDrawRank(rank(1, LayerOrder.Painter, 10), rank(1, LayerOrder.Painter, -10)))
+ .toBeGreaterThan(0);
+ expect(compareDrawRank(rank(1, LayerOrder.YSort, 0, -50), rank(1, LayerOrder.YSort, 0, 50)))
+ .toBeGreaterThan(0);
+ });
+
+ it('is antisymmetric, so a sort cannot depend on the input order', () => {
+ const a = rank(2, LayerOrder.Depth, 40);
+ const b = rank(7, LayerOrder.Depth, -40);
+ expect(Math.sign(compareDrawRank(a, b))).toBe(-Math.sign(compareDrawRank(b, a)));
+ expect(compareDrawRank(a, a)).toBe(0);
+ });
+});