Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(editor): picking and dragging happen on the plane the entity is actually on

esenginea104f56fa794
diff --git a/desktop/src/engine/ViewportController.ts b/desktop/src/engine/ViewportController.ts
index 162000f29..70196de58 100644
--- a/desktop/src/engine/ViewportController.ts
+++ b/desktop/src/engine/ViewportController.ts
@@ -65,7 +65,7 @@ function jointConnectedRuntime(j: JointGizmoData): EntityId | null {
// Structural shape of the engine's CameraView resource (screen<->world).
interface CameraViewLike {
- screenToWorld(x: number, y: number): { x: number; y: number } | null;
+ screenToWorld(x: number, y: number, planeZ?: number): { x: number; y: number } | null;
worldToScreen(x: number, y: number): { x: number; y: number } | null;
}
@@ -162,12 +162,27 @@ const MINIMAP_BOX_CAP = 800;
// Picking and screen<->world conversions for the viewport, all routed through
// the engine's own camera matrices (no projection assumptions).
export const ViewportController = {
- /** DOM pointer position → world coordinates. */
- canvasToWorld(clientX: number, clientY: number): { x: number; y: number } | null {
+ /**
+ * DOM pointer position → world coordinates on the plane at @p planeZ.
+ *
+ * The default plane is the 2D one, which is the whole answer under an
+ * orthographic view — a screen point's world x/y do not change with depth
+ * there. In a perspective view they do, so anything that hits or drags a
+ * specific entity should pass that entity's z; navigation (pan, zoom, framing)
+ * keeps the 2D plane, which is the plane it is navigating over.
+ */
+ canvasToWorld(clientX: number, clientY: number, planeZ = 0): { x: number; y: number } | null {
const cv = cameraView();
const s = clientToScreen(clientX, clientY);
if (!cv || !s) return null;
- return cv.screenToWorld(s.sx, s.sy);
+ return cv.screenToWorld(s.sx, s.sy, planeZ);
+ },
+
+ /** The world z an entity sits on — the plane its picking and dragging happen on. */
+ entityPlaneZ(e: EntityId): number {
+ const world = EngineHost.world;
+ if (!world || !world.has(e, Transform)) return 0;
+ return (world.get(e, Transform) as { position?: { z?: number } }).position?.z ?? 0;
},
/** World coordinates → CSS pixels relative to the canvas top-left (gizmo placement). */
@@ -250,16 +265,22 @@ export const ViewportController = {
const out: EntityId[] = [...this.pickUIEntities(clientX, clientY)];
const world = EngineHost.world;
- const wp = this.canvasToWorld(clientX, clientY);
- if (world && wp) {
+ if (world) {
const hits: { e: EntityId; layer: number; i: number }[] = [];
+ // The cursor is tested against each candidate ON ITS OWN PLANE. One shared
+ // world point would be the 2D projection of the cursor, which under a
+ // perspective view is where a sprite's shadow falls rather than where the
+ // sprite is — so anything off the z = 0 plane would be unclickable at the
+ // place it is drawn. Orthographically every plane gives the same point, so
+ // this costs an unproject per candidate and changes no answer.
for (const e of world.getAllEntities()) {
if (!world.has(e, Transform)) continue;
// Locked / editor-hidden / environment entities aren't click-selectable.
const src = SceneModel.sourceFor(e);
if (src != null && !SceneModel.isPickable(src)) continue;
+ const wp = this.canvasToWorld(clientX, clientY, this.entityPlaneZ(e));
const b = this.entityBounds(e);
- if (!b || !pointInOBB(wp.x, wp.y, b)) continue;
+ if (!wp || !b || !pointInOBB(wp.x, wp.y, b)) continue;
const layer = world.has(e, Sprite) ? world.get(e, Sprite).layer : ICON_PICK_LAYER;
hits.push({ e, layer, i: hits.length });
}
diff --git a/desktop/src/tools/transformTools.ts b/desktop/src/tools/transformTools.ts
index 8826d0ba7..dc3866671 100644
--- a/desktop/src/tools/transformTools.ts
+++ b/desktop/src/tools/transformTools.ts
@@ -52,6 +52,16 @@ interface Drag {
axis: GizmoAxis;
pivotWorld: Pt;
pivotClient: Pt; // canvas-relative
+ /**
+ * The world plane this drag happens on — the active target's z.
+ *
+ * A drag converts screen motion into world motion, and under a perspective
+ * view that conversion depends on depth: the same pixels are a larger world
+ * distance further away. Measuring on the 2D plane would make anything with a
+ * z drift away from the cursor. Orthographically the plane is irrelevant and
+ * this changes nothing.
+ */
+ planeZ: number;
downWorld: Pt;
/** Where the press landed, in window-client px — what the slop is measured from. */
downClient: Pt;
@@ -279,7 +289,12 @@ function beginDrag(
armed = true,
): Drag {
const label = kind === 'rotate' ? 'Rotate' : kind === 'scale' ? 'Scale' : 'Move';
- const downWorld = ViewportController.canvasToWorld(p.clientX, p.clientY) ?? { x: 0, y: 0 };
+ // The active target is the one the gizmo sits on, so it is the plane the user
+ // is reaching for; a mixed-depth selection still moves rigidly with it.
+ const planeZ = targets.length
+ ? ViewportController.entityPlaneZ(targets[targets.length - 1].sourceId)
+ : 0;
+ const downWorld = ViewportController.canvasToWorld(p.clientX, p.clientY, planeZ) ?? { x: 0, y: 0 };
// Freeze React panel re-renders (Details/Outliner) for the drag — the model
// mutates every frame while the viewport stays live via the Reconciler. Resumed
// on commit (onPointerUp) or abort (cancel), flushing one final bump.
@@ -290,6 +305,7 @@ function beginDrag(
axis,
pivotWorld,
pivotClient,
+ planeZ,
downWorld,
downClient: { x: p.clientX, y: p.clientY },
startAngle: Math.atan2(cur.y - pivotClient.y, cur.x - pivotClient.x),
@@ -517,7 +533,7 @@ function makeTransformTool(mode: ToolMode): EditorTool {
? { x: p.clientX - origin.left, y: p.clientY - origin.top }
: { x: p.clientX, y: p.clientY };
if (drag.kind === 'move') {
- const w = ViewportController.canvasToWorld(p.clientX, p.clientY);
+ const w = ViewportController.canvasToWorld(p.clientX, p.clientY, drag.planeZ);
if (w) applyMove(drag, w);
} else if (drag.kind === 'rotate') {
applyRotate(drag, cur);
diff --git a/desktop/tests/transform-tool.test.ts b/desktop/tests/transform-tool.test.ts
index d5918ab58..02e8acf85 100644
--- a/desktop/tests/transform-tool.test.ts
+++ b/desktop/tests/transform-tool.test.ts
@@ -23,6 +23,9 @@ vi.mock('@/engine/EngineHost', () => ({
vi.mock('@/engine/ViewportController', () => ({
ViewportController: {
canvasToWorld: (x: number, y: number) => ({ x, y }), // identity: world == client px
+ // Flat scene: every plane is z = 0, which is what the orthographic editor
+ // answers anyway — the drag plane must not change these expectations.
+ entityPlaneZ: () => 0,
worldToClient: (x: number, y: number) => ({ x, y }),
getEntityWorldXY: (rt: number) => h.pos.get(rt) ?? { x: 0, y: 0 },
getEntityWorldAngleRad: () => 0,
diff --git a/sdk/etc/index.api.md b/sdk/etc/index.api.md
index e4c848a08..f6e0fbd49 100644
--- a/sdk/etc/index.api.md
+++ b/sdk/etc/index.api.md
@@ -1410,7 +1410,7 @@ ResourceDef<CameraViewAPI>
```
getWorldBounds: () => { left: number; right: number; bottom: number; top: number; } | null
getWorldMousePosition: () => { x: number; y: number; } | null
-screenToWorld: (screenX: number, screenY: number) => { x: number; y: number; } | null
+screenToWorld: (screenX: number, screenY: number, planeZ?: number) => { x: number; y: number; } | null
worldToScreen: (worldX: number, worldY: number) => { x: number; y: number; } | null
static new (app_: App): CameraViewAPI
static prototype: CameraViewAPI
diff --git a/sdk/etc/index.native.api.md b/sdk/etc/index.native.api.md
index 15f41aa8f..6883e184d 100644
--- a/sdk/etc/index.native.api.md
+++ b/sdk/etc/index.native.api.md
@@ -1410,7 +1410,7 @@ ResourceDef<CameraViewAPI>
```
getWorldBounds: () => { left: number; right: number; bottom: number; top: number; } | null
getWorldMousePosition: () => { x: number; y: number; } | null
-screenToWorld: (screenX: number, screenY: number) => { x: number; y: number; } | null
+screenToWorld: (screenX: number, screenY: number, planeZ?: number) => { x: number; y: number; } | null
worldToScreen: (worldX: number, worldY: number) => { x: number; y: number; } | null
static new (app_: App): CameraViewAPI
static prototype: CameraViewAPI
diff --git a/sdk/etc/index.node.api.md b/sdk/etc/index.node.api.md
index ce8c7d1eb..7d935bab9 100644
--- a/sdk/etc/index.node.api.md
+++ b/sdk/etc/index.node.api.md
@@ -1405,7 +1405,7 @@ ResourceDef<CameraViewAPI>
```
getWorldBounds: () => { left: number; right: number; bottom: number; top: number; } | null
getWorldMousePosition: () => { x: number; y: number; } | null
-screenToWorld: (screenX: number, screenY: number) => { x: number; y: number; } | null
+screenToWorld: (screenX: number, screenY: number, planeZ?: number) => { x: number; y: number; } | null
worldToScreen: (worldX: number, worldY: number) => { x: number; y: number; } | null
static new (app_: App): CameraViewAPI
static prototype: CameraViewAPI
diff --git a/sdk/etc/minigame.api.md b/sdk/etc/minigame.api.md
index d7552c104..7d7a633a4 100644
--- a/sdk/etc/minigame.api.md
+++ b/sdk/etc/minigame.api.md
@@ -1405,7 +1405,7 @@ ResourceDef<CameraViewAPI>
```
getWorldBounds: () => { left: number; right: number; bottom: number; top: number; } | null
getWorldMousePosition: () => { x: number; y: number; } | null
-screenToWorld: (screenX: number, screenY: number) => { x: number; y: number; } | null
+screenToWorld: (screenX: number, screenY: number, planeZ?: number) => { x: number; y: number; } | null
worldToScreen: (worldX: number, worldY: number) => { x: number; y: number; } | null
static new (app_: App): CameraViewAPI
static prototype: CameraViewAPI
diff --git a/sdk/etc/wechat.api.md b/sdk/etc/wechat.api.md
index cc6b86f7f..b1780243a 100644
--- a/sdk/etc/wechat.api.md
+++ b/sdk/etc/wechat.api.md
@@ -1405,7 +1405,7 @@ ResourceDef<CameraViewAPI>
```
getWorldBounds: () => { left: number; right: number; bottom: number; top: number; } | null
getWorldMousePosition: () => { x: number; y: number; } | null
-screenToWorld: (screenX: number, screenY: number) => { x: number; y: number; } | null
+screenToWorld: (screenX: number, screenY: number, planeZ?: number) => { x: number; y: number; } | null
worldToScreen: (worldX: number, worldY: number) => { x: number; y: number; } | null
static new (app_: App): CameraViewAPI
static prototype: CameraViewAPI
diff --git a/sdk/src/camera/Camera.ts b/sdk/src/camera/Camera.ts
index b8ed0806a..69154fb46 100644
--- a/sdk/src/camera/Camera.ts
+++ b/sdk/src/camera/Camera.ts
@@ -24,12 +24,21 @@ export class CameraViewAPI {
return cam.valid ? cam : null;
}
- screenToWorld(screenX: number, screenY: number): { x: number; y: number } | null {
+ /**
+ * Where a screen point lands on the world plane at @p planeZ.
+ *
+ * A screen point is a ray. Orthographically its x/y do not vary along it, so
+ * the plane is irrelevant and the default answers for everyone. Under a
+ * perspective camera it decides the answer, and the right plane is the one
+ * the thing being hit or dragged actually sits on — z = 0 would place a
+ * sprite at z = -400 wherever its shadow on the 2D plane happens to fall.
+ */
+ screenToWorld(screenX: number, screenY: number, planeZ = 0): { x: number; y: number } | null {
const cam = this.cam();
if (!cam) return null;
this.invVPCache.update(cam.viewProjection);
const invVP = this.invVPCache.getInverse(cam.viewProjection);
- return screenToWorld(screenX, screenY, invVP, cam.vpX, cam.vpY, cam.vpW, cam.vpH);
+ return screenToWorld(screenX, screenY, invVP, cam.vpX, cam.vpY, cam.vpW, cam.vpH, planeZ);
}
worldToScreen(worldX: number, worldY: number): { x: number; y: number } | null {
diff --git a/sdk/tests/uiMath.test.ts b/sdk/tests/uiMath.test.ts
index 13d659e20..26181199e 100644
--- a/sdk/tests/uiMath.test.ts
+++ b/sdk/tests/uiMath.test.ts
@@ -548,3 +548,29 @@ describe('screenToWorld across projections', () => {
expect(Math.abs(onPlane.x)).toBeGreaterThan(50);
});
});
+
+describe('screenToWorld picking planes', () => {
+ // What the editor relies on when it tests each candidate on its own plane:
+ // under an orthographic view that costs an unproject and changes no answer,
+ // so 2D picking cannot regress.
+ it('gives one answer per screen point orthographically, at any entity depth', () => {
+ const vp = ortho4(160, 120, 1000);
+ const inv = invertMatrix4(vp)!;
+ const depths = [0, -5, 1, -400, 900];
+ const answers = depths.map((z) => screenToWorld(500, 200, inv, 0, 0, 640, 480, z));
+ for (const a of answers) {
+ expect(a.x).toBeCloseTo(answers[0].x, 6);
+ expect(a.y).toBeCloseTo(answers[0].y, 6);
+ }
+ });
+
+ // And why it is needed: perspective spreads those same depths apart, so a
+ // sprite off the 2D plane would be unclickable where it is drawn.
+ it('spreads them apart under perspective', () => {
+ const vp = perspective4(Math.PI / 2, 4 / 3, 0.1, 1000);
+ const inv = invertMatrix4(vp)!;
+ const near = screenToWorld(500, 200, inv, 0, 0, 640, 480, -10);
+ const far = screenToWorld(500, 200, inv, 0, 0, 640, 480, -400);
+ expect(Math.abs(far.x - near.x)).toBeGreaterThan(1);
+ });
+});