Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

fix(sdk): a screen point is a ray, and an optional binding is not a requirement

esengine9bc5b084f875
diff --git a/sdk/etc/index.api.md b/sdk/etc/index.api.md
index 2d0d2a8c2..fb1d57db5 100644
--- a/sdk/etc/index.api.md
+++ b/sdk/etc/index.api.md
@@ -10407,7 +10407,7 @@ Plugin
## screenToWorld — function
```
-(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number): { x: number; y: number; }
+(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number, planeZ?: number): { x: number; y: number; }
```
## selectBlendClip — function
diff --git a/sdk/etc/index.native.api.md b/sdk/etc/index.native.api.md
index d7e8bc8c4..bff739286 100644
--- a/sdk/etc/index.native.api.md
+++ b/sdk/etc/index.native.api.md
@@ -10627,7 +10627,7 @@ Plugin
## screenToWorld — function
```
-(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number): { x: number; y: number; }
+(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number, planeZ?: number): { x: number; y: number; }
```
## selectBlendClip — function
diff --git a/sdk/etc/index.node.api.md b/sdk/etc/index.node.api.md
index eb3a54c81..923e8b03c 100644
--- a/sdk/etc/index.node.api.md
+++ b/sdk/etc/index.node.api.md
@@ -10412,7 +10412,7 @@ Plugin
## screenToWorld — function
```
-(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number): { x: number; y: number; }
+(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number, planeZ?: number): { x: number; y: number; }
```
## selectBlendClip — function
diff --git a/sdk/etc/minigame.api.md b/sdk/etc/minigame.api.md
index 44542c9ad..f91da0210 100644
--- a/sdk/etc/minigame.api.md
+++ b/sdk/etc/minigame.api.md
@@ -10661,7 +10661,7 @@ Plugin
## screenToWorld — function
```
-(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number): { x: number; y: number; }
+(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number, planeZ?: number): { x: number; y: number; }
```
## selectBlendClip — function
diff --git a/sdk/etc/wechat.api.md b/sdk/etc/wechat.api.md
index a0599ba10..db61d0176 100644
--- a/sdk/etc/wechat.api.md
+++ b/sdk/etc/wechat.api.md
@@ -10433,7 +10433,7 @@ Plugin
## screenToWorld — function
```
-(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number): { x: number; y: number; }
+(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number, planeZ?: number): { x: number; y: number; }
```
## selectBlendClip — function
diff --git a/sdk/src/ecs/bridge/nativeBindings.ts b/sdk/src/ecs/bridge/nativeBindings.ts
index b2279786f..8fc6d06ba 100644
--- a/sdk/src/ecs/bridge/nativeBindings.ts
+++ b/sdk/src/ecs/bridge/nativeBindings.ts
@@ -105,7 +105,6 @@ export const RENDERER_BINDINGS = {
setStage: 'es_renderer_setStage',
setViewport: 'es_renderer_setViewport',
setYSortLayers: 'es_renderer_setYSortLayers',
- setDepthLayers: 'es_renderer_setDepthLayers',
/** Host-specific: the drawable size, which no wasm entry point has. */
surfaceSize: 'es_renderer_surfaceSize',
} as const;
@@ -134,8 +133,24 @@ export const HOST_FLAGS = {
ownsFrame: 'es_jsOwnsFrame',
} as const;
+/**
+ * Renderer entry points a host may legitimately NOT have, and which therefore
+ * must stay out of `hasRendererBindings`.
+ *
+ * That probe is an all-or-nothing gate on the SDK driving the frame, so adding an
+ * entry to RENDERER_BINDINGS retroactively declares every host built before it
+ * rendererless — the SDK would hand the frame back and the game would go blank
+ * on a shell that was working. A capability added after a host shipped belongs
+ * here and is called through the optional path.
+ */
+export const RENDERER_OPTIONAL_BINDINGS = {
+ /** 2.5D depth layers; hosts predating the feature simply stay painter-ordered. */
+ setDepthLayers: 'es_renderer_setDepthLayers',
+} as const;
+
/** Whether the host bound the whole frame surface — the gate for the SDK driving
- * the frame (cameras and all) rather than the host hard-coding one. */
+ * the frame (cameras and all) rather than the host hard-coding one. Optional
+ * entry points (RENDERER_OPTIONAL_BINDINGS) are deliberately not part of it. */
export function hasRendererBindings(
scope: Record<string, unknown> = globalThis as unknown as Record<string, unknown>,
): boolean {
diff --git a/sdk/src/ui/util/math.ts b/sdk/src/ui/util/math.ts
index b3f50d0b5..ad6ca6a63 100644
--- a/sdk/src/ui/util/math.ts
+++ b/sdk/src/ui/util/math.ts
@@ -60,19 +60,59 @@ export function invertMatrix4(m: Float32Array, result?: Float32Array): Float32Ar
return out;
}
+/**
+ * Unprojects one NDC point, depth included.
+ *
+ * The full 4×4 product, unlike the x/y-only shortcut it replaces — under a
+ * perspective projection the third column is not zero, so a screen point does
+ * not name a world point at all until a depth is chosen.
+ */
+function unproject(
+ ndcX: number, ndcY: number, ndcZ: number, m: Float32Array,
+): { x: number; y: number; z: number } {
+ const x = m[0] * ndcX + m[4] * ndcY + m[8] * ndcZ + m[12];
+ const y = m[1] * ndcX + m[5] * ndcY + m[9] * ndcZ + m[13];
+ const z = m[2] * ndcX + m[6] * ndcY + m[10] * ndcZ + m[14];
+ const w = m[3] * ndcX + m[7] * ndcY + m[11] * ndcZ + m[15];
+ const inv = w !== 0 ? 1 / w : 0;
+ return { x: x * inv, y: y * inv, z: z * inv };
+}
+
+/**
+ * Where a screen point lands on the world plane at @p planeZ.
+ *
+ * @details A screen point is a RAY, not a position. Orthographically the ray's
+ * x/y are constant along its length, which is why taking z = 0 worked
+ * and why every 2D caller can keep calling this unchanged. Under a
+ * perspective camera they are not constant, and the same call would
+ * answer with wherever the near plane happened to be — off by more the
+ * further the content sits from z = 0.
+ *
+ * So the ray is intersected with the plane, which is one implementation
+ * with the orthographic case as its degenerate form rather than two
+ * branches that have to agree. `planeZ` defaults to the 2D plane, so a
+ * caller that has no depth in mind keeps getting the 2D answer.
+ */
export function screenToWorld(
screenX: number, screenY: number,
inverseVP: Float32Array,
- vpX: number, vpY: number, vpW: number, vpH: number
+ vpX: number, vpY: number, vpW: number, vpH: number,
+ planeZ = 0,
): { x: number; y: number } {
const ndcX = ((screenX - vpX) / vpW) * 2 - 1;
const ndcY = ((screenY - vpY) / vpH) * 2 - 1;
- const wx = inverseVP[0] * ndcX + inverseVP[4] * ndcY + inverseVP[12];
- const wy = inverseVP[1] * ndcX + inverseVP[5] * ndcY + inverseVP[13];
- const ww = inverseVP[3] * ndcX + inverseVP[7] * ndcY + inverseVP[15];
+ const near = unproject(ndcX, ndcY, -1, inverseVP);
+ const far = unproject(ndcX, ndcY, 1, inverseVP);
+
+ const dz = far.z - near.z;
+ // A ray parallel to the plane it is being intersected with has no answer;
+ // the near point is the closest thing to one, and it is what the
+ // orthographic path returned before this existed.
+ if (dz === 0) return { x: near.x, y: near.y };
- return { x: wx / ww, y: wy / ww };
+ const t = (planeZ - near.z) / dz;
+ return { x: near.x + (far.x - near.x) * t, y: near.y + (far.y - near.y) * t };
}
export function pointInWorldRect(
diff --git a/sdk/tests/__snapshots__/component-fieldmeta-golden.test.ts.snap b/sdk/tests/__snapshots__/component-fieldmeta-golden.test.ts.snap
index 603d3998e..b092c47bd 100644
--- a/sdk/tests/__snapshots__/component-fieldmeta-golden.test.ts.snap
+++ b/sdk/tests/__snapshots__/component-fieldmeta-golden.test.ts.snap
@@ -262,6 +262,26 @@ exports[`builtin FieldMeta (RC9-1 equivalence gate) > matches the golden snapsho
"label": "PremultipliedAlpha",
"value": 4,
},
+ {
+ "label": "PmaAdditive",
+ "value": 5,
+ },
+ {
+ "label": "Lighten",
+ "value": 6,
+ },
+ {
+ "label": "Darken",
+ "value": 7,
+ },
+ {
+ "label": "Overlay",
+ "value": 8,
+ },
+ {
+ "label": "None",
+ "value": 9,
+ },
],
},
"burstCount": {
diff --git a/sdk/tests/uiMath.test.ts b/sdk/tests/uiMath.test.ts
index 56d729c34..13d659e20 100644
--- a/sdk/tests/uiMath.test.ts
+++ b/sdk/tests/uiMath.test.ts
@@ -479,3 +479,72 @@ describe('uiMath', () => {
});
});
});
+
+// =============================================================================
+// screenToWorld under a perspective projection
+// =============================================================================
+
+/** Right-handed perspective, matching the engine's `perspective()` in mat4. */
+function perspective4(fovY: number, aspect: number, near: number, far: number): Float32Array {
+ const m = new Float32Array(16);
+ const f = 1 / Math.tan(fovY / 2);
+ m[0] = f / aspect;
+ m[5] = f;
+ m[10] = (far + near) / (near - far);
+ m[11] = -1;
+ m[14] = (2 * far * near) / (near - far);
+ return m;
+}
+
+/** Right-handed orthographic, the symmetric [-far, far] the 2D camera builds. */
+function ortho4(halfW: number, halfH: number, far: number): Float32Array {
+ const m = new Float32Array(16);
+ m[0] = 1 / halfW;
+ m[5] = 1 / halfH;
+ m[10] = -1 / far;
+ m[15] = 1;
+ return m;
+}
+
+describe('screenToWorld across projections', () => {
+ // The claim every 2D caller depends on: intersecting the ray costs nothing
+ // under an orthographic camera, because x/y do not vary along it.
+ it('is unchanged orthographically, whatever plane is asked for', () => {
+ const vp = ortho4(160, 120, 1000);
+ const inv = invertMatrix4(vp)!;
+ const atZero = screenToWorld(480, 180, inv, 0, 0, 640, 480);
+ const atFar = screenToWorld(480, 180, inv, 0, 0, 640, 480, -300);
+
+ expect(atZero.x).toBeCloseTo(80, 5);
+ expect(atZero.y).toBeCloseTo(-30, 5);
+ expect(atFar.x).toBeCloseTo(atZero.x, 5);
+ expect(atFar.y).toBeCloseTo(atZero.y, 5);
+ });
+
+ // And the claim that makes it necessary: under perspective the same screen
+ // point names different world points depending on the depth asked for, so a
+ // single answer would have been wrong for every plane but one.
+ it('follows the ray under perspective, so the plane decides the answer', () => {
+ // Camera at the origin looking down -z; the plane sits at z = -d.
+ const vp = perspective4(Math.PI / 2, 1, 0.1, 1000);
+ const inv = invertMatrix4(vp)!;
+
+ // Half a viewport to the right ⇒ the ray leaves at 45° with fov 90°,
+ // so it crosses z = -d at x = d exactly.
+ const near = screenToWorld(640, 240, inv, 0, 0, 640, 480, -10);
+ const far = screenToWorld(640, 240, inv, 0, 0, 640, 480, -100);
+
+ expect(near.x).toBeCloseTo(10, 3);
+ expect(far.x).toBeCloseTo(100, 3);
+ expect(near.y).toBeCloseTo(0, 3);
+ });
+
+ it('does not answer with the near plane when a depth is given', () => {
+ const vp = perspective4(Math.PI / 2, 1, 0.1, 1000);
+ const inv = invertMatrix4(vp)!;
+ // The pre-ray implementation ignored ndcZ, which under perspective is the
+ // near plane's answer — two orders of magnitude off at this depth.
+ const onPlane = screenToWorld(640, 240, inv, 0, 0, 640, 480, -100);
+ expect(Math.abs(onPlane.x)).toBeGreaterThan(50);
+ });
+});