A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
fix(sdk): a screen point is a ray, and an optional binding is not a requirement
esengine9bc5b084f875
diff --git a/sdk/etc/index.api.md b/sdk/etc/index.api.mdindex 2d0d2a8c2..fb1d57db5 100644--- a/sdk/etc/index.api.md+++ b/sdk/etc/index.api.md@@ -10407,7 +10407,7 @@ Plugin## screenToWorld — function```-(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number): { x: number; y: number; }+(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number, planeZ?: number): { x: number; y: number; }```## selectBlendClip — functiondiff --git a/sdk/etc/index.native.api.md b/sdk/etc/index.native.api.mdindex d7e8bc8c4..bff739286 100644--- a/sdk/etc/index.native.api.md+++ b/sdk/etc/index.native.api.md@@ -10627,7 +10627,7 @@ Plugin## screenToWorld — function```-(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number): { x: number; y: number; }+(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number, planeZ?: number): { x: number; y: number; }```## selectBlendClip — functiondiff --git a/sdk/etc/index.node.api.md b/sdk/etc/index.node.api.mdindex eb3a54c81..923e8b03c 100644--- a/sdk/etc/index.node.api.md+++ b/sdk/etc/index.node.api.md@@ -10412,7 +10412,7 @@ Plugin## screenToWorld — function```-(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number): { x: number; y: number; }+(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number, planeZ?: number): { x: number; y: number; }```## selectBlendClip — functiondiff --git a/sdk/etc/minigame.api.md b/sdk/etc/minigame.api.mdindex 44542c9ad..f91da0210 100644--- a/sdk/etc/minigame.api.md+++ b/sdk/etc/minigame.api.md@@ -10661,7 +10661,7 @@ Plugin## screenToWorld — function```-(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number): { x: number; y: number; }+(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number, planeZ?: number): { x: number; y: number; }```## selectBlendClip — functiondiff --git a/sdk/etc/wechat.api.md b/sdk/etc/wechat.api.mdindex a0599ba10..db61d0176 100644--- a/sdk/etc/wechat.api.md+++ b/sdk/etc/wechat.api.md@@ -10433,7 +10433,7 @@ Plugin## screenToWorld — function```-(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number): { x: number; y: number; }+(screenX: number, screenY: number, inverseVP: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number, planeZ?: number): { x: number; y: number; }```## selectBlendClip — functiondiff --git a/sdk/src/ecs/bridge/nativeBindings.ts b/sdk/src/ecs/bridge/nativeBindings.tsindex b2279786f..8fc6d06ba 100644--- a/sdk/src/ecs/bridge/nativeBindings.ts+++ b/sdk/src/ecs/bridge/nativeBindings.ts@@ -105,7 +105,6 @@ export const RENDERER_BINDINGS = {setStage: 'es_renderer_setStage',setViewport: 'es_renderer_setViewport',setYSortLayers: 'es_renderer_setYSortLayers',- setDepthLayers: 'es_renderer_setDepthLayers',/** Host-specific: the drawable size, which no wasm entry point has. */surfaceSize: 'es_renderer_surfaceSize',} as const;@@ -134,8 +133,24 @@ export const HOST_FLAGS = {ownsFrame: 'es_jsOwnsFrame',} as const;+/**+ * Renderer entry points a host may legitimately NOT have, and which therefore+ * must stay out of `hasRendererBindings`.+ *+ * That probe is an all-or-nothing gate on the SDK driving the frame, so adding an+ * entry to RENDERER_BINDINGS retroactively declares every host built before it+ * rendererless — the SDK would hand the frame back and the game would go blank+ * on a shell that was working. A capability added after a host shipped belongs+ * here and is called through the optional path.+ */+export const RENDERER_OPTIONAL_BINDINGS = {+ /** 2.5D depth layers; hosts predating the feature simply stay painter-ordered. */+ setDepthLayers: 'es_renderer_setDepthLayers',+} as const;+/** Whether the host bound the whole frame surface — the gate for the SDK driving- * the frame (cameras and all) rather than the host hard-coding one. */+ * the frame (cameras and all) rather than the host hard-coding one. Optional+ * entry points (RENDERER_OPTIONAL_BINDINGS) are deliberately not part of it. */export function hasRendererBindings(scope: Record<string, unknown> = globalThis as unknown as Record<string, unknown>,): boolean {diff --git a/sdk/src/ui/util/math.ts b/sdk/src/ui/util/math.tsindex b3f50d0b5..ad6ca6a63 100644--- a/sdk/src/ui/util/math.ts+++ b/sdk/src/ui/util/math.ts@@ -60,19 +60,59 @@ export function invertMatrix4(m: Float32Array, result?: Float32Array): Float32Arreturn out;}+/**+ * Unprojects one NDC point, depth included.+ *+ * The full 4×4 product, unlike the x/y-only shortcut it replaces — under a+ * perspective projection the third column is not zero, so a screen point does+ * not name a world point at all until a depth is chosen.+ */+function unproject(+ ndcX: number, ndcY: number, ndcZ: number, m: Float32Array,+): { x: number; y: number; z: number } {+ const x = m[0] * ndcX + m[4] * ndcY + m[8] * ndcZ + m[12];+ const y = m[1] * ndcX + m[5] * ndcY + m[9] * ndcZ + m[13];+ const z = m[2] * ndcX + m[6] * ndcY + m[10] * ndcZ + m[14];+ const w = m[3] * ndcX + m[7] * ndcY + m[11] * ndcZ + m[15];+ const inv = w !== 0 ? 1 / w : 0;+ return { x: x * inv, y: y * inv, z: z * inv };+}++/**+ * Where a screen point lands on the world plane at @p planeZ.+ *+ * @details A screen point is a RAY, not a position. Orthographically the ray's+ * x/y are constant along its length, which is why taking z = 0 worked+ * and why every 2D caller can keep calling this unchanged. Under a+ * perspective camera they are not constant, and the same call would+ * answer with wherever the near plane happened to be — off by more the+ * further the content sits from z = 0.+ *+ * So the ray is intersected with the plane, which is one implementation+ * with the orthographic case as its degenerate form rather than two+ * branches that have to agree. `planeZ` defaults to the 2D plane, so a+ * caller that has no depth in mind keeps getting the 2D answer.+ */export function screenToWorld(screenX: number, screenY: number,inverseVP: Float32Array,- vpX: number, vpY: number, vpW: number, vpH: number+ vpX: number, vpY: number, vpW: number, vpH: number,+ planeZ = 0,): { x: number; y: number } {const ndcX = ((screenX - vpX) / vpW) * 2 - 1;const ndcY = ((screenY - vpY) / vpH) * 2 - 1;- const wx = inverseVP[0] * ndcX + inverseVP[4] * ndcY + inverseVP[12];- const wy = inverseVP[1] * ndcX + inverseVP[5] * ndcY + inverseVP[13];- const ww = inverseVP[3] * ndcX + inverseVP[7] * ndcY + inverseVP[15];+ const near = unproject(ndcX, ndcY, -1, inverseVP);+ const far = unproject(ndcX, ndcY, 1, inverseVP);++ const dz = far.z - near.z;+ // A ray parallel to the plane it is being intersected with has no answer;+ // the near point is the closest thing to one, and it is what the+ // orthographic path returned before this existed.+ if (dz === 0) return { x: near.x, y: near.y };- return { x: wx / ww, y: wy / ww };+ const t = (planeZ - near.z) / dz;+ return { x: near.x + (far.x - near.x) * t, y: near.y + (far.y - near.y) * t };}export function pointInWorldRect(diff --git a/sdk/tests/__snapshots__/component-fieldmeta-golden.test.ts.snap b/sdk/tests/__snapshots__/component-fieldmeta-golden.test.ts.snapindex 603d3998e..b092c47bd 100644--- a/sdk/tests/__snapshots__/component-fieldmeta-golden.test.ts.snap+++ b/sdk/tests/__snapshots__/component-fieldmeta-golden.test.ts.snap@@ -262,6 +262,26 @@ exports[`builtin FieldMeta (RC9-1 equivalence gate) > matches the golden snapsho"label": "PremultipliedAlpha","value": 4,},+ {+ "label": "PmaAdditive",+ "value": 5,+ },+ {+ "label": "Lighten",+ "value": 6,+ },+ {+ "label": "Darken",+ "value": 7,+ },+ {+ "label": "Overlay",+ "value": 8,+ },+ {+ "label": "None",+ "value": 9,+ },],},"burstCount": {diff --git a/sdk/tests/uiMath.test.ts b/sdk/tests/uiMath.test.tsindex 56d729c34..13d659e20 100644--- a/sdk/tests/uiMath.test.ts+++ b/sdk/tests/uiMath.test.ts@@ -479,3 +479,72 @@ describe('uiMath', () => {});});});++// =============================================================================+// screenToWorld under a perspective projection+// =============================================================================++/** Right-handed perspective, matching the engine's `perspective()` in mat4. */+function perspective4(fovY: number, aspect: number, near: number, far: number): Float32Array {+ const m = new Float32Array(16);+ const f = 1 / Math.tan(fovY / 2);+ m[0] = f / aspect;+ m[5] = f;+ m[10] = (far + near) / (near - far);+ m[11] = -1;+ m[14] = (2 * far * near) / (near - far);+ return m;+}++/** Right-handed orthographic, the symmetric [-far, far] the 2D camera builds. */+function ortho4(halfW: number, halfH: number, far: number): Float32Array {+ const m = new Float32Array(16);+ m[0] = 1 / halfW;+ m[5] = 1 / halfH;+ m[10] = -1 / far;+ m[15] = 1;+ return m;+}++describe('screenToWorld across projections', () => {+ // The claim every 2D caller depends on: intersecting the ray costs nothing+ // under an orthographic camera, because x/y do not vary along it.+ it('is unchanged orthographically, whatever plane is asked for', () => {+ const vp = ortho4(160, 120, 1000);+ const inv = invertMatrix4(vp)!;+ const atZero = screenToWorld(480, 180, inv, 0, 0, 640, 480);+ const atFar = screenToWorld(480, 180, inv, 0, 0, 640, 480, -300);++ expect(atZero.x).toBeCloseTo(80, 5);+ expect(atZero.y).toBeCloseTo(-30, 5);+ expect(atFar.x).toBeCloseTo(atZero.x, 5);+ expect(atFar.y).toBeCloseTo(atZero.y, 5);+ });++ // And the claim that makes it necessary: under perspective the same screen+ // point names different world points depending on the depth asked for, so a+ // single answer would have been wrong for every plane but one.+ it('follows the ray under perspective, so the plane decides the answer', () => {+ // Camera at the origin looking down -z; the plane sits at z = -d.+ const vp = perspective4(Math.PI / 2, 1, 0.1, 1000);+ const inv = invertMatrix4(vp)!;++ // Half a viewport to the right ⇒ the ray leaves at 45° with fov 90°,+ // so it crosses z = -d at x = d exactly.+ const near = screenToWorld(640, 240, inv, 0, 0, 640, 480, -10);+ const far = screenToWorld(640, 240, inv, 0, 0, 640, 480, -100);++ expect(near.x).toBeCloseTo(10, 3);+ expect(far.x).toBeCloseTo(100, 3);+ expect(near.y).toBeCloseTo(0, 3);+ });++ it('does not answer with the near plane when a depth is given', () => {+ const vp = perspective4(Math.PI / 2, 1, 0.1, 1000);+ const inv = invertMatrix4(vp)!;+ // The pre-ray implementation ignored ndcZ, which under perspective is the+ // near plane's answer — two orders of magnitude off at this depth.+ const onPlane = screenToWorld(640, 240, inv, 0, 0, 640, 480, -100);+ expect(Math.abs(onPlane.x)).toBeGreaterThan(50);+ });+});