Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

fix(camera): a point behind the eye has no screen position

esengine2c50c6d2ecab
diff --git a/desktop b/desktop
index 83cd2ab10..88ad087c2 160000
--- a/desktop
+++ b/desktop
@@ -1 +1 @@
-Subproject commit 83cd2ab101d66bb23cc9e9ba942f378515ad1932
+Subproject commit 88ad087c2844c7c3ca687e8865338cd7011195fb
diff --git a/sdk/src/camera/Camera.ts b/sdk/src/camera/Camera.ts
index 219eb1a4f..fc7cfd107 100644
--- a/sdk/src/camera/Camera.ts
+++ b/sdk/src/camera/Camera.ts
@@ -3,7 +3,7 @@
import type { App } from '../app/app';
import { defineResource } from '../ecs/resource';
import { UICameraInfo, type UICameraData } from '../ui/core/ui-camera-info';
-import { screenToWorld, worldToScreen, createInvVPCache, screenRay, type WorldRay } from '../ui/util/math';
+import { screenToWorld, projectWorldPoint, isProjectable, createInvVPCache, screenRay, type WorldRay } from '../ui/util/math';
/**
* Per-App camera-space query API: screen<->world conversions, the world-space
@@ -61,12 +61,19 @@ export class CameraViewAPI {
* @p worldZ projects nowhere near its shadow on the 2D plane, so anything
* drawing an overlay ON an entity (an outline, a gizmo, a screen rect) has to
* pass the entity's z or it draws where the entity is not.
+ *
+ * Null means the point has NO screen position: there is no camera, or it
+ * stands at or behind the eye, where the perspective divide mirrors it to the
+ * opposite side of the view. Being outside the viewport is not that — a
+ * collider or a frustum reaching past the edge still has coordinates there,
+ * and an overlay clipped to the canvas needs them.
*/
worldToScreen(worldX: number, worldY: number, worldZ = 0): { x: number; y: number } | null {
const cam = this.cam();
if (!cam) return null;
- const [sx, sy] = worldToScreen(worldX, worldY, cam.viewProjection, cam.vpX, cam.vpY, cam.vpW, cam.vpH, worldZ);
- return { x: sx, y: sy };
+ const p = projectWorldPoint(worldX, worldY, worldZ,
+ cam.viewProjection, cam.vpX, cam.vpY, cam.vpW, cam.vpH);
+ return isProjectable(p) ? { x: p.x, y: p.y } : null;
}
getWorldMousePosition(): { x: number; y: number } | null {
diff --git a/sdk/src/ui/index.ts b/sdk/src/ui/index.ts
index c40a3329e..d7494e0d2 100644
--- a/sdk/src/ui/index.ts
+++ b/sdk/src/ui/index.ts
@@ -339,10 +339,12 @@ export {
pointInWorldRect,
pointInOBB,
quaternionToAngle2D,
- worldToScreen,
+ projectWorldPoint,
+ isProjectable,
createInvVPCache,
type ScreenRect,
type WorldRay,
+ type ProjectedPoint,
} from './util/math';
// ─── Plugins (composed UI pipeline + the concept plugins it builds) ─────────
diff --git a/sdk/src/ui/util/math.ts b/sdk/src/ui/util/math.ts
index 44207127b..89a0fd90a 100644
--- a/sdk/src/ui/util/math.ts
+++ b/sdk/src/ui/util/math.ts
@@ -137,6 +137,35 @@ export function quaternionToAngle2D(rz: number, rw: number): number {
return q.angleZ({ z: rz, w: rw });
}
+/**
+ * A world point in clip space, and where that lands on screen.
+ *
+ * @details @p x / @p y are a screen position only while @p clipW is positive.
+ * Behind the eye the perspective divide flips the sign of both
+ * coordinates, so a point BEHIND the camera comes out as a perfectly
+ * plausible position on the OTHER side of the view — indistinguishable
+ * from a real one once the w is thrown away. That is why this carries
+ * the clip facts rather than answering with a bare pair of numbers.
+ *
+ * @p clipZ is kept so a segment straddling the near plane
+ * (`clipZ = -clipW`) can be cut there without projecting its endpoints
+ * a second time.
+ */
+export interface ProjectedPoint {
+ x: number;
+ y: number;
+ clipZ: number;
+ clipW: number;
+}
+
+/** At or below this, w names the eye itself and the divide has no answer. */
+export const CLIP_W_EPSILON = 1e-6;
+
+/** Whether the point has a screen position at all — see {@link ProjectedPoint}. */
+export function isProjectable(p: ProjectedPoint): boolean {
+ return p.clipW > CLIP_W_EPSILON;
+}
+
/**
* Where a world point lands on screen — the inverse of {@link screenToWorld}, and
* it takes the same third dimension.
@@ -145,23 +174,25 @@ export function quaternionToAngle2D(rz: number, rw: number): number {
* shadow on it: nearer content is larger and further from the centre.
* Dropping @p wz would put an entity's outline, gizmo and screen rect
* where the entity is NOT drawn — the exact error the unproject side
- * already fixed by taking a plane. Defaults to the 2D plane, so every
- * existing 2D caller keeps its answer to the bit.
+ * already fixed by taking a plane. There is no default: which plane a
+ * point sits on is the caller's fact, not this function's guess.
*/
-export function worldToScreen(
- wx: number, wy: number,
+export function projectWorldPoint(
+ wx: number, wy: number, wz: number,
vp: Float32Array, vpX: number, vpY: number, vpW: number, vpH: number,
- wz = 0,
-): [number, number] {
+): ProjectedPoint {
const clipX = vp[0] * wx + vp[4] * wy + vp[8] * wz + vp[12];
const clipY = vp[1] * wx + vp[5] * wy + vp[9] * wz + vp[13];
+ const clipZ = vp[2] * wx + vp[6] * wy + vp[10] * wz + vp[14];
const clipW = vp[3] * wx + vp[7] * wy + vp[11] * wz + vp[15];
const ndcX = clipX / clipW;
const ndcY = clipY / clipW;
- return [
- vpX + (ndcX * 0.5 + 0.5) * vpW,
- vpY + (ndcY * 0.5 + 0.5) * vpH,
- ];
+ return {
+ x: vpX + (ndcX * 0.5 + 0.5) * vpW,
+ y: vpY + (ndcY * 0.5 + 0.5) * vpH,
+ clipZ,
+ clipW,
+ };
}
export function createInvVPCache() {
diff --git a/sdk/src/ui/util/ui-pick.ts b/sdk/src/ui/util/ui-pick.ts
index 3725ec125..2f197fa6f 100644
--- a/sdk/src/ui/util/ui-pick.ts
+++ b/sdk/src/ui/util/ui-pick.ts
@@ -5,7 +5,7 @@ import type { World } from '../../ecs/world';
import type { UICameraData } from '../core/ui-camera-info';
import type { ScreenOverlayData } from '../core/screen-overlay';
import { worldEngineApi } from '../../ecs/bridge/engineApi';
-import { screenToWorld, worldToScreen, createInvVPCache, screenRay, type WorldRay } from './math';
+import { screenToWorld, projectWorldPoint, createInvVPCache, screenRay, type WorldRay } from './math';
const NO_HIT = 0xffffffff;
const vpCache = createInvVPCache();
@@ -86,14 +86,15 @@ export function isScreenEntity(world: PickableWorld, entity: Entity): boolean {
export function uiLayoutToScreen(
overlay: ScreenOverlayData, x: number, y: number,
): { x: number; y: number } {
- const [sx, sy] = worldToScreen(x, y, overlay.projection,
- overlay.vpX, overlay.vpY, overlay.vpW, overlay.vpH);
- return { x: sx, y: sy };
+ const p = projectWorldPoint(x, y, 0, overlay.projection,
+ overlay.vpX, overlay.vpY, overlay.vpW, overlay.vpH);
+ return { x: p.x, y: p.y };
}
export function uiWorldToScreen(camera: UICameraData, worldX: number, worldY: number): { x: number; y: number } {
- const [x, y] = worldToScreen(worldX, worldY, camera.viewProjection, camera.vpX, camera.vpY, camera.vpW, camera.vpH);
- return { x, y };
+ const p = projectWorldPoint(worldX, worldY, 0, camera.viewProjection,
+ camera.vpX, camera.vpY, camera.vpW, camera.vpH);
+ return { x: p.x, y: p.y };
}
/**
diff --git a/sdk/tests/uiMath.test.ts b/sdk/tests/uiMath.test.ts
index ae3da75c7..a9139ceea 100644
--- a/sdk/tests/uiMath.test.ts
+++ b/sdk/tests/uiMath.test.ts
@@ -7,7 +7,8 @@ import {
screenToWorld,
pointInWorldRect,
quaternionToAngle2D,
- worldToScreen,
+ projectWorldPoint,
+ isProjectable,
createInvVPCache,
pointInOBB,
screenRay,
@@ -237,45 +238,45 @@ describe('uiMath', () => {
expect(result.y).toBeCloseTo(0);
});
- it('should roundtrip with worldToScreen for identity VP', () => {
+ it('should roundtrip with projectWorldPoint for identity VP', () => {
const vp = identity4();
const invVP = invertMatrix4(vp);
const sx = 250, sy = 180;
const vpX = 0, vpY = 0, vpW = 800, vpH = 600;
const world = screenToWorld(sx, sy, invVP, vpX, vpY, vpW, vpH);
- const [backX, backY] = worldToScreen(world.x, world.y, vp, vpX, vpY, vpW, vpH);
+ const { x: backX, y: backY } = projectWorldPoint(world.x, world.y, 0, vp, vpX, vpY, vpW, vpH);
expect(backX).toBeCloseTo(sx);
expect(backY).toBeCloseTo(sy);
});
- it('should roundtrip with worldToScreen for scaled VP', () => {
+ it('should roundtrip with projectWorldPoint for scaled VP', () => {
const vp = scale4(0.5, 0.5, 1);
vp[15] = 1;
const invVP = invertMatrix4(vp);
const sx = 300, sy = 400;
const vpX = 0, vpY = 0, vpW = 800, vpH = 600;
const world = screenToWorld(sx, sy, invVP, vpX, vpY, vpW, vpH);
- const [backX, backY] = worldToScreen(world.x, world.y, vp, vpX, vpY, vpW, vpH);
+ const { x: backX, y: backY } = projectWorldPoint(world.x, world.y, 0, vp, vpX, vpY, vpW, vpH);
expect(backX).toBeCloseTo(sx);
expect(backY).toBeCloseTo(sy);
});
});
// =========================================================================
- // worldToScreen
+ // projectWorldPoint
// =========================================================================
- describe('worldToScreen', () => {
+ describe('projectWorldPoint', () => {
it('should map world origin to viewport center with identity VP', () => {
const vp = identity4();
- const [sx, sy] = worldToScreen(0, 0, vp, 0, 0, 800, 600);
+ const { x: sx, y: sy } = projectWorldPoint(0, 0, 0, vp, 0, 0, 800, 600);
expect(sx).toBeCloseTo(400);
expect(sy).toBeCloseTo(300);
});
it('should map world (-1,-1) to viewport origin with identity VP', () => {
const vp = identity4();
- const [sx, sy] = worldToScreen(-1, -1, vp, 0, 0, 800, 600);
+ const { x: sx, y: sy } = projectWorldPoint(-1, -1, 0, vp, 0, 0, 800, 600);
expect(sx).toBeCloseTo(0);
expect(sy).toBeCloseTo(0);
});
@@ -283,14 +284,14 @@ describe('uiMath', () => {
it('should handle scaled VP matrix', () => {
const vp = scale4(2, 2, 1);
vp[15] = 1;
- const [sx, sy] = worldToScreen(0.5, 0.5, vp, 0, 0, 800, 600);
+ const { x: sx, y: sy } = projectWorldPoint(0.5, 0.5, 0, vp, 0, 0, 800, 600);
expect(sx).toBeCloseTo(800);
expect(sy).toBeCloseTo(600);
});
it('should apply viewport offset', () => {
const vp = identity4();
- const [sx, sy] = worldToScreen(0, 0, vp, 50, 100, 800, 600);
+ const { x: sx, y: sy } = projectWorldPoint(0, 0, 0, vp, 50, 100, 800, 600);
expect(sx).toBeCloseTo(450);
expect(sy).toBeCloseTo(400);
});
@@ -300,7 +301,7 @@ describe('uiMath', () => {
const invVP = invertMatrix4(vp);
const vpX = 10, vpY = 20, vpW = 640, vpH = 480;
const wx = 3.5, wy = -2.1;
- const [sx, sy] = worldToScreen(wx, wy, vp, vpX, vpY, vpW, vpH);
+ const { x: sx, y: sy } = projectWorldPoint(wx, wy, 0, vp, vpX, vpY, vpW, vpH);
const back = screenToWorld(sx, sy, invVP, vpX, vpY, vpW, vpH);
expect(back.x).toBeCloseTo(wx);
expect(back.y).toBeCloseTo(wy);
@@ -577,24 +578,24 @@ describe('screenToWorld picking planes', () => {
});
});
-describe('worldToScreen carries the same third dimension', () => {
+describe('projectWorldPoint carries the same third dimension', () => {
// The unproject took a plane; the project has to take the point's depth, or
// an overlay drawn ON an entity (outline, gizmo, screen rect) lands on the
// entity's shadow at z = 0 instead of on the entity.
it('is unchanged orthographically, whatever depth the point is at', () => {
const vp = ortho4(160, 120, 1000);
- const flat = worldToScreen(80, -30, vp, 0, 0, 640, 480);
- const deep = worldToScreen(80, -30, vp, 0, 0, 640, 480, -400);
- expect(deep[0]).toBeCloseTo(flat[0], 6);
- expect(deep[1]).toBeCloseTo(flat[1], 6);
+ const flat = projectWorldPoint(80, -30, 0, vp, 0, 0, 640, 480);
+ const deep = projectWorldPoint(80, -30, -400, vp, 0, 0, 640, 480);
+ expect(deep.x).toBeCloseTo(flat.x, 6);
+ expect(deep.y).toBeCloseTo(flat.y, 6);
});
it('projects nearer content further from the centre under perspective', () => {
const vp = perspective4(Math.PI / 2, 1, 0.1, 1000);
- const near = worldToScreen(50, 0, vp, 0, 0, 640, 480, -100);
- const far = worldToScreen(50, 0, vp, 0, 0, 640, 480, -400);
+ const near = projectWorldPoint(50, 0, -100, vp, 0, 0, 640, 480);
+ const far = projectWorldPoint(50, 0, -400, vp, 0, 0, 640, 480);
const centre = 320;
- expect(Math.abs(near[0] - centre)).toBeGreaterThan(Math.abs(far[0] - centre) * 2);
+ expect(Math.abs(near.x - centre)).toBeGreaterThan(Math.abs(far.x - centre) * 2);
});
// The round trip is the whole contract: project a point at its depth, ask the
@@ -607,7 +608,7 @@ describe('worldToScreen carries the same third dimension', () => {
// In front of the camera (which sits at the origin looking down -z); z = 0
// is the eye itself, where a projection divides by zero for anyone.
for (const z of [-1, -100, -400, -900]) {
- const [sx, sy] = worldToScreen(37, -21, vp, 0, 0, 640, 480, z);
+ const { x: sx, y: sy } = projectWorldPoint(37, -21, z, vp, 0, 0, 640, 480);
const back = screenToWorld(sx, sy, inv, 0, 0, 640, 480, z);
expect(back.x).toBeCloseTo(37, 3);
expect(back.y).toBeCloseTo(-21, 3);
@@ -615,6 +616,54 @@ describe('worldToScreen carries the same third dimension', () => {
});
});
+describe('projectWorldPoint says whether a point has a screen position at all', () => {
+ // Behind the eye the divide flips the sign of x, y AND w, so the point comes
+ // back mirrored through the centre of the view — inside the viewport, on the
+ // wrong side, and indistinguishable from a real position once w is dropped.
+ // Every editor overlay that "follows" an entity used to follow that mirror.
+ it('rejects a point behind a perspective eye, which the mirror hides', () => {
+ const vp = perspective4(Math.PI / 2, 4 / 3, 0.1, 1000);
+ const centre = 320;
+ const front = projectWorldPoint(50, 0, -200, vp, 0, 0, 640, 480);
+ const behind = projectWorldPoint(50, 0, 200, vp, 0, 0, 640, 480);
+ expect(isProjectable(front)).toBe(true);
+ expect(isProjectable(behind)).toBe(false);
+ expect(front.x).toBeGreaterThan(centre);
+ // The mirror: same world x, opposite side, and on screen the whole time.
+ expect(behind.x).toBeLessThan(centre);
+ expect(behind.x).toBeGreaterThan(0);
+ });
+
+ // Off-screen is not the same fact. A collider or a frustum reaching past the
+ // edge still HAS coordinates there, and an overlay clipped to the canvas needs
+ // them to know which way it left.
+ it('keeps a point that is merely outside the viewport', () => {
+ const vp = perspective4(Math.PI / 2, 4 / 3, 0.1, 1000);
+ const off = projectWorldPoint(5000, 0, -200, vp, 0, 0, 640, 480);
+ expect(isProjectable(off)).toBe(true);
+ expect(off.x).toBeGreaterThan(640);
+ });
+
+ // An orthographic projection has no eye to stand behind: w is 1 everywhere,
+ // and the 2D editor — whose content sits at and below z = 0 — depends on that.
+ it('accepts every depth orthographically', () => {
+ const vp = ortho4(160, 120, 1000);
+ for (const z of [-400, 0, 400]) {
+ expect(isProjectable(projectWorldPoint(80, -30, z, vp, 0, 0, 640, 480))).toBe(true);
+ }
+ });
+
+ // What a segment cut at the near plane interpolates on; see the editor's
+ // near-plane clipping of collider and frustum wireframes.
+ it('reports the clip depth the near plane is expressed in', () => {
+ const vp = perspective4(Math.PI / 2, 4 / 3, 0.1, 1000);
+ const atNear = projectWorldPoint(0, 0, -0.1, vp, 0, 0, 640, 480);
+ expect(atNear.clipZ / atNear.clipW).toBeCloseTo(-1, 5);
+ const atFar = projectWorldPoint(0, 0, -1000, vp, 0, 0, 640, 480);
+ expect(atFar.clipZ / atFar.clipW).toBeCloseTo(1, 5);
+ });
+});
+
/**
* The property a game reads a click through, spelled out.
*