Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

refactor(ui): the readers of the screen move off the camera, where they can

esengine701db13b99b3
diff --git a/sdk/src/app/app.ts b/sdk/src/app/app.ts
index 24f568dd2..de6cf7d2c 100644
--- a/sdk/src/app/app.ts
+++ b/sdk/src/app/app.ts
@@ -18,6 +18,7 @@ import type { ESEngineModule, CppRegistry } from '../wasm';
import { outputTransformCode, type OutputTransform } from '../postprocess';
import type { BridgeConnectOptions } from '../ecs/bridge/BuiltinBridge';
import { UICameraInfo } from '../ui/core/ui-camera-info';
+import { ScreenLayout } from '../ui/core/screen-layout';
import { inputPlugin, Input } from '../input/input';
import { assetPlugin } from '../asset';
import { prefabsPlugin } from '../prefab/prefabServer';
@@ -1706,6 +1707,7 @@ export function createWebApp(module: ESEngineModule, options?: WebAppOptions): A
app.addPlugin(corePlugin);
app.setPipeline(new RenderPipeline());
app.insertResource(UICameraInfo, { ...DEFAULT_UI_CAMERA_INFO });
+ app.insertResource(ScreenLayout, { ...ScreenLayout._default });
app.addPlugin(cameraPlugin(options?.getViewportSize));
app.addPlugin(assetPlugin);
app.addPlugin(prefabsPlugin);
diff --git a/sdk/src/camera/CameraPlugin.ts b/sdk/src/camera/CameraPlugin.ts
index 5f15fb77c..39478e02b 100644
--- a/sdk/src/camera/CameraPlugin.ts
+++ b/sdk/src/camera/CameraPlugin.ts
@@ -16,6 +16,7 @@ import type { ESEngineModule, CppRegistry } from '../wasm';
import type { World } from '../ecs/world';
import type { Entity } from '../types';
import { UICameraInfo } from '../ui/core/ui-camera-info';
+import { ScreenLayout, screenLayoutRect } from '../ui/core/screen-layout';
import { ProjectionType, SceneOwner, ClearFlags } from '../ecs/component';
import { uiLayoutRect, computeEffectiveOrthoSize, EDITOR_VIEW_ENTITY, type CanvasScale } from './uiLayoutRect';
import { EditorView, DEFAULT_EDITOR_VIEW, editorViewStandoff, editorViewClipFar, editorViewClipNear, type EditorViewData } from './EditorView';
@@ -590,6 +591,13 @@ function syncUICameraInfo(
// and a shipped build lay UI out in one box.
const previewAspect = app.hasResource(EditorView) ? app.getResource(EditorView).uiPreviewAspect : 0;
const fit = resolveFitSource(app, findCanvasData(module, cppRegistry));
+ // The screen domain is published from here because this is where the fit
+ // is resolved, and from NOTHING here: `screenLayoutRect` takes the fit and
+ // the viewport, and a camera has no way to reach it.
+ if (app.hasResource(ScreenLayout)) {
+ Object.assign(app.getResource(ScreenLayout),
+ screenLayoutRect(fit, uiCam.vpW, uiCam.vpH));
+ }
const rect = uiLayoutRect(cam, fit, width, height, previewAspect);
uiCam.worldLeft = rect.left;
uiCam.worldRight = rect.right;
diff --git a/sdk/src/ecs/bridge/nativeRuntime.ts b/sdk/src/ecs/bridge/nativeRuntime.ts
index 8dfeb5354..c505394ee 100644
--- a/sdk/src/ecs/bridge/nativeRuntime.ts
+++ b/sdk/src/ecs/bridge/nativeRuntime.ts
@@ -39,6 +39,7 @@ import { setRendererBackend } from '../../render/renderer';
import { RenderPipeline } from '../../render/renderPipeline';
import { cameraPlugin } from '../../camera/CameraPlugin';
import { UICameraInfo } from '../../ui/core/ui-camera-info';
+import { ScreenLayout } from '../../ui/core/screen-layout';
import { DEFAULT_UI_CAMERA_INFO } from '../../app/corePlugin';
import { uiPlugin } from '../../ui/ui-plugin';
import { SpinePlugin } from '../../spine';
@@ -239,6 +240,7 @@ function installNativeRenderer(app: App, scope: Record<string, unknown>): void {
setRendererBackend(createNativeRendererBackend(scope));
app.setPipeline(new RenderPipeline());
app.insertResource(UICameraInfo, { ...DEFAULT_UI_CAMERA_INFO });
+ app.insertResource(ScreenLayout, { ...ScreenLayout._default });
// The viewport is read per frame, so a rotation reaches the projection without
// anyone pushing a resize event.
app.addPlugin(cameraPlugin(() => nativeSurfaceSize(scope)));
diff --git a/sdk/src/ui/behavior/plugin.ts b/sdk/src/ui/behavior/plugin.ts
index 900a97272..117a30dde 100644
--- a/sdk/src/ui/behavior/plugin.ts
+++ b/sdk/src/ui/behavior/plugin.ts
@@ -10,6 +10,7 @@ import type { Entity, Vec2 } from '../../types';
import { UIEvents, UIEventQueue } from '../core/events';
import { UICameraInfo, type UICameraData } from '../core/ui-camera-info';
+import { ScreenLayout, type ScreenLayoutData } from '../core/screen-layout';
import { PluginName, SystemLabel } from '../../ecs/systemLabels';
import { ListView, ListViewRegistry } from '../collection/list-view';
import { ScrollContainer, ScrollContainerRegistry } from '../collection/scroll-container';
@@ -223,8 +224,8 @@ export class UIBehaviorPlugin implements Plugin {
let lastOffset: Vec2 = { x: 0, y: 0 };
app.addSystemToSchedule(Schedule.PreUpdate, defineSystem(
- [Res(Input), Res(UICameraInfo), Res(Time)],
- (input: InputState, camera: UICameraData, time: TimeData) => {
+ [Res(Input), Res(UICameraInfo), Res(Time), Res(ScreenLayout)],
+ (input: InputState, camera: UICameraData, time: TimeData, layout: ScreenLayoutData) => {
if (!camera.valid) return;
const worldMouse = { x: camera.worldMouseX, y: camera.worldMouseY };
@@ -254,8 +255,7 @@ export class UIBehaviorPlugin implements Plugin {
} else {
const dx = worldMouse.x - grabStartWorld.x;
const dy = worldMouse.y - grabStartWorld.y;
- const worldSpan = camera.worldRight - camera.worldLeft;
- const screenDist = Math.hypot(dx, dy) * (worldSpan !== 0 ? camera.vpW / worldSpan : 1);
+ const screenDist = Math.hypot(dx, dy) * layout.scale;
if (screenDist >= SCROLL_DRAG_THRESHOLD_PX) {
activeEntity = pendingEntity;
pendingEntity = null;
diff --git a/sdk/src/ui/input/drag.ts b/sdk/src/ui/input/drag.ts
index 4bbe5dece..d1bce2794 100644
--- a/sdk/src/ui/input/drag.ts
+++ b/sdk/src/ui/input/drag.ts
@@ -18,6 +18,7 @@ import { UIInteraction } from './interactable';
import type { UIInteractionData } from './interactable';
import { UIEvents, UIEventQueue } from '../core/events';
import { UICameraInfo } from '../core/ui-camera-info';
+import { ScreenLayout, type ScreenLayoutData } from '../core/screen-layout';
import type { UICameraData } from '../core/ui-camera-info';
import { playModeOnly } from '../../ecs/env';
import { getEntityDepth } from '../util/helpers';
@@ -91,8 +92,8 @@ export class DragPlugin implements Plugin {
let activeEntity: Entity | null = null;
app.addSystemToSchedule(Schedule.PreUpdate, defineSystem(
- [Res(Input), Res(UICameraInfo)],
- (input: InputState, camera: UICameraData) => {
+ [Res(Input), Res(UICameraInfo), Res(ScreenLayout)],
+ (input: InputState, camera: UICameraData, layout: ScreenLayoutData) => {
if (!camera.valid) return;
const worldMouse = { x: camera.worldMouseX, y: camera.worldMouseY };
@@ -157,7 +158,7 @@ export class DragPlugin implements Plugin {
const dist = Math.sqrt(dx * dx + dy * dy);
const draggable = world.get(pendingEntity, Draggable) as DraggableData;
- const screenDist = dist * (camera.vpW / (camera.worldRight - camera.worldLeft));
+ const screenDist = dist * layout.scale;
if (screenDist >= draggable.dragThreshold) {
activeEntity = pendingEntity;
pendingEntity = null;
diff --git a/sdk/src/ui/layout/safe-area.ts b/sdk/src/ui/layout/safe-area.ts
index fb6c9b307..81c8e4d8c 100644
--- a/sdk/src/ui/layout/safe-area.ts
+++ b/sdk/src/ui/layout/safe-area.ts
@@ -11,6 +11,7 @@ import { defineComponent, registerComponent } from '../../ecs/component';
import { Res } from '../../ecs/resource';
import { defineSystem, Schedule } from '../../ecs/system';
import { UICameraInfo } from '../core/ui-camera-info';
+import { ScreenLayout, type ScreenLayoutData } from '../core/screen-layout';
import type { UICameraData } from '../core/ui-camera-info';
import { UINode } from '../core/ui-node';
import type { UINodeData } from '../core/ui-node';
@@ -107,7 +108,7 @@ export class SafeAreaPlugin implements Plugin {
let cachedInsets: SafeAreaInsets = { top: 0, bottom: 0, left: 0, right: 0 };
let stale = true;
let prevScreenH = 0;
- let prevWorldH = 0;
+ let prevLayoutH = 0;
if (isWeChat()) {
const g = globalThis as unknown as WxGlobal;
@@ -118,14 +119,14 @@ export class SafeAreaPlugin implements Plugin {
}
app.addSystemToSchedule(Schedule.PreUpdate, defineSystem(
- [Res(UICameraInfo)],
- (camera: UICameraData) => {
- if (!camera.valid || camera.screenH === 0) return;
-
- const worldH = camera.worldTop - camera.worldBottom;
- if (camera.screenH !== prevScreenH || worldH !== prevWorldH) {
- prevScreenH = camera.screenH;
- prevWorldH = worldH;
+ [Res(ScreenLayout)],
+ (layout: ScreenLayoutData) => {
+ if (!layout.valid || layout.viewportH === 0) return;
+
+ const layoutH = layout.top - layout.bottom;
+ if (layout.viewportH !== prevScreenH || layoutH !== prevLayoutH) {
+ prevScreenH = layout.viewportH;
+ prevLayoutH = layoutH;
stale = true;
}
@@ -138,7 +139,7 @@ export class SafeAreaPlugin implements Plugin {
}
const dpr = platformDevicePixelRatio();
- const insetScale = isWeChat() ? (worldH / camera.screenH) : (dpr * worldH / camera.screenH);
+ const insetScale = isWeChat() ? (1 / layout.scale) : (dpr / layout.scale);
// UINode targets (CSS box): safe-area insets map to the node's
// absolute insets. The node is expected to be position:Absolute.
diff --git a/sdk/src/ui/text/plugin.ts b/sdk/src/ui/text/plugin.ts
index ad344d0fc..280dc15e2 100644
--- a/sdk/src/ui/text/plugin.ts
+++ b/sdk/src/ui/text/plugin.ts
@@ -124,6 +124,9 @@ export class TextPlugin implements Plugin {
// Design→device scale beyond DPR (vpW is device px); the bitmap
// atlas folds it into rasterization. Resolved per frame — the
// camera plugin may build after this one.
+ // Camera-scaled ON PURPOSE, and correct for WORLD text: zooming in
+ // rasterizes sharper. Screen text wants the layout domain, which
+ // needs the two told apart — the collection split, not here.
const uiCamera = app.getResource(UICameraInfo) as UICameraData | undefined;
const dpr = platformDevicePixelRatio();
const contentScale = glyphContentScale(uiCamera, dpr);
diff --git a/tools/check-screen-domain.mjs b/tools/check-screen-domain.mjs
new file mode 100644
index 000000000..076378d8a
--- /dev/null
+++ b/tools/check-screen-domain.mjs
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file check-screen-domain.mjs — who is still asking a camera where the screen is.
+ *
+ * Screen-space UI has one coordinate authority (`ScreenLayoutData`) and a camera
+ * is not part of it. `UICameraInfo.worldLeft/Right/Top/Bottom` is a camera's
+ * world rect; under a 2D camera it happens to be the screen, which is the
+ * equivalence a 3D camera breaks.
+ *
+ * Migrating the readers one at a time is only safe if the set is CLOSED, so this
+ * lists every one and fails on an undeclared reader. A reader that still belongs
+ * to the camera says so here, with the reason it is not screen-space.
+ *
+ * node tools/check-screen-domain.mjs
+ */
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { execFileSync } from 'node:child_process';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const FIELD = /\b(?:worldLeft|worldRight|worldTop|worldBottom)\b/;
+
+/**
+ * The readers that may still exist, and why each is not the screen. Removing an
+ * entry is the migration; adding one needs a reason that survives being read
+ * aloud next to "a camera does not know where the screen is".
+ */
+const DECLARED = new Map([
+ ['sdk/src/ui/core/ui-camera-info.ts', 'declares the fields'],
+ ['sdk/src/camera/CameraPlugin.ts', 'publishes them from the camera'],
+ ['sdk/src/app/corePlugin.ts', 'zeroes the resource at startup'],
+ ['sdk/src/ecs/resourceShapes.ts', 'declares the resource shape'],
+ ['sdk/src/camera/Camera.ts', 'getWorldBounds is a camera query, not a UI one'],
+ ['sdk/src/ui/layout/layout.ts',
+ 'still hands the box to uiLayout_update — moves with the collection split'],
+ ['sdk/src/ui/text/plugin.ts',
+ 'glyph raster scale is camera-correct for WORLD text; screen text needs the '
+ + 'two told apart, which is the collection split'],
+]);
+
+const files = execFileSync('git', ['ls-files', 'sdk/src'], { cwd: ROOT, encoding: 'utf8' })
+ .split('\n').filter((f) => f.endsWith('.ts'));
+
+const found = [];
+for (const file of files) {
+ const text = readFileSync(path.join(ROOT, file), 'utf8');
+ const lines = text.split('\n');
+ for (const [i, line] of lines.entries()) {
+ if (!FIELD.test(line)) continue;
+ found.push({ file, line: i + 1, text: line.trim() });
+ }
+}
+
+const readers = new Set(found.map((f) => f.file));
+const undeclared = [...readers].filter((f) => !DECLARED.has(f)).sort();
+const stale = [...DECLARED.keys()].filter((f) => !readers.has(f)).sort();
+
+for (const f of undeclared) {
+ console.log(`✗ ${f} reads a camera's world rect for what looks like screen space`);
+ for (const hit of found.filter((h) => h.file === f)) {
+ console.log(` ${hit.line}: ${hit.text.slice(0, 100)}`);
+ }
+}
+// A declaration nothing needs any more is the migration finishing; it must be
+// deleted, or the list stops describing the tree it claims to.
+for (const f of stale) console.log(`✗ ${f} is declared but no longer reads one — drop the entry`);
+
+const bad = undeclared.length + stale.length;
+console.log(bad === 0
+ ? `check-screen-domain: ${readers.size} declared reader(s) of the camera's world rect, `
+ + 'and no undeclared one'
+ : `check-screen-domain: ${bad} finding(s)`);
+process.exit(bad === 0 ? 0 : 1);
diff --git a/tools/gates.mjs b/tools/gates.mjs
index 04dabeb5e..617db0670 100644
--- a/tools/gates.mjs
+++ b/tools/gates.mjs
@@ -250,6 +250,7 @@ export const GATES = [
// the temp tree a test made comes back ENOTEMPTY at random. Node's retry loop
// is right there and defaults to OFF.
{ id: 'script-failure', run: 'node tools/check-script-failure.mjs' },
+ { id: 'screen-domain', run: 'node tools/check-screen-domain.mjs' },
{ id: 'rm-retries', run: 'node tools/check-rm-retries.mjs' },
{ id: 'examples', run: 'node build-tools/cli.js check-examples' },
{ id: 'documents', run: 'node build-tools/cli.js validate-documents' },