A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
fix(ui): invert rendered transforms for virtual joysticks
yhh647deb3fb947
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 48a8b384a..7a1941071 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -16,6 +16,8 @@ published separately; it ships inside the editor.### Fixed+- **Virtual joysticks follow the rendered transform through sheared UI hierarchies.** Pointer coordinates use the complete 2D render matrix instead of decomposed rotation and scale, preserving local direction and strength under rotated, nonuniform ancestors and keeping follow-mode origins aligned. Singular or nonfinite transforms release capture until a fresh press. Dragging shares the same inverse-basis reader; its scratch allocation is reused, refreshed after heap growth and freed by its owning core.+- **Virtual controls share each sample's pointer projection.** Stable buttons and joysticks group held pointers and completed taps once instead of cloning and scanning every pointer for every control. A change callback invalidates the grouping before later controls run, preserving capture cancellation and concurrent move/fire behavior.- **Virtual controls stop when their authored UI is hidden.** Buttons and joysticks read current display ancestry instead of the previous layout's hidden cache. Hiding a captured control releases its source on the next sample and showing it requires a fresh press; new presses on shown controls are accepted before layout catches up. Change callbacks invalidate shared visibility reads before later controls are sampled.diff --git a/sdk/src/ui/behavior/virtual-controls.ts b/sdk/src/ui/behavior/virtual-controls.tsindex b1e1e2d88..cd99f466a 100644--- a/sdk/src/ui/behavior/virtual-controls.ts+++ b/sdk/src/ui/behavior/virtual-controls.ts@@ -16,6 +16,7 @@ import { UIControlPointers, type ControlPointer } from '../input/control-pointerimport { isScreenEntity } from '../util/ui-pick';import { walkParentChain } from '../util/helpers';import { createLiveVisibilityQuery } from '../util/live-visibility';+import { createLocalTransformReader } from '../util/local-transform';export enum VirtualJoystickMode { Fixed = 0, Follow = 1 }export interface VirtualJoystickData {@@ -60,6 +61,7 @@ const NO_HELD_POINTERS: readonly ControlPointer[] = [];export function createVirtualControlSystem(world: World, events: UIEventQueue) {const states = new Map<Entity, ControlState>();const engine = worldEngineApi(world); const registry = world.getCppRegistry();+ const localTransform = createLocalTransformReader(world);let input: InputState | null = null;const release = (entity: Entity) => {const state = states.get(entity); if (!state) return;@@ -84,15 +86,19 @@ export function createVirtualControlSystem(world: World, events: UIEventQueue) {thumb: { x: 0, y: 0 }, value: { x: 0, y: 0 } }; states.set(entity, state); }return state;};- const point = (entity: Entity, pointer: ControlPointer): Vec2 => {- const t = world.get(entity, Transform);+ const point = (entity: Entity, pointer: ControlPointer): Vec2 | null => {const screen = isScreenEntity(world, entity);- const x = (screen ? pointer.layoutX : pointer.worldX) - t.worldPosition.x;- const y = (screen ? pointer.layoutY : pointer.worldY) - t.worldPosition.y;+ const px = screen ? pointer.layoutX : pointer.worldX, py = screen ? pointer.layoutY : pointer.worldY;+ const resolved = localTransform.point(entity, px, py);+ if (resolved !== undefined) return resolved;+ const t = world.get(entity, Transform);+ if (!t.worldScale.x || !t.worldScale.y) return null;+ const x = px - t.worldPosition.x, y = py - t.worldPosition.y;const q = t.worldRotation;const angle = Math.atan2(2 * (q.w * q.z + q.x * q.y), 1 - 2 * (q.y * q.y + q.z * q.z));- return { x: (x * Math.cos(angle) + y * Math.sin(angle)) / (t.worldScale.x || 1),- y: (-x * Math.sin(angle) + y * Math.cos(angle)) / (t.worldScale.y || 1) };+ const local = { x: (x * Math.cos(angle) + y * Math.sin(angle)) / t.worldScale.x,+ y: (-x * Math.sin(angle) + y * Math.cos(angle)) / t.worldScale.y };+ return Number.isFinite(local.x) && Number.isFinite(local.y) ? local : null;};const positionChild = (entity: Entity, child: Entity, offset: Vec2) => {if (!world.valid(child) || !world.has(child, UINode)) return;@@ -140,21 +146,24 @@ export function createVirtualControlSystem(world: World, events: UIEventQueue) {const data = world.get(entity, VirtualJoystick);id = data.inputId;let pointer = held.find(p => p.id === state.pointer);+ let fresh = false;if (!pointer) {state.pointer = null;pointer = held.find(p => p.pressed);- if (pointer) {+ fresh = !!pointer;+ }+ const local = pointer ? point(entity, pointer) : null;+ if (pointer && local) {+ if (fresh) {state.pointer = pointer.id;- state.origin = data.mode === VirtualJoystickMode.Follow ? point(entity, pointer) : { x: 0, y: 0 };+ state.origin = data.mode === VirtualJoystickMode.Follow ? local : { x: 0, y: 0 };}- }- if (pointer) {- const local = point(entity, pointer);const dx = local.x - state.origin.x, dy = local.y - state.origin.y;value = joystickAxis(dx, dy, data.radius, data.deadZone);const ratio = Math.min(1, Math.max(1, data.radius) / (Math.hypot(dx, dy) || 1));state.thumb = { x: dx * ratio, y: dy * ratio };} else {+ state.pointer = null;const spring = Math.exp(-Math.max(0, data.returnSpeed) * Math.max(0, time.delta));state.thumb = { x: state.thumb.x * spring, y: state.thumb.y * spring };if (Math.hypot(state.thumb.x, state.thumb.y) < 0.1 || data.returnSpeed <= 0) {@@ -180,6 +189,6 @@ export function createVirtualControlSystem(world: World, events: UIEventQueue) {}}}, { name: SystemLabel.VirtualControls }),- dispose: () => { offDespawn(); for (const entity of states.keys()) release(entity); },+ dispose: () => { offDespawn(); for (const entity of states.keys()) release(entity); localTransform.dispose(); },};}diff --git a/sdk/src/ui/input/drag.ts b/sdk/src/ui/input/drag.tsindex 862e07a2f..aa23ed0b4 100644--- a/sdk/src/ui/input/drag.ts+++ b/sdk/src/ui/input/drag.ts@@ -10,7 +10,7 @@ import { defineSystem, Schedule } from '../../ecs/system';import { Res } from '../../ecs/resource';import { Input } from '../../input/input';import { primaryPointerCancellationVersion } from '../../input/pointerCancellation';-import { engineApi, type EngineApi } from '../../ecs/bridge/engineApi';+import { engineApi } from '../../ecs/bridge/engineApi';import type { InputState } from '../../input/input';import type { Entity } from '../../types';import type { World } from '../../ecs/world';@@ -27,6 +27,7 @@ import type { UICameraData } from '../core/ui-camera-info';import { playModeOnly } from '../../ecs/env';import { getEntityDepth } from '../util/helpers';import { createLiveVisibilityQuery } from '../util/live-visibility';+import { createLocalTransformReader } from '../util/local-transform';import { SystemLabel, PluginName } from '../../ecs/systemLabels';import { quaternionToAngle2D } from '../util/math';@@ -111,34 +112,9 @@ export class DragPlugin implements Plugin {const events = app.getResource(UIEvents) as UIEventQueue;const engine = engineApi(app);const registry = world.getCppRegistry();- let matrixOwner: EngineApi | null = null;- let matrixPointer = 0;- let matrixHeap: Float32Array | null = null;- const releaseMatrix = (): void => {- const pointer = matrixPointer, owner = matrixOwner;- matrixPointer = 0;- matrixOwner = null;- matrixHeap = null;- if (pointer) owner?._free?.(pointer);- };- const readParentDelta = (parent: Entity, dx: number, dy: number): { x: number; y: number } | null | undefined => {- const api = engineApi(app), registry = world.getCppRegistry();- if (!api?.transform_copyFlatRenderMatrix || !api.HEAPU8 || !api._malloc || !api._free || !registry) return undefined;- if (matrixOwner !== api) { releaseMatrix();matrixOwner = api; }- if (!matrixPointer) {- matrixPointer = api._malloc(64);- if (!matrixPointer) throw new Error('Could not allocate drag transform scratch.');- }- if (!api.transform_copyFlatRenderMatrix(registry, parent, matrixPointer)) return null;- const heap = api.HEAPU8;- if (matrixHeap?.buffer !== heap.buffer) matrixHeap = new Float32Array(heap.buffer);- const at = (heap.byteOffset + matrixPointer) / 4;- const a = matrixHeap[at], b = matrixHeap[at + 1], c = matrixHeap[at + 4], d = matrixHeap[at + 5];- const det = a * d - b * c;- if (!Number.isFinite(det) || det === 0) return null;- const x = (d * dx - c * dy) / det, y = (a * dy - b * dx) / det;- return Number.isFinite(x) && Number.isFinite(y) ? { x, y } : null;- };+ const localTransform = createLocalTransformReader(world);+ const releaseMatrix = localTransform.dispose;+ const readParentDelta = localTransform.vector;const eligible = (entity: Entity): boolean => world.valid(entity) && world.isEntityActive(entity)&& !world.isDestructionPending(entity)&& world.has(entity, Draggable) && world.get(entity, Draggable).enableddiff --git a/sdk/src/ui/util/local-transform.ts b/sdk/src/ui/util/local-transform.tsnew file mode 100644index 000000000..9eca7108d--- /dev/null+++ b/sdk/src/ui/util/local-transform.ts@@ -0,0 +1,42 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+import type { World } from '../../ecs/world';+import type { Entity, Vec2 } from '../../types';+import { worldEngineApi, type EngineApi } from '../../ecs/bridge/engineApi';++/** Invert the rendered 2D basis, including shear. Undefined permits a legacy+ * fallback; null rejects a missing, singular or nonfinite transform. */+export function createLocalTransformReader(world: World) {+ let owner: EngineApi | null = null, pointer = 0;+ let matrixHeap: Float32Array | null = null;+ const dispose = (): void => {+ const oldPointer = pointer, oldOwner = owner;+ pointer = 0; owner = null; matrixHeap = null;+ if (oldPointer) oldOwner?._free?.(oldPointer);+ };+ const read = (entity: Entity, x: number, y: number, translate: boolean): Vec2 | null | undefined => {+ if (!Number.isFinite(x) || !Number.isFinite(y)) return null;+ const api = worldEngineApi(world), registry = world.getCppRegistry();+ if (owner !== api) { dispose(); owner = api; }+ if (!api?.transform_copyFlatRenderMatrix || !api.HEAPU8 || !api._malloc || !api._free || !registry) return undefined;+ if (!pointer) {+ pointer = api._malloc(64);+ if (!pointer) throw new Error('Could not allocate UI transform scratch.');+ }+ if (!api.transform_copyFlatRenderMatrix(registry, entity, pointer)) return null;+ const heap = api.HEAPU8;+ if (matrixHeap?.buffer !== heap.buffer) matrixHeap = new Float32Array(heap.buffer);+ const at = (heap.byteOffset + pointer) / 4;+ const a = matrixHeap[at], b = matrixHeap[at + 1], c = matrixHeap[at + 4], d = matrixHeap[at + 5];+ const det = a * d - b * c;+ if (!Number.isFinite(det) || det === 0) return null;+ if (translate) { x -= matrixHeap[at + 12]; y -= matrixHeap[at + 13]; }+ const localX = (d * x - c * y) / det, localY = (a * y - b * x) / det;+ return Number.isFinite(localX) && Number.isFinite(localY) ? { x: localX, y: localY } : null;+ };+ return {+ point: (entity: Entity, x: number, y: number) => read(entity, x, y, true),+ vector: (entity: Entity, x: number, y: number) => read(entity, x, y, false),+ dispose,+ };+}diff --git a/sdk/tests/ui-local-transform.test.ts b/sdk/tests/ui-local-transform.test.tsnew file mode 100644index 000000000..dadc27c11--- /dev/null+++ b/sdk/tests/ui-local-transform.test.ts@@ -0,0 +1,54 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+import { describe, expect, it, vi } from 'vitest';+import { createLocalTransformReader } from '../src/ui/util/local-transform';+import type { World } from '../src/ecs/world';+import type { Entity } from '../src/types';++function host() {+ const matrix = [2, 0, 0, 0, 1, 3, 0, 0, 0, 0, 1, 0, 10, -5, 0, 1];+ const api = {+ HEAPU8: new Uint8Array(256), _malloc: vi.fn(() => 16), _free: vi.fn(),+ transform_copyFlatRenderMatrix: vi.fn((_registry: unknown, _entity: number, pointer: number) => {+ new Float32Array(api.HEAPU8.buffer, pointer, 16).set(matrix); return true;+ }),+ };+ return { api, matrix };+}+const entity = 1 as Entity;++describe('rendered local transform reader', () => {+ it('inverts shear and translation for points, and only the basis for vectors', () => {+ const { api } = host(), reader = createLocalTransformReader({ getWasmModule: () => api, getCppRegistry: () => ({}) } as unknown as World);+ expect(reader.point(entity, 24, 7)).toEqual({ x: 5, y: 4 });+ expect(reader.vector(entity, 14, 12)).toEqual({ x: 5, y: 4 });+ expect(api._malloc).toHaveBeenCalledTimes(1);+ reader.dispose(); reader.dispose(); expect(api._free).toHaveBeenCalledTimes(1);+ });+ it('reads the current heap after growth instead of the retired typed view', () => {+ const { api, matrix } = host(), reader = createLocalTransformReader({ getWasmModule: () => api, getCppRegistry: () => ({}) } as unknown as World);+ expect(reader.point(entity, 24, 7)).toEqual({ x: 5, y: 4 });+ api.HEAPU8 = new Uint8Array(512); matrix[12] = 20;+ expect(reader.point(entity, 34, 7)).toEqual({ x: 5, y: 4 });+ expect(api._malloc).toHaveBeenCalledTimes(1); reader.dispose();+ });+ it('releases scratch through its original host when the world changes cores', () => {+ const first = host(), second = host(); let api = first.api;+ const reader = createLocalTransformReader({ getWasmModule: () => api, getCppRegistry: () => ({}) } as unknown as World);+ reader.point(entity, 24, 7); api = second.api; reader.point(entity, 24, 7);+ expect(first.api._free).toHaveBeenCalledTimes(1); expect(first.api._free).toHaveBeenCalledWith(16);+ expect(second.api._free).not.toHaveBeenCalled(); reader.dispose();+ expect(second.api._free).toHaveBeenCalledTimes(1); expect(second.api._free).toHaveBeenCalledWith(16);+ });+ it('rejects singular, missing and nonfinite transforms instead of requesting fallback', () => {+ const { api, matrix } = host(), reader = createLocalTransformReader({ getWasmModule: () => api, getCppRegistry: () => ({}) } as unknown as World);+ matrix[0] = 0; expect(reader.point(entity, 24, 7)).toBeNull();+ matrix[0] = Infinity; expect(reader.vector(entity, 14, 12)).toBeNull();+ api.transform_copyFlatRenderMatrix.mockReturnValue(false); expect(reader.point(entity, 24, 7)).toBeNull();+ expect(reader.point(entity, NaN, 7)).toBeNull(); reader.dispose();+ });+ it('permits the legacy path only when the host lacks the matrix capability', () => {+ const reader = createLocalTransformReader({ getWasmModule: () => ({}), getCppRegistry: () => ({}) } as unknown as World);+ expect(reader.point(entity, 24, 7)).toBeUndefined(); reader.dispose();+ });+});diff --git a/sdk/tests/virtual-controls.test.ts b/sdk/tests/virtual-controls.test.tsindex 363bf7633..f90aa8ea4 100644--- a/sdk/tests/virtual-controls.test.ts+++ b/sdk/tests/virtual-controls.test.ts@@ -44,6 +44,15 @@ function fixture() {return { world, input, pointers, events, tick, point, add, remove, dispose: control.dispose };}describe('virtual control ownership and InputMap', () => {+ it('releases a collapsed legacy transform without rearming an already held pointer', () => {+ const f = fixture(); f.add(1, VirtualJoystick); f.point(1, 1, 50, 0, true); f.tick();+ const transform = f.world.get(1 as Entity, Transform);+ f.world.insert(1 as Entity, Transform, { ...transform, worldScale: { x: 0, y: 1, z: 1 } });+ f.point(1, 1, 50, 0); f.tick(); expect(f.input.getVirtual('move')).toEqual({ x: 0, y: 0 });+ f.world.insert(1 as Entity, Transform, transform); f.tick();+ expect(f.input.getVirtual('move')).toEqual({ x: 0, y: 0 });+ f.point(1, 1, 50, 0, true); f.tick(); expect(f.input.getVirtual('move').x).toBe(1); f.dispose();+ });it('does not rescan every pointer for every idle or held control', () => {const f = fixture();for (let entity = 1; entity <= 100; entity++) f.add(entity, VirtualButton);diff --git a/sdk/tests/virtual-joystick-transform-wasm.test.ts b/sdk/tests/virtual-joystick-transform-wasm.test.tsnew file mode 100644index 000000000..84225306e--- /dev/null+++ b/sdk/tests/virtual-joystick-transform-wasm.test.ts@@ -0,0 +1,64 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';+import { App } from '../src/app/app';+import { AppContext, setDefaultContext } from '../src/ecs/context';+import { Transform } from '../src/ecs/component';+import { InputState } from '../src/input/input';+import { UINode } from '../src/ui/core/ui-node';+import { UIEventQueue } from '../src/ui/core/events';+import { Interactable } from '../src/ui/input/interactable';+import { createVirtualControlSystem, VirtualJoystick, VirtualJoystickMode } from '../src/ui/behavior/virtual-controls';+import type { ControlPointer } from '../src/ui/input/control-pointers';+import type { CppRegistry, ESEngineModule } from '../src/wasm';+import type { Entity } from '../src/types';+import { HAS_WASM, loadWasmModule } from './helpers/loadWasm';++describe.skipIf(!HAS_WASM)('virtual joystick coordinates on real WASM', () => {+ let engine: ESEngineModule, registry: CppRegistry, app: App, input: InputState;+ let controls: ReturnType<typeof createVirtualControlSystem>, grand: Entity, parent: Entity, entity: Entity;+ const pointers = new Map<number, ControlPointer>();+ beforeAll(async () => { engine = await loadWasmModule(); });+ beforeEach(() => {+ setDefaultContext(new AppContext()); registry = new engine.Registry();+ app = App.new(); app.connectCpp(registry, engine); input = new InputState(); pointers.clear();+ grand = app.world.spawn(); parent = app.world.spawn(); entity = app.world.spawn();+ app.world.insert(grand, Transform, { position: { x: 7, y: -3, z: 0 }, scale: { x: 2, y: 1, z: 1 } });+ app.world.insert(parent, Transform, { position: { x: -30, y: 15, z: 0 },+ rotation: { x: 0, y: 0, z: Math.sin(Math.PI / 8), w: Math.cos(Math.PI / 8) } });+ app.world.insert(entity, Transform, { position: { x: 10, y: 20, z: 0 } });+ app.world.setParent(parent, grand); app.world.setParent(entity, parent);+ app.world.insert(entity, UINode); app.world.insert(entity, Interactable);+ app.world.insert(entity, VirtualJoystick, { deadZone: 0, radius: 50 });+ engine.transform_update(registry);+ controls = createVirtualControlSystem(app.world, new UIEventQueue());+ });+ afterEach(() => { controls.dispose(); app.quit({ keepRenderer: true }); registry.delete(); });+ const point = (x: number, y: number, pressed: boolean) => {+ const c = Math.SQRT1_2, worldX = 7 + 2 * (-30 + c * (10 + x) - c * (20 + y));+ const worldY = -3 + 15 + c * (10 + x) + c * (20 + y);+ pointers.set(1, { id: 1, entity, pressedEntity: pressed ? entity : null, down: true, pressed,+ layoutX: worldX, layoutY: worldY, worldX, worldY });+ };+ const tick = () => (controls.system._fn as (...args: unknown[]) => void)(input, { pointers }, { delta: 1 / 60 });+ it.each([[25, 0], [0, 25], [15, 20]])('preserves local direction and strength at (%s, %s) through sheared ancestors', (x, y) => {+ point(x, y, true); tick();+ expect(input.getVirtual('move').x).toBeCloseTo(x / 50, 5);+ expect(input.getVirtual('move').y).toBeCloseTo(y / 50, 5);+ });+ it('keeps a follow joystick origin in the same local coordinate space', () => {+ app.world.update(entity, VirtualJoystick, data => { data.mode = VirtualJoystickMode.Follow; });+ point(10, -5, true); tick(); expect(input.getVirtual('move')).toEqual({ x: 0, y: 0 });+ point(10, 20, false); tick();+ expect(input.getVirtual('move').x).toBeCloseTo(0, 5);+ expect(input.getVirtual('move').y).toBeCloseTo(.5, 5);+ });+ it('releases a capture when its transform collapses and requires a fresh press after restoration', () => {+ point(25, 0, true); tick();+ app.world.update(grand, Transform, data => { data.scale.x = 0; }); engine.transform_update(registry);+ point(25, 0, false); tick(); expect(input.getVirtual('move')).toEqual({ x: 0, y: 0 });+ app.world.update(grand, Transform, data => { data.scale.x = 2; }); engine.transform_update(registry);+ tick(); expect(input.getVirtual('move')).toEqual({ x: 0, y: 0 });+ point(25, 0, true); tick(); expect(input.getVirtual('move').x).toBeCloseTo(.5, 5);+ });+});diff --git a/tools/releaseNotes.mjs b/tools/releaseNotes.mjsindex 3d18fb4ef..58f4130cb 100644--- a/tools/releaseNotes.mjs+++ b/tools/releaseNotes.mjs@@ -28,6 +28,8 @@ export const NOTES_FLOOR = 'v0.65.0';* the reader recognises anyway.*/export const NOTED = {+ 'fix(ui): invert rendered transforms for virtual joysticks':+ { note: 'Virtual joysticks follow the rendered transform through sheared UI hierarchies.' },'perf(ui): share virtual control pointer groups per sample':{ note: 'Virtual controls share each sample\'s pointer projection.' },'fix(ui): release hidden virtual controls before layout':