Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

fix(ui): invert rendered transforms for virtual joysticks

yhh647deb3fb947
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 48a8b384a..7a1941071 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,8 @@ published separately; it ships inside the editor.
### Fixed
+- **Virtual joysticks follow the rendered transform through sheared UI hierarchies.** Pointer coordinates use the complete 2D render matrix instead of decomposed rotation and scale, preserving local direction and strength under rotated, nonuniform ancestors and keeping follow-mode origins aligned. Singular or nonfinite transforms release capture until a fresh press. Dragging shares the same inverse-basis reader; its scratch allocation is reused, refreshed after heap growth and freed by its owning core.
+
- **Virtual controls share each sample's pointer projection.** Stable buttons and joysticks group held pointers and completed taps once instead of cloning and scanning every pointer for every control. A change callback invalidates the grouping before later controls run, preserving capture cancellation and concurrent move/fire behavior.
- **Virtual controls stop when their authored UI is hidden.** Buttons and joysticks read current display ancestry instead of the previous layout's hidden cache. Hiding a captured control releases its source on the next sample and showing it requires a fresh press; new presses on shown controls are accepted before layout catches up. Change callbacks invalidate shared visibility reads before later controls are sampled.
diff --git a/sdk/src/ui/behavior/virtual-controls.ts b/sdk/src/ui/behavior/virtual-controls.ts
index b1e1e2d88..cd99f466a 100644
--- a/sdk/src/ui/behavior/virtual-controls.ts
+++ b/sdk/src/ui/behavior/virtual-controls.ts
@@ -16,6 +16,7 @@ import { UIControlPointers, type ControlPointer } from '../input/control-pointer
import { isScreenEntity } from '../util/ui-pick';
import { walkParentChain } from '../util/helpers';
import { createLiveVisibilityQuery } from '../util/live-visibility';
+import { createLocalTransformReader } from '../util/local-transform';
export enum VirtualJoystickMode { Fixed = 0, Follow = 1 }
export interface VirtualJoystickData {
@@ -60,6 +61,7 @@ const NO_HELD_POINTERS: readonly ControlPointer[] = [];
export function createVirtualControlSystem(world: World, events: UIEventQueue) {
const states = new Map<Entity, ControlState>();
const engine = worldEngineApi(world); const registry = world.getCppRegistry();
+ const localTransform = createLocalTransformReader(world);
let input: InputState | null = null;
const release = (entity: Entity) => {
const state = states.get(entity); if (!state) return;
@@ -84,15 +86,19 @@ export function createVirtualControlSystem(world: World, events: UIEventQueue) {
thumb: { x: 0, y: 0 }, value: { x: 0, y: 0 } }; states.set(entity, state); }
return state;
};
- const point = (entity: Entity, pointer: ControlPointer): Vec2 => {
- const t = world.get(entity, Transform);
+ const point = (entity: Entity, pointer: ControlPointer): Vec2 | null => {
const screen = isScreenEntity(world, entity);
- const x = (screen ? pointer.layoutX : pointer.worldX) - t.worldPosition.x;
- const y = (screen ? pointer.layoutY : pointer.worldY) - t.worldPosition.y;
+ const px = screen ? pointer.layoutX : pointer.worldX, py = screen ? pointer.layoutY : pointer.worldY;
+ const resolved = localTransform.point(entity, px, py);
+ if (resolved !== undefined) return resolved;
+ const t = world.get(entity, Transform);
+ if (!t.worldScale.x || !t.worldScale.y) return null;
+ const x = px - t.worldPosition.x, y = py - t.worldPosition.y;
const q = t.worldRotation;
const angle = Math.atan2(2 * (q.w * q.z + q.x * q.y), 1 - 2 * (q.y * q.y + q.z * q.z));
- return { x: (x * Math.cos(angle) + y * Math.sin(angle)) / (t.worldScale.x || 1),
- y: (-x * Math.sin(angle) + y * Math.cos(angle)) / (t.worldScale.y || 1) };
+ const local = { x: (x * Math.cos(angle) + y * Math.sin(angle)) / t.worldScale.x,
+ y: (-x * Math.sin(angle) + y * Math.cos(angle)) / t.worldScale.y };
+ return Number.isFinite(local.x) && Number.isFinite(local.y) ? local : null;
};
const positionChild = (entity: Entity, child: Entity, offset: Vec2) => {
if (!world.valid(child) || !world.has(child, UINode)) return;
@@ -140,21 +146,24 @@ export function createVirtualControlSystem(world: World, events: UIEventQueue) {
const data = world.get(entity, VirtualJoystick);
id = data.inputId;
let pointer = held.find(p => p.id === state.pointer);
+ let fresh = false;
if (!pointer) {
state.pointer = null;
pointer = held.find(p => p.pressed);
- if (pointer) {
+ fresh = !!pointer;
+ }
+ const local = pointer ? point(entity, pointer) : null;
+ if (pointer && local) {
+ if (fresh) {
state.pointer = pointer.id;
- state.origin = data.mode === VirtualJoystickMode.Follow ? point(entity, pointer) : { x: 0, y: 0 };
+ state.origin = data.mode === VirtualJoystickMode.Follow ? local : { x: 0, y: 0 };
}
- }
- if (pointer) {
- const local = point(entity, pointer);
const dx = local.x - state.origin.x, dy = local.y - state.origin.y;
value = joystickAxis(dx, dy, data.radius, data.deadZone);
const ratio = Math.min(1, Math.max(1, data.radius) / (Math.hypot(dx, dy) || 1));
state.thumb = { x: dx * ratio, y: dy * ratio };
} else {
+ state.pointer = null;
const spring = Math.exp(-Math.max(0, data.returnSpeed) * Math.max(0, time.delta));
state.thumb = { x: state.thumb.x * spring, y: state.thumb.y * spring };
if (Math.hypot(state.thumb.x, state.thumb.y) < 0.1 || data.returnSpeed <= 0) {
@@ -180,6 +189,6 @@ export function createVirtualControlSystem(world: World, events: UIEventQueue) {
}
}
}, { name: SystemLabel.VirtualControls }),
- dispose: () => { offDespawn(); for (const entity of states.keys()) release(entity); },
+ dispose: () => { offDespawn(); for (const entity of states.keys()) release(entity); localTransform.dispose(); },
};
}
diff --git a/sdk/src/ui/input/drag.ts b/sdk/src/ui/input/drag.ts
index 862e07a2f..aa23ed0b4 100644
--- a/sdk/src/ui/input/drag.ts
+++ b/sdk/src/ui/input/drag.ts
@@ -10,7 +10,7 @@ import { defineSystem, Schedule } from '../../ecs/system';
import { Res } from '../../ecs/resource';
import { Input } from '../../input/input';
import { primaryPointerCancellationVersion } from '../../input/pointerCancellation';
-import { engineApi, type EngineApi } from '../../ecs/bridge/engineApi';
+import { engineApi } from '../../ecs/bridge/engineApi';
import type { InputState } from '../../input/input';
import type { Entity } from '../../types';
import type { World } from '../../ecs/world';
@@ -27,6 +27,7 @@ import type { UICameraData } from '../core/ui-camera-info';
import { playModeOnly } from '../../ecs/env';
import { getEntityDepth } from '../util/helpers';
import { createLiveVisibilityQuery } from '../util/live-visibility';
+import { createLocalTransformReader } from '../util/local-transform';
import { SystemLabel, PluginName } from '../../ecs/systemLabels';
import { quaternionToAngle2D } from '../util/math';
@@ -111,34 +112,9 @@ export class DragPlugin implements Plugin {
const events = app.getResource(UIEvents) as UIEventQueue;
const engine = engineApi(app);
const registry = world.getCppRegistry();
- let matrixOwner: EngineApi | null = null;
- let matrixPointer = 0;
- let matrixHeap: Float32Array | null = null;
- const releaseMatrix = (): void => {
- const pointer = matrixPointer, owner = matrixOwner;
- matrixPointer = 0;
- matrixOwner = null;
- matrixHeap = null;
- if (pointer) owner?._free?.(pointer);
- };
- const readParentDelta = (parent: Entity, dx: number, dy: number): { x: number; y: number } | null | undefined => {
- const api = engineApi(app), registry = world.getCppRegistry();
- if (!api?.transform_copyFlatRenderMatrix || !api.HEAPU8 || !api._malloc || !api._free || !registry) return undefined;
- if (matrixOwner !== api) { releaseMatrix();matrixOwner = api; }
- if (!matrixPointer) {
- matrixPointer = api._malloc(64);
- if (!matrixPointer) throw new Error('Could not allocate drag transform scratch.');
- }
- if (!api.transform_copyFlatRenderMatrix(registry, parent, matrixPointer)) return null;
- const heap = api.HEAPU8;
- if (matrixHeap?.buffer !== heap.buffer) matrixHeap = new Float32Array(heap.buffer);
- const at = (heap.byteOffset + matrixPointer) / 4;
- const a = matrixHeap[at], b = matrixHeap[at + 1], c = matrixHeap[at + 4], d = matrixHeap[at + 5];
- const det = a * d - b * c;
- if (!Number.isFinite(det) || det === 0) return null;
- const x = (d * dx - c * dy) / det, y = (a * dy - b * dx) / det;
- return Number.isFinite(x) && Number.isFinite(y) ? { x, y } : null;
- };
+ const localTransform = createLocalTransformReader(world);
+ const releaseMatrix = localTransform.dispose;
+ const readParentDelta = localTransform.vector;
const eligible = (entity: Entity): boolean => world.valid(entity) && world.isEntityActive(entity)
&& !world.isDestructionPending(entity)
&& world.has(entity, Draggable) && world.get(entity, Draggable).enabled
diff --git a/sdk/src/ui/util/local-transform.ts b/sdk/src/ui/util/local-transform.ts
new file mode 100644
index 000000000..9eca7108d
--- /dev/null
+++ b/sdk/src/ui/util/local-transform.ts
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+import type { World } from '../../ecs/world';
+import type { Entity, Vec2 } from '../../types';
+import { worldEngineApi, type EngineApi } from '../../ecs/bridge/engineApi';
+
+/** Invert the rendered 2D basis, including shear. Undefined permits a legacy
+ * fallback; null rejects a missing, singular or nonfinite transform. */
+export function createLocalTransformReader(world: World) {
+ let owner: EngineApi | null = null, pointer = 0;
+ let matrixHeap: Float32Array | null = null;
+ const dispose = (): void => {
+ const oldPointer = pointer, oldOwner = owner;
+ pointer = 0; owner = null; matrixHeap = null;
+ if (oldPointer) oldOwner?._free?.(oldPointer);
+ };
+ const read = (entity: Entity, x: number, y: number, translate: boolean): Vec2 | null | undefined => {
+ if (!Number.isFinite(x) || !Number.isFinite(y)) return null;
+ const api = worldEngineApi(world), registry = world.getCppRegistry();
+ if (owner !== api) { dispose(); owner = api; }
+ if (!api?.transform_copyFlatRenderMatrix || !api.HEAPU8 || !api._malloc || !api._free || !registry) return undefined;
+ if (!pointer) {
+ pointer = api._malloc(64);
+ if (!pointer) throw new Error('Could not allocate UI transform scratch.');
+ }
+ if (!api.transform_copyFlatRenderMatrix(registry, entity, pointer)) return null;
+ const heap = api.HEAPU8;
+ if (matrixHeap?.buffer !== heap.buffer) matrixHeap = new Float32Array(heap.buffer);
+ const at = (heap.byteOffset + pointer) / 4;
+ const a = matrixHeap[at], b = matrixHeap[at + 1], c = matrixHeap[at + 4], d = matrixHeap[at + 5];
+ const det = a * d - b * c;
+ if (!Number.isFinite(det) || det === 0) return null;
+ if (translate) { x -= matrixHeap[at + 12]; y -= matrixHeap[at + 13]; }
+ const localX = (d * x - c * y) / det, localY = (a * y - b * x) / det;
+ return Number.isFinite(localX) && Number.isFinite(localY) ? { x: localX, y: localY } : null;
+ };
+ return {
+ point: (entity: Entity, x: number, y: number) => read(entity, x, y, true),
+ vector: (entity: Entity, x: number, y: number) => read(entity, x, y, false),
+ dispose,
+ };
+}
diff --git a/sdk/tests/ui-local-transform.test.ts b/sdk/tests/ui-local-transform.test.ts
new file mode 100644
index 000000000..dadc27c11
--- /dev/null
+++ b/sdk/tests/ui-local-transform.test.ts
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+import { describe, expect, it, vi } from 'vitest';
+import { createLocalTransformReader } from '../src/ui/util/local-transform';
+import type { World } from '../src/ecs/world';
+import type { Entity } from '../src/types';
+
+function host() {
+ const matrix = [2, 0, 0, 0, 1, 3, 0, 0, 0, 0, 1, 0, 10, -5, 0, 1];
+ const api = {
+ HEAPU8: new Uint8Array(256), _malloc: vi.fn(() => 16), _free: vi.fn(),
+ transform_copyFlatRenderMatrix: vi.fn((_registry: unknown, _entity: number, pointer: number) => {
+ new Float32Array(api.HEAPU8.buffer, pointer, 16).set(matrix); return true;
+ }),
+ };
+ return { api, matrix };
+}
+const entity = 1 as Entity;
+
+describe('rendered local transform reader', () => {
+ it('inverts shear and translation for points, and only the basis for vectors', () => {
+ const { api } = host(), reader = createLocalTransformReader({ getWasmModule: () => api, getCppRegistry: () => ({}) } as unknown as World);
+ expect(reader.point(entity, 24, 7)).toEqual({ x: 5, y: 4 });
+ expect(reader.vector(entity, 14, 12)).toEqual({ x: 5, y: 4 });
+ expect(api._malloc).toHaveBeenCalledTimes(1);
+ reader.dispose(); reader.dispose(); expect(api._free).toHaveBeenCalledTimes(1);
+ });
+ it('reads the current heap after growth instead of the retired typed view', () => {
+ const { api, matrix } = host(), reader = createLocalTransformReader({ getWasmModule: () => api, getCppRegistry: () => ({}) } as unknown as World);
+ expect(reader.point(entity, 24, 7)).toEqual({ x: 5, y: 4 });
+ api.HEAPU8 = new Uint8Array(512); matrix[12] = 20;
+ expect(reader.point(entity, 34, 7)).toEqual({ x: 5, y: 4 });
+ expect(api._malloc).toHaveBeenCalledTimes(1); reader.dispose();
+ });
+ it('releases scratch through its original host when the world changes cores', () => {
+ const first = host(), second = host(); let api = first.api;
+ const reader = createLocalTransformReader({ getWasmModule: () => api, getCppRegistry: () => ({}) } as unknown as World);
+ reader.point(entity, 24, 7); api = second.api; reader.point(entity, 24, 7);
+ expect(first.api._free).toHaveBeenCalledTimes(1); expect(first.api._free).toHaveBeenCalledWith(16);
+ expect(second.api._free).not.toHaveBeenCalled(); reader.dispose();
+ expect(second.api._free).toHaveBeenCalledTimes(1); expect(second.api._free).toHaveBeenCalledWith(16);
+ });
+ it('rejects singular, missing and nonfinite transforms instead of requesting fallback', () => {
+ const { api, matrix } = host(), reader = createLocalTransformReader({ getWasmModule: () => api, getCppRegistry: () => ({}) } as unknown as World);
+ matrix[0] = 0; expect(reader.point(entity, 24, 7)).toBeNull();
+ matrix[0] = Infinity; expect(reader.vector(entity, 14, 12)).toBeNull();
+ api.transform_copyFlatRenderMatrix.mockReturnValue(false); expect(reader.point(entity, 24, 7)).toBeNull();
+ expect(reader.point(entity, NaN, 7)).toBeNull(); reader.dispose();
+ });
+ it('permits the legacy path only when the host lacks the matrix capability', () => {
+ const reader = createLocalTransformReader({ getWasmModule: () => ({}), getCppRegistry: () => ({}) } as unknown as World);
+ expect(reader.point(entity, 24, 7)).toBeUndefined(); reader.dispose();
+ });
+});
diff --git a/sdk/tests/virtual-controls.test.ts b/sdk/tests/virtual-controls.test.ts
index 363bf7633..f90aa8ea4 100644
--- a/sdk/tests/virtual-controls.test.ts
+++ b/sdk/tests/virtual-controls.test.ts
@@ -44,6 +44,15 @@ function fixture() {
return { world, input, pointers, events, tick, point, add, remove, dispose: control.dispose };
}
describe('virtual control ownership and InputMap', () => {
+ it('releases a collapsed legacy transform without rearming an already held pointer', () => {
+ const f = fixture(); f.add(1, VirtualJoystick); f.point(1, 1, 50, 0, true); f.tick();
+ const transform = f.world.get(1 as Entity, Transform);
+ f.world.insert(1 as Entity, Transform, { ...transform, worldScale: { x: 0, y: 1, z: 1 } });
+ f.point(1, 1, 50, 0); f.tick(); expect(f.input.getVirtual('move')).toEqual({ x: 0, y: 0 });
+ f.world.insert(1 as Entity, Transform, transform); f.tick();
+ expect(f.input.getVirtual('move')).toEqual({ x: 0, y: 0 });
+ f.point(1, 1, 50, 0, true); f.tick(); expect(f.input.getVirtual('move').x).toBe(1); f.dispose();
+ });
it('does not rescan every pointer for every idle or held control', () => {
const f = fixture();
for (let entity = 1; entity <= 100; entity++) f.add(entity, VirtualButton);
diff --git a/sdk/tests/virtual-joystick-transform-wasm.test.ts b/sdk/tests/virtual-joystick-transform-wasm.test.ts
new file mode 100644
index 000000000..84225306e
--- /dev/null
+++ b/sdk/tests/virtual-joystick-transform-wasm.test.ts
@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { App } from '../src/app/app';
+import { AppContext, setDefaultContext } from '../src/ecs/context';
+import { Transform } from '../src/ecs/component';
+import { InputState } from '../src/input/input';
+import { UINode } from '../src/ui/core/ui-node';
+import { UIEventQueue } from '../src/ui/core/events';
+import { Interactable } from '../src/ui/input/interactable';
+import { createVirtualControlSystem, VirtualJoystick, VirtualJoystickMode } from '../src/ui/behavior/virtual-controls';
+import type { ControlPointer } from '../src/ui/input/control-pointers';
+import type { CppRegistry, ESEngineModule } from '../src/wasm';
+import type { Entity } from '../src/types';
+import { HAS_WASM, loadWasmModule } from './helpers/loadWasm';
+
+describe.skipIf(!HAS_WASM)('virtual joystick coordinates on real WASM', () => {
+ let engine: ESEngineModule, registry: CppRegistry, app: App, input: InputState;
+ let controls: ReturnType<typeof createVirtualControlSystem>, grand: Entity, parent: Entity, entity: Entity;
+ const pointers = new Map<number, ControlPointer>();
+ beforeAll(async () => { engine = await loadWasmModule(); });
+ beforeEach(() => {
+ setDefaultContext(new AppContext()); registry = new engine.Registry();
+ app = App.new(); app.connectCpp(registry, engine); input = new InputState(); pointers.clear();
+ grand = app.world.spawn(); parent = app.world.spawn(); entity = app.world.spawn();
+ app.world.insert(grand, Transform, { position: { x: 7, y: -3, z: 0 }, scale: { x: 2, y: 1, z: 1 } });
+ app.world.insert(parent, Transform, { position: { x: -30, y: 15, z: 0 },
+ rotation: { x: 0, y: 0, z: Math.sin(Math.PI / 8), w: Math.cos(Math.PI / 8) } });
+ app.world.insert(entity, Transform, { position: { x: 10, y: 20, z: 0 } });
+ app.world.setParent(parent, grand); app.world.setParent(entity, parent);
+ app.world.insert(entity, UINode); app.world.insert(entity, Interactable);
+ app.world.insert(entity, VirtualJoystick, { deadZone: 0, radius: 50 });
+ engine.transform_update(registry);
+ controls = createVirtualControlSystem(app.world, new UIEventQueue());
+ });
+ afterEach(() => { controls.dispose(); app.quit({ keepRenderer: true }); registry.delete(); });
+ const point = (x: number, y: number, pressed: boolean) => {
+ const c = Math.SQRT1_2, worldX = 7 + 2 * (-30 + c * (10 + x) - c * (20 + y));
+ const worldY = -3 + 15 + c * (10 + x) + c * (20 + y);
+ pointers.set(1, { id: 1, entity, pressedEntity: pressed ? entity : null, down: true, pressed,
+ layoutX: worldX, layoutY: worldY, worldX, worldY });
+ };
+ const tick = () => (controls.system._fn as (...args: unknown[]) => void)(input, { pointers }, { delta: 1 / 60 });
+ it.each([[25, 0], [0, 25], [15, 20]])('preserves local direction and strength at (%s, %s) through sheared ancestors', (x, y) => {
+ point(x, y, true); tick();
+ expect(input.getVirtual('move').x).toBeCloseTo(x / 50, 5);
+ expect(input.getVirtual('move').y).toBeCloseTo(y / 50, 5);
+ });
+ it('keeps a follow joystick origin in the same local coordinate space', () => {
+ app.world.update(entity, VirtualJoystick, data => { data.mode = VirtualJoystickMode.Follow; });
+ point(10, -5, true); tick(); expect(input.getVirtual('move')).toEqual({ x: 0, y: 0 });
+ point(10, 20, false); tick();
+ expect(input.getVirtual('move').x).toBeCloseTo(0, 5);
+ expect(input.getVirtual('move').y).toBeCloseTo(.5, 5);
+ });
+ it('releases a capture when its transform collapses and requires a fresh press after restoration', () => {
+ point(25, 0, true); tick();
+ app.world.update(grand, Transform, data => { data.scale.x = 0; }); engine.transform_update(registry);
+ point(25, 0, false); tick(); expect(input.getVirtual('move')).toEqual({ x: 0, y: 0 });
+ app.world.update(grand, Transform, data => { data.scale.x = 2; }); engine.transform_update(registry);
+ tick(); expect(input.getVirtual('move')).toEqual({ x: 0, y: 0 });
+ point(25, 0, true); tick(); expect(input.getVirtual('move').x).toBeCloseTo(.5, 5);
+ });
+});
diff --git a/tools/releaseNotes.mjs b/tools/releaseNotes.mjs
index 3d18fb4ef..58f4130cb 100644
--- a/tools/releaseNotes.mjs
+++ b/tools/releaseNotes.mjs
@@ -28,6 +28,8 @@ export const NOTES_FLOOR = 'v0.65.0';
* the reader recognises anyway.
*/
export const NOTED = {
+ 'fix(ui): invert rendered transforms for virtual joysticks':
+ { note: 'Virtual joysticks follow the rendered transform through sheared UI hierarchies.' },
'perf(ui): share virtual control pointer groups per sample':
{ note: 'Virtual controls share each sample\'s pointer projection.' },
'fix(ui): release hidden virtual controls before layout':