A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(ecs): a pool is handed over as memory, so nothing calls back per entity
yhh7bdc2b1fcfe4
diff --git a/sdk/src/ecs/ScriptPool.ts b/sdk/src/ecs/ScriptPool.tsindex cf8ab80d1..607119e0e 100644--- a/sdk/src/ecs/ScriptPool.ts+++ b/sdk/src/ecs/ScriptPool.ts@@ -21,7 +21,7 @@* same write it always was, one indirection later.*/-import type { Entity } from '../types';+import { entityIndex, type Entity } from '../types';/** What a pooled field holds. Booleans are 0/1 in the slot, `true`/`false` out. */export type PoolKind = 'number' | 'boolean';@@ -34,25 +34,33 @@ export interface PoolField {/** One f64 per field: what the ABI gives a host record (REARCH_AOT_ABI §2.4). */export const POOL_SLOT_BYTES = 8;+/** A byte range the pool was given, and where it starts in the host's memory. */+export interface PoolBlock {+ readonly buffer: ArrayBufferLike;+ readonly byteOffset: number;+ readonly byteLength: number;+}+/**- * Where a pool's rows are allocated FROM. The default is a plain array, which- * suits a native host: same process, real address. A wasm host must pass one- * carved out of linear memory, because compiled code inside the module cannot- * reach a JS-heap array — and unreachable is what rows exist to stop being.+ * Where a pool's storage is allocated FROM. The default is the JS heap, which+ * suits a native host: same process, real address. A wasm host must pass linear+ * memory, because compiled code inside the module cannot reach a JS-heap array —+ * and unreachable is what rows exist to stop being.*/export interface PoolMemory {- /** A view of `slots` doubles the pool may keep. */- alloc(slots: number): Float64Array;- /** A view the pool has finished with. */- release(view: Float64Array): void;+ alloc(bytes: number): PoolBlock;+ release(block: PoolBlock): void;}-/** Rows on the JS heap: right for native, useless to a wasm module. */+/** Storage on the JS heap: right for native, useless to a wasm module. */export const HEAP_MEMORY: PoolMemory = {- alloc: (slots) => new Float64Array(slots),+ alloc: (bytes) => ({ buffer: new ArrayBuffer(bytes), byteOffset: 0, byteLength: bytes }),release: () => { /* the collector owns it */ },};+/** Absent, in the sparse table: a slot is stored as `slot + 1`. */+export const POOL_ABSENT = 0;+/*** The fields a pool can back, or null when the defaults are not all scalars.* Anything else — a string, an asset ref, a nested object, an array — has no@@ -80,7 +88,12 @@ export class ScriptPool {/** Bytes per row, and what the ABI's `stride` must agree with. */readonly stride: number;+ private rows_: PoolBlock;private slots_: Float64Array;+ private sparseBlock_: PoolBlock;+ /** Entity index -> slot + 1, so zero means absent. The engine's sparse set+ * in the same shape, and the reason a host needs no call to resolve a row. */+ private sparse_: Uint32Array;private capacity_: number;private readonly slotOf_ = new Map<Entity, number>();private readonly views_ = new Map<Entity, Record<string, unknown>>();@@ -95,11 +108,37 @@ export class ScriptPool {this.fields = fields;this.stride = fields.length * POOL_SLOT_BYTES;this.capacity_ = Math.max(1, capacity);- this.slots_ = memory_.alloc(this.capacity_ * fields.length);+ this.rows_ = memory_.alloc(this.capacity_ * this.stride);+ this.slots_ = new Float64Array(this.rows_.buffer, this.rows_.byteOffset,+ this.capacity_ * fields.length);+ this.sparseBlock_ = memory_.alloc(this.capacity_ * 4);+ this.sparse_ = new Uint32Array(this.sparseBlock_.buffer, this.sparseBlock_.byteOffset,+ this.capacity_);+ }++ /**+ * What a host needs to resolve this component WITHOUT calling back: rows at+ * a constant stride, and a sparse table saying which slot an entity is in.+ * Every address is in the memory the pool allocated from.+ *+ * Valid until the next growth, which is per call anyway (§2.2).+ */+ span(): { rows: number; stride: number; sparse: number; sparseCount: number } {+ return {+ rows: this.rows_.byteOffset,+ stride: this.stride,+ sparse: this.sparseBlock_.byteOffset,+ sparseCount: this.sparse_.length,+ };}- /** The backing store. Its byteOffset moves when the pool grows, so an- * address taken from `baseOf` is valid only until the next `add`. */+ /** The sparse table itself, for a host that reads it in this address space. */+ get sparseTable(): Uint32Array {+ return this.sparse_;+ }++ /** The rows themselves. Its byteOffset moves when the pool grows, so an+ * address taken from `address` is valid only until the next `put`. */get buffer(): Float64Array {return this.slots_;}@@ -144,8 +183,9 @@ export class ScriptPool {let slot = this.slotOf_.get(entity);const isNew = slot === undefined;if (slot === undefined) {- slot = this.claim_();+ slot = this.claim_(entity);this.slotOf_.set(entity, slot);+ this.mark_(entity, slot + 1);this.views_.set(entity, this.makeView_(slot));// A fresh row is seeded whole; an existing one keeps the fields the// caller did not name, which is what `world.set` has always done.@@ -160,6 +200,7 @@ export class ScriptPool {if (slot === undefined) return false;this.slotOf_.delete(entity);this.views_.delete(entity);+ this.mark_(entity, POOL_ABSENT);this.free_.push(slot);return true;}@@ -170,28 +211,53 @@ export class ScriptPool {// -------------------------------------------------------------------------- private claim_(): number {+ private claim_(entity: Entity): number {const reused = this.free_.pop();if (reused !== undefined) return reused;- if (this.next_ >= this.capacity_) this.grow_();+ if (this.next_ >= this.capacity_) this.growRows_();+ this.reserveSparse_(entityIndex(entity) + 1);return this.next_++;}- private grow_(): void {+ private mark_(entity: Entity, value: number): void {+ const at = entityIndex(entity);+ this.reserveSparse_(at + 1);+ this.sparse_[at] = value;+ }++ private growRows_(): void {this.capacity_ *= 2;- const wider = this.memory_.alloc(this.capacity_ * this.fields.length);+ const block = this.memory_.alloc(this.capacity_ * this.stride);+ const wider = new Float64Array(block.buffer, block.byteOffset,+ this.capacity_ * this.fields.length);wider.set(this.slots_);- const old = this.slots_;+ const old = this.rows_;+ this.rows_ = block;this.slots_ = wider;this.memory_.release(old);// Views hold a slot index, not a reference into the old array, so they// survive this. An ADDRESS taken before it does not, which is why- // `baseOf` says so.+ // `address` says so.for (const [entity, slot] of this.slotOf_) {this.views_.set(entity, this.makeView_(slot));}}+ /** The sparse table is indexed by ENTITY INDEX, which the world hands out+ * and this pool does not control, so it grows on its own schedule. */+ private reserveSparse_(want: number): void {+ if (want <= this.sparse_.length) return;+ let size = Math.max(this.sparse_.length * 2, 8);+ while (size < want) size *= 2;+ const block = this.memory_.alloc(size * 4);+ const wider = new Uint32Array(block.buffer, block.byteOffset, size);+ wider.set(this.sparse_);+ const old = this.sparseBlock_;+ this.sparseBlock_ = block;+ this.sparse_ = wider;+ this.memory_.release(old);+ }+private writeAll_(slot: number, from: Record<string, unknown>): void {const base = slot * this.fields.length;this.fields.forEach((f, i) => {diff --git a/sdk/tests/script-pool.test.ts b/sdk/tests/script-pool.test.tsindex c25da5bf1..57e6bcd59 100644--- a/sdk/tests/script-pool.test.ts+++ b/sdk/tests/script-pool.test.ts@@ -11,7 +11,8 @@* component still reads and writes as an object.*/import { describe, it, expect } from 'vitest';-import { ScriptPool, poolShape, POOL_SLOT_BYTES, HEAP_MEMORY, type PoolMemory } from '../src/ecs/ScriptPool';+import { ScriptPool, poolShape, POOL_SLOT_BYTES, POOL_ABSENT, HEAP_MEMORY, type PoolMemory } from '../src/ecs/ScriptPool';+import { entityIndex, makeEntity } from '../src/types';import type { Entity } from '../src/types';const e = (n: number): Entity => n as unknown as Entity;@@ -137,16 +138,18 @@ describe('a row, and the view onto it', () => {describe('rows carved out of a host-owned block', () => {/** Stands in for the wasm heap: one buffer, a bump pointer, nothing freed. */function heapOf(bytes: number): { memory: PoolMemory; all: Float64Array; used: () => number } {- const all = new Float64Array(bytes / POOL_SLOT_BYTES);- let at = 8; // a nonzero start, so an address that forgot byteOffset is wrong+ const block = new ArrayBuffer(bytes);+ const all = new Float64Array(block);+ let at = 64; // a nonzero start, so an address that forgot byteOffset is wrongreturn {all,used: () => at,memory: {- alloc(slots) {- const view = new Float64Array(all.buffer, at * POOL_SLOT_BYTES, slots);- at += slots;- return view;+ alloc(want) {+ const byteLength = (want + 7) & ~7;+ const out = { buffer: block, byteOffset: at, byteLength };+ at += byteLength;+ return out;},release: () => { /* a bump allocator frees nothing */ },},@@ -177,7 +180,7 @@ describe('rows carved out of a host-owned block', () => {const heap = heapOf(1 << 16);const pool = new ScriptPool(poolShape(DEFAULTS)!, 2, heap.memory);for (let i = 1; i <= 12; i++) pool.put(e(i), DEFAULTS, { speed: i });- expect(heap.used()).toBeGreaterThan(8);+ expect(heap.used()).toBeGreaterThan(64);for (let i = 1; i <= 12; i++) {const at = pool.address(e(i))! / POOL_SLOT_BYTES;@@ -185,6 +188,27 @@ describe('rows carved out of a host-owned block', () => {}});+ it('resolves a row the way a host would, with no call back', () => {+ // One memory, because that is what `span` means: the offsets are into+ // the allocator's block, and only there do rows and table share one+ // address space. The JS heap gives each block its own buffer.+ const heap = heapOf(1 << 20);+ const pool = new ScriptPool(poolShape(DEFAULTS)!, 8, heap.memory);+ for (let i = 1; i <= 6; i++) pool.put(e(i), DEFAULTS, { speed: i * 3 });++ const { rows, stride, sparse, sparseCount } = pool.span();+ const table = new Uint32Array(heap.all.buffer, sparse, sparseCount);++ // Exactly the two loads a host does: one into the table, one into a row.+ for (let i = 1; i <= 6; i++) {+ const slot = table[entityIndex(e(i))]!;+ expect(slot, `entity ${i} present`).not.toBe(POOL_ABSENT);+ const at = (rows + (slot - 1) * stride) / POOL_SLOT_BYTES;+ expect(heap.all[at], `entity ${i} speed`).toBe(i * 3);+ }+ expect(table[entityIndex(e(7))]).toBe(POOL_ABSENT);+ });+it('the default keeps the rows on the JS heap, where native wants them', () => {const pool = new ScriptPool(poolShape(DEFAULTS)!, 4, HEAP_MEMORY);pool.put(e(1), DEFAULTS);@@ -192,3 +216,43 @@ describe('rows carved out of a host-owned block', () => {expect(pool.address(e(1))).toBe(pool.baseOf(e(1)));});});++/**+ * What a host is handed so it needs no call to find a row. `span()` is the whole+ * answer: rows at a stride, and a sparse table of slot+1 by entity index — the+ * engine's own sparse-set shape, so C++ resolves a component with two loads.+ */+describe('a pool handed over as memory', () => {+ it('the sparse table says which slot, and zero says absent', () => {+ const pool = new ScriptPool(poolShape(DEFAULTS)!);+ pool.put(e(5), DEFAULTS, { speed: 55 });+ pool.put(e(9), DEFAULTS, { speed: 99 });++ expect(pool.sparseTable[entityIndex(e(5))]).toBe(1); // slot 0+ expect(pool.sparseTable[entityIndex(e(9))]).toBe(2); // slot 1+ expect(pool.sparseTable[entityIndex(e(7))]).toBe(POOL_ABSENT);++ pool.delete(e(5));+ expect(pool.sparseTable[entityIndex(e(5))]).toBe(POOL_ABSENT);+ });++ it('a generation bump does not move an entity in the table', () => {+ // The table is indexed by INDEX, as the engine's sparse set is: a recycled+ // id has the same index and a different generation, and looking it up by+ // the raw id would miss.+ const pool = new ScriptPool(poolShape(DEFAULTS)!);+ const first = makeEntity(12, 0);+ const recycled = makeEntity(12, 1);+ pool.put(first, DEFAULTS, { speed: 1 });+ expect(entityIndex(recycled)).toBe(entityIndex(first));+ expect(pool.sparseTable[entityIndex(recycled)]).not.toBe(POOL_ABSENT);+ });++ it('the table grows with the entity index, not with the row count', () => {+ const pool = new ScriptPool(poolShape(DEFAULTS)!, 4);+ pool.put(e(100000), DEFAULTS, { speed: 7 });+ expect(pool.size).toBe(1);+ expect(pool.sparseTable.length).toBeGreaterThan(100000);+ expect(pool.sparseTable[entityIndex(e(100000))]).toBe(1);+ });+});diff --git a/src/esengine/aot/AotHost.hpp b/src/esengine/aot/AotHost.hppindex 13339e293..19bfafff3 100644--- a/src/esengine/aot/AotHost.hpp+++ b/src/esengine/aot/AotHost.hpp@@ -20,6 +20,7 @@*/#pragma once+#include <cstddef>#include <cstdint>#include <functional>#include <span>@@ -124,6 +125,33 @@ inline std::span<const EsCmd> run(SystemFn fn,return std::span<const EsCmd>(arena.cmds_.data(), arena.count_);}+/**+ * A component handed over as MEMORY rather than as a callback: rows at a stride,+ * and a sparse table of `slot + 1` by entity index. What `ScriptPool::span`+ * gives — a script component's entity-to-slot map lives in the scripting+ * language, and asking it per entity per frame is what AOT exists to delete.+ */+struct RowSpan {+ const std::uint32_t* sparse = nullptr;+ std::uint32_t sparseCount = 0;+ unsigned char* rows = nullptr;+ std::uint32_t stride = 0;+ /** Entity::Layout::INDEX_MASK, passed rather than included so this header+ * stays free of the engine and the harness needs no link. */+ std::uint32_t indexMask = 0;+};++/** `span` as a resolver. Absent is zero in the table, as it is in a sparse set. */+inline ComponentAt fromRows(const RowSpan& span) {+ return [span](std::uint32_t entity) -> void* {+ const std::uint32_t at = entity & span.indexMask;+ if (at >= span.sparseCount) return nullptr;+ const std::uint32_t slot = span.sparse[at];+ if (slot == 0u) return nullptr;+ return span.rows + static_cast<std::size_t>(slot - 1u) * span.stride;+ };+}+/*** How a host answers the two questions a declaration asks: where one component* is for an entity, and where one resource is. Both by NAME, because names arediff --git a/tests/aot/test_aot_host.cpp b/tests/aot/test_aot_host.cppindex 7a19eba36..18de7e237 100644--- a/tests/aot/test_aot_host.cpp+++ b/tests/aot/test_aot_host.cpp@@ -75,11 +75,28 @@ std::size_t fieldOffset(const C& obj, const F& field) {- reinterpret_cast<const unsigned char*>(&obj));}+/**+ * The world, with `Mover` held the way ScriptStorage holds it: rows at a stride,+ * and a sparse table of slot+1 by entity index. Not a convenience — a script+ * component's map lives in the scripting language, so the only way the compiled+ * code reaches it without a call per entity is for it to arrive as memory.+ */struct World {ecs::Registry registry;std::vector<std::uint32_t> entities;- std::vector<MoverRow> movers; // index-parallel with `entities`+ std::vector<MoverRow> movers; // the pool's rows+ std::vector<std::uint32_t> moverSparse; // entity index -> slot + 1TimeRow time{1.0 / 30.0};++ aot::RowSpan moverSpan() {+ return aot::RowSpan{+ moverSparse.data(),+ static_cast<std::uint32_t>(moverSparse.size()),+ reinterpret_cast<unsigned char*>(movers.data()),+ static_cast<std::uint32_t>(sizeof(MoverRow)),+ Entity::Layout::INDEX_MASK,+ };+ }};constexpr int ENTITY_COUNT = 24;@@ -98,6 +115,9 @@ World makeWorld() {static_cast<double>((i % 3) - 1),((i % 4) - 2) / 2.0,});+ const std::uint32_t at = e.index();+ if (at >= w.moverSparse.size()) w.moverSparse.resize(at + 1u, 0u);+ w.moverSparse[at] = static_cast<std::uint32_t>(w.movers.size()); // slot + 1}return w;}@@ -127,14 +147,9 @@ aot::ComponentLookup componentsOf(World& w) {return w.registry.tryGet<ecs::Transform>(Entity::fromRaw(e));};}- if (std::strcmp(name, "Mover") == 0) {- return [&w](std::uint32_t e) -> void* {- for (std::size_t i = 0; i < w.entities.size(); ++i) {- if (w.entities[i] == e) return &w.movers[i];- }- return nullptr;- };- }+ // Handed over as memory, so nothing here calls back into the language+ // that owns the rows.+ if (std::strcmp(name, "Mover") == 0) return aot::fromRows(w.moverSpan());return nullptr;};}@@ -236,11 +251,28 @@ TEST_CASE("a compiled system moves a real Registry exactly as C++ does") {CHECK(moved->position.x != start->position.x);}+TEST_CASE("a component handed over as memory resolves without a call") {+ World w = makeWorld();+ const aot::ComponentAt at = aot::fromRows(w.moverSpan());++ for (std::size_t i = 0; i < w.entities.size(); ++i) {+ CHECK(at(w.entities[i]) == &w.movers[i]);+ }+ // Absent is zero in the table, and past the end is absent too — a host must+ // not read a row for an entity the pool never saw.+ CHECK(at(Entity::Layout::pack(9999u, 0u)) == nullptr);++ // A recycled id has the same INDEX and a different generation, which is why+ // the table is indexed by index and masked here rather than looked up raw.+ const std::uint32_t recycled = Entity::Layout::pack(Entity::fromRaw(w.entities[2]).index(), 3u);+ CHECK(at(recycled) == &w.movers[2]);+}+TEST_CASE("a row the query cannot complete is not a row") {World w = makeWorld();aot::CallArena arena;- // Take Transform away from one entity: it has a Mover, so only the filter- // keeps it out of the walk.+ // Take Transform away from one entity: its Mover row is still there, so only+ // the filter keeps it out of the walk.w.registry.remove<ecs::Transform>(Entity::fromRaw(w.entities[3]));const aot::BoundSystem bound = aot::bind(*declOf("MoveSystem"), componentsOf(w), resourcesOf(w));