Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(ecs): rows on the engine heap, and the growth that detaches them

yhhd182d5595daf
diff --git a/sdk/src/ecs/ScriptPool.ts b/sdk/src/ecs/ScriptPool.ts
index 478db9598..b7de8315c 100644
--- a/sdk/src/ecs/ScriptPool.ts
+++ b/sdk/src/ecs/ScriptPool.ts
@@ -100,6 +100,7 @@ export class ScriptPool {
/** Entity index -> slot + 1, so zero means absent. The engine's sparse set
* in the same shape, and the reason a host needs no call to resolve a row. */
private sparse_: Uint32Array;
+ private sparseLen_: number;
private capacity_: number;
private readonly slotOf_ = new Map<Entity, number>();
private readonly views_ = new Map<Entity, Record<string, unknown>>();
@@ -114,12 +115,15 @@ export class ScriptPool {
this.fields = fields;
this.stride = fields.length * POOL_SLOT_BYTES;
this.capacity_ = Math.max(1, capacity);
+ // BOTH allocations before EITHER view. The second can grow the heap, and
+ // a grown heap detaches a view built after the first — which is a bug
+ // this constructor had, and the reason every view is built in one place.
this.rows_ = memory_.alloc(this.capacity_ * this.stride);
- this.slots_ = new Float64Array(this.rows_.buffer, this.rows_.byteOffset,
- this.capacity_ * fields.length);
this.sparseBlock_ = memory_.alloc(this.capacity_ * 4);
- this.sparse_ = new Uint32Array(this.sparseBlock_.buffer, this.sparseBlock_.byteOffset,
- this.capacity_);
+ this.sparseLen_ = this.capacity_;
+ this.slots_ = EMPTY_F64;
+ this.sparse_ = EMPTY_U32;
+ this.rebuild_();
}
/**
@@ -150,16 +154,21 @@ export class ScriptPool {
* per-entity views read `slots_` on access and need no rebuild.
*/
refresh(): void {
- const rows = this.memory_.current?.(this.rows_);
- if (rows && rows !== this.slots_.buffer) {
- this.slots_ = new Float64Array(rows, this.rows_.byteOffset,
- this.capacity_ * this.fields.length);
- }
- const sparse = this.memory_.current?.(this.sparseBlock_);
- if (sparse && sparse !== this.sparse_.buffer) {
- this.sparse_ = new Uint32Array(sparse, this.sparseBlock_.byteOffset,
- this.sparse_.length);
- }
+ this.rebuild_();
+ }
+
+ /** Every view of this pool's storage, made in ONE place. Anything that can
+ * have detached a view calls this rather than fixing up the one it knows
+ * about — the heap grows for all of them at once. */
+ private rebuild_(): void {
+ this.slots_ = new Float64Array(this.bufferOf_(this.rows_), this.rows_.byteOffset,
+ this.capacity_ * this.fields.length);
+ this.sparse_ = new Uint32Array(this.bufferOf_(this.sparseBlock_),
+ this.sparseBlock_.byteOffset, this.sparseLen_);
+ }
+
+ private bufferOf_(block: PoolBlock): ArrayBufferLike {
+ return this.memory_.current?.(block) ?? block.buffer;
}
/** The rows themselves. Its byteOffset moves when the pool grows, so an
@@ -255,8 +264,8 @@ export class ScriptPool {
const block = this.memory_.alloc(want * this.stride);
// Before reading the old rows, not after: allocating may have grown the
// heap, and a grown heap detaches the view this is about to copy FROM.
- this.refresh();
- const wider = new Float64Array(block.buffer, block.byteOffset,
+ this.rebuild_();
+ const wider = new Float64Array(this.bufferOf_(block), block.byteOffset,
want * this.fields.length);
wider.set(this.slots_);
const old = this.rows_;
@@ -275,16 +284,17 @@ export class ScriptPool {
/** The sparse table is indexed by ENTITY INDEX, which the world hands out
* and this pool does not control, so it grows on its own schedule. */
private reserveSparse_(want: number): void {
- if (want <= this.sparse_.length) return;
- let size = Math.max(this.sparse_.length * 2, 8);
+ if (want <= this.sparseLen_) return;
+ let size = Math.max(this.sparseLen_ * 2, 8);
while (size < want) size *= 2;
const block = this.memory_.alloc(size * 4);
- this.refresh(); // same trap as growRows_: the source may have detached
- const wider = new Uint32Array(block.buffer, block.byteOffset, size);
+ this.rebuild_(); // same trap as growRows_: the source may have detached
+ const wider = new Uint32Array(this.bufferOf_(block), block.byteOffset, size);
wider.set(this.sparse_);
const old = this.sparseBlock_;
this.sparseBlock_ = block;
this.sparse_ = wider;
+ this.sparseLen_ = size;
this.memory_.release(old);
}
@@ -322,6 +332,10 @@ export class ScriptPool {
}
}
+/** Placeholders, so the constructor can allocate before it views. */
+const EMPTY_F64 = new Float64Array(0);
+const EMPTY_U32 = new Uint32Array(0);
+
/** A slot holds a number; a boolean is the 0/1 the ABI stores it as. */
function toSlot(v: unknown): number {
if (typeof v === 'boolean') return v ? 1 : 0;
diff --git a/sdk/src/ecs/ScriptStorage.ts b/sdk/src/ecs/ScriptStorage.ts
index 8de05dcd1..a3f5aebf5 100644
--- a/sdk/src/ecs/ScriptStorage.ts
+++ b/sdk/src/ecs/ScriptStorage.ts
@@ -46,6 +46,15 @@ export class ScriptStorage {
return this.pools_.get(id);
}
+ /**
+ * @internal Re-view every pool. A wasm heap grows for one pool and detaches
+ * the views of all of them, so this is a set operation and not a per-pool
+ * one. Idempotent: a caller that is unsure may call it.
+ */
+ refreshPools(): void {
+ for (const pool of this.pools_.values()) pool.refresh();
+ }
+
/**
* The pool `component` uses, creating it on first sight. `null` once for a
* shape that cannot be flat, and remembered as null so the check is not
@@ -59,6 +68,8 @@ export class ScriptStorage {
if (!fields) { this.notPooled_.add(component._id); return null; }
const pool = new ScriptPool(fields, 64, this.poolMemory_);
this.pools_.set(component._id, pool);
+ // Creating one may have grown the heap every OTHER pool also lives in.
+ this.refreshPools();
return pool;
}
diff --git a/sdk/src/ecs/WasmPoolMemory.ts b/sdk/src/ecs/WasmPoolMemory.ts
new file mode 100644
index 000000000..8b277dc30
--- /dev/null
+++ b/sdk/src/ecs/WasmPoolMemory.ts
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file WasmPoolMemory.ts
+ * @brief Component rows on the engine's wasm heap, where compiled code is.
+ *
+ * @details A compiled system runs inside the engine's module and reads memory by
+ * offset. Rows on the JS heap are as unreachable to it as the objects
+ * they replaced, so on web they come from `_malloc` instead — one
+ * linear memory, one set of offsets, and `ScriptPool.address` is
+ * already the address the compiled code wants.
+ *
+ * The trap is growth: `_malloc` may grow the heap, and growing DETACHES
+ * every ArrayBuffer view of it. The bytes survive at the same offsets —
+ * it is the same linear memory — but a held `Float64Array` stops
+ * working, silently returning undefined rather than throwing. So a
+ * block is identified by its OFFSET and the buffer is fetched fresh,
+ * which is the same thing emscripten does for its own HEAP views.
+ */
+
+import type { PoolBlock, PoolMemory } from './ScriptPool';
+
+/** What this needs of the engine module: an allocator and its heap. */
+export interface WasmHeap {
+ _malloc(size: number): number;
+ _free(ptr: number): void;
+ /** The live view. Emscripten replaces this object when the heap grows. */
+ readonly HEAPU8: Uint8Array;
+}
+
+/**
+ * Rows on the engine's heap. `byteOffset` is a wasm address, which is what makes
+ * a pool's `span()` directly usable by compiled code.
+ */
+export class WasmPoolMemory implements PoolMemory {
+ constructor(private readonly heap: WasmHeap) {}
+
+ alloc(bytes: number): PoolBlock {
+ // Eight-aligned because a row's fields are f64 and an unaligned typed
+ // array view is a range error rather than a slow read.
+ const size = (bytes + 7) & ~7;
+ const at = this.heap._malloc(size);
+ if (at === 0) throw new Error(`ScriptPool: the wasm heap refused ${size} bytes`);
+ return { buffer: this.heap.HEAPU8.buffer, byteOffset: at, byteLength: size };
+ }
+
+ release(block: PoolBlock): void {
+ this.heap._free(block.byteOffset);
+ }
+
+ /**
+ * The block's buffer as it is NOW. After a growth the one handed out at
+ * `alloc` is detached and this is a different object holding the same bytes
+ * at the same offset.
+ */
+ current(block: PoolBlock): ArrayBufferLike {
+ void block;
+ return this.heap.HEAPU8.buffer;
+ }
+}