Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

test(soak): churn the engine ten thousand times and demand it come back

esengineb94270abfbf9
diff --git a/package.json b/package.json
index aeb7748f8..e54d436e2 100644
--- a/package.json
+++ b/package.json
@@ -19,7 +19,8 @@
"migrate-asset-refs": "node tools/migrate-asset-refs.js",
"verify": "pnpm --filter ./sdk exec tsc --noEmit && node tools/api-surface.mjs --check && node tools/check-cycles.mjs && node tools/check-layers.mjs && node tools/check-project-settings.mjs && node tools/check-workflows.mjs && node tools/check-tool-calls.mjs && node tools/check-inspector-door.mjs && node tools/check-dirty-source.mjs && node tools/check-path-sandbox.mjs && node tools/check-comment-style.mjs && node tools/check-release-metadata.mjs && node tools/component-reference.mjs --check",
"perf": "node tools/perf-guard.mjs --check",
- "prepare": "git config core.hooksPath .githooks || true"
+ "prepare": "git config core.hooksPath .githooks || true",
+ "soak": "pnpm --filter ./sdk exec vitest run tests/soak"
},
"dependencies": {
"chalk": "^5.3.0",
diff --git a/sdk/tests/soak/churn.test.ts b/sdk/tests/soak/churn.test.ts
new file mode 100644
index 000000000..5830026a5
--- /dev/null
+++ b/sdk/tests/soak/churn.test.ts
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file Churn the engine and demand the counters come back.
+ *
+ * Sprites that fail to draw get found on the first run. What is never found is
+ * the cost of the forty-seventh Play/Stop, because nobody plays a scene forty-
+ * seven times by hand and nothing was watching if they did. This does, over the
+ * headless half of the loop: entities, components, subscriptions, prefabs and
+ * whole Apps built and torn down, with a census after every cycle.
+ *
+ * The verdict is a SLOPE, not a comparison against baseline. Caches plateau and
+ * heaps wander, so "did it return to where it started" fails honest runs and
+ * "did it grow every cycle" does not. See diagnostics/census.ts for the tiers.
+ *
+ * SOAK_CYCLES=200 SOAK_ENTITIES=50000 pnpm --filter ./sdk test soak
+ * SOAK_ONLY=prefab … one step, to place a leak
+ */
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { App } from '../../src/app/app';
+import { Transform, defineComponent } from '../../src/ecs/component';
+import { Emitter } from '../../src/ecs/emitter';
+import { instantiatePrefab } from '../../src/prefab';
+import { PREFAB_FORMAT_VERSION } from '../../src/prefab/migrate';
+import type { PrefabData } from '../../src/prefab/types';
+import {
+ takeCensus, analyzeCensusSeries, formatCensusReport, censusProbeIds, collectGarbage,
+ type Census,
+} from '../../src/diagnostics';
+import type { CppRegistry, ESEngineModule } from '../../src/wasm';
+import type { Entity } from '../../src/types';
+import { loadWasmModule, HAS_WASM } from '../helpers/loadWasm';
+
+const CYCLES = Number(process.env.SOAK_CYCLES ?? 12);
+const ENTITIES = Number(process.env.SOAK_ENTITIES ?? 500);
+const ONLY = process.env.SOAK_ONLY;
+
+/**
+ * Defined per test, never at module scope: tests/setup.ts clears user components
+ * in beforeEach, so a module-scope one is gone before the first test body runs.
+ * The prefab step then resolves it BY NAME, finds nothing, and logs a warning
+ * while churning two thirds less than it claims to.
+ */
+const marker = () => defineComponent('SoakMarker', { hp: 0, tag: '' });
+type Marker = ReturnType<typeof marker>;
+
+interface ChurnContext {
+ app: App;
+ module: ESEngineModule;
+ marker: Marker;
+ cycle: number;
+}
+
+interface ChurnStep {
+ name: string;
+ run(ctx: ChurnContext): Promise<void> | void;
+}
+
+// =============================================================================
+// The steps
+// =============================================================================
+
+/** A root with N children, each carrying a builtin and a script component. */
+const sceneStep: ChurnStep = {
+ name: 'scene',
+ run({ app, marker: Marker }) {
+ const world = app.world;
+ const root = world.spawn('soak-root');
+ for (let i = 0; i < ENTITIES; i++) {
+ const e = world.spawn(`soak-${i}`);
+ world.insert(e, Transform, { position: { x: i, y: -i, z: 0 } });
+ world.insert(e, Marker, { hp: i, tag: `t${i & 7}` });
+ world.setParent(e, root);
+ }
+ // Despawning the root alone: the subtree teardown is the path a scene
+ // unload actually takes, and the one where a child can be left behind.
+ world.despawn(root);
+ },
+};
+
+/** Components added and removed without the entity going anywhere. */
+const componentStep: ChurnStep = {
+ name: 'component',
+ run({ app, marker: Marker }) {
+ const world = app.world;
+ const held: Entity[] = [];
+ for (let i = 0; i < 64; i++) {
+ const e = world.spawn(`hold-${i}`);
+ world.insert(e, Transform, {});
+ held.push(e);
+ }
+ for (const e of held) {
+ world.insert(e, Marker, { hp: 1, tag: 'x' });
+ world.remove(e, Marker);
+ }
+ for (const e of held) world.despawn(e);
+ },
+};
+
+/** Every subscription surface that hands back an unsubscribe. */
+const subscriptionStep: ChurnStep = {
+ name: 'subscription',
+ run({ app }) {
+ const offs: Array<() => void> = [];
+ for (let i = 0; i < 32; i++) {
+ offs.push(app.world.onSpawn(() => {}));
+ offs.push(app.world.onDespawn(() => {}));
+ }
+ const emitter = new Emitter<{ ping: [number] }>();
+ for (let i = 0; i < 32; i++) offs.push(emitter.on('ping', () => {}));
+ emitter.emit('ping', 1);
+ for (const off of offs) off();
+ },
+};
+
+function soakPrefab(children: number): PrefabData {
+ return {
+ version: PREFAB_FORMAT_VERSION,
+ name: 'SoakPrefab',
+ rootEntityId: 'root',
+ entities: [
+ {
+ prefabEntityId: 'root', name: 'Root', parent: null, visible: true,
+ children: Array.from({ length: children }, (_, i) => `c${i}`),
+ components: [{ type: 'Transform', data: { position: { x: 0, y: 0, z: 0 } } }],
+ },
+ ...Array.from({ length: children }, (_, i) => ({
+ prefabEntityId: `c${i}`, name: `Child${i}`, parent: 'root', children: [], visible: true,
+ components: [
+ { type: 'Transform', data: { position: { x: i, y: 0, z: 0 } } },
+ { type: 'SoakMarker', data: { hp: i, tag: 'p' } },
+ ],
+ })),
+ ],
+ } as PrefabData;
+}
+
+const prefabStep: ChurnStep = {
+ name: 'prefab',
+ async run({ app, marker: Marker }) {
+ const data = soakPrefab(16);
+ const a = await instantiatePrefab(app.world, data);
+ const b = await instantiatePrefab(app.world, data, { parent: a.root });
+ // The prefab loader resolves components BY NAME and merely warns when it
+ // cannot, so an unregistered one leaves this step churning Transform
+ // alone while still looking busy. Checked every cycle, not once.
+ const child = a.entities.get(1);
+ if (child === undefined || !app.world.has(child, Marker)) {
+ throw new Error('prefab instantiate dropped SoakMarker — this step is not churning what it claims');
+ }
+ app.world.despawn(b.root);
+ app.world.despawn(a.root);
+ },
+};
+
+/**
+ * A whole App built and quit — the headless half of Play/Stop.
+ *
+ * Deliberately its own registry: quit() must return the plugins, systems and
+ * subscriptions an install created, and sharing the outer app's world would let
+ * a leaked one hide among counters the other steps are already moving.
+ */
+const appStep: ChurnStep = {
+ name: 'app',
+ async run({ module }) {
+ const inner = App.new();
+ const registry = new (module as unknown as { Registry: new () => CppRegistry }).Registry();
+ inner.connectCpp(registry, module);
+ const e = inner.world.spawn('inner');
+ inner.world.insert(e, Transform, {});
+ await inner.tick(1 / 60);
+ inner.quit({ keepRenderer: true });
+ (registry as unknown as { delete?: () => void }).delete?.();
+ },
+};
+
+const ALL_STEPS: ChurnStep[] = [sceneStep, componentStep, subscriptionStep, prefabStep, appStep];
+
+// =============================================================================
+// The run
+// =============================================================================
+
+describe.skipIf(!HAS_WASM)('resource census over a churn soak', () => {
+ let module: ESEngineModule;
+ let app: App;
+ let registry: CppRegistry;
+
+ beforeAll(async () => {
+ module = await loadWasmModule();
+ app = App.new();
+ registry = new (module as unknown as { Registry: new () => CppRegistry }).Registry();
+ app.connectCpp(registry, module);
+ });
+
+ afterAll(() => {
+ app?.quit({ keepRenderer: true });
+ });
+
+ it('has probes registered at all', () => {
+ // A census with no probes reports nothing and passes everything. That is
+ // the failure mode this whole file exists to not have, so it is asserted
+ // before any of it runs.
+ const ids = censusProbeIds();
+ expect(ids).toContain('ecs');
+ expect(ids).toContain('events');
+ expect(ids).toContain('heap');
+ const census = takeCensus({ app, module });
+ expect(census.failedProbes).toEqual([]);
+ expect(census.entries.size).toBeGreaterThan(10);
+ });
+
+ it(`returns every conserved counter to baseline over ${CYCLES} cycles`, async () => {
+ const steps = ONLY ? ALL_STEPS.filter((s) => s.name === ONLY) : ALL_STEPS;
+ expect(steps.length, `SOAK_ONLY=${ONLY} matched no step`).toBeGreaterThan(0);
+
+ const Marker = marker();
+ const samples: Census[] = [];
+ for (let cycle = 0; cycle < CYCLES; cycle++) {
+ for (const step of steps) await step.run({ app, module, marker: Marker, cycle });
+ // Between cycles, never inside one: mid-cycle a conserved counter is
+ // legitimately up by a whole scene. The collection is what makes the heap
+ // counters mean "kept" rather than "allocated".
+ collectGarbage();
+ samples.push(takeCensus({ app, module }));
+ }
+
+ const report = analyzeCensusSeries(samples, { projectCycles: 10_000 });
+ // A passing run prints nothing by default and so teaches nothing. This is
+ // how you read the counters that did NOT move, and how the noise floor of
+ // the harness itself gets measured.
+ if (process.env.SOAK_REPORT) console.log(formatCensusReport(report));
+ expect(report.failedProbes).toEqual([]);
+ expect(report.leaks, `\n${formatCensusReport(report)}\n`).toEqual([]);
+ }, 600_000);
+});
diff --git a/sdk/tests/soak/dist-probes.test.ts b/sdk/tests/soak/dist-probes.test.ts
new file mode 100644
index 000000000..c0cad65f8
--- /dev/null
+++ b/sdk/tests/soak/dist-probes.test.ts
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file The census survives bundling — asserted against dist, not src.
+ *
+ * Every other soak test imports `../../src`, so all of them passed while the
+ * SHIPPED engine carried a takeCensus with zero probes: the bundler treats every
+ * non-entry module as side-effect free, and the registrations were reached by a
+ * bare import. The editor asked for a census and got an empty one. A green suite
+ * and a blind instrument, at the same time.
+ *
+ * So this one reads what actually ships. It skips when dist is absent rather
+ * than failing, because a source-only checkout has nothing to say here — but CI
+ * builds the SDK before running tests, so there it is the real gate.
+ */
+import { describe, it, expect } from 'vitest';
+import { existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+const DIST = resolve(__dirname, '../../dist/index.js');
+const HAS_DIST = existsSync(DIST);
+
+describe.skipIf(!HAS_DIST)('the built SDK', () => {
+ it('ships a census with its probes attached', async () => {
+ const sdk = await import(DIST) as {
+ takeCensus: (ctx?: unknown) => { entries: Map<string, unknown>; failedProbes: string[] };
+ censusProbeIds: () => string[];
+ };
+
+ expect(typeof sdk.takeCensus).toBe('function');
+ const ids = sdk.censusProbeIds();
+ // Named explicitly: a bundler that drops one probe module leaves the
+ // others answering, and a count check would still pass.
+ expect(ids).toEqual(expect.arrayContaining(['ecs', 'events', 'heap', 'render', 'asset', 'physics']));
+
+ // With no App the subsystem probes report nothing, but the ones that read
+ // process-wide state must still answer — that is the difference between
+ // "nothing to measure here" and "no instrument at all".
+ const census = sdk.takeCensus();
+ expect(census.failedProbes).toEqual([]);
+ expect([...census.entries.keys()]).toEqual(
+ expect.arrayContaining(['events.emitterHandlers', 'events.domListeners']),
+ );
+ });
+});
diff --git a/sdk/tests/soak/sabotage.test.ts b/sdk/tests/soak/sabotage.test.ts
new file mode 100644
index 000000000..4d144c8d2
--- /dev/null
+++ b/sdk/tests/soak/sabotage.test.ts
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file Break the engine on purpose and demand the census notices.
+ *
+ * A leak detector that has never caught anything is indistinguishable from one
+ * that cannot. This project has been here before: eleven of fifteen benchmarks
+ * had not run since the ECS moved and reported success anyway. So every counter
+ * the soak asserts gets a matching sabotage here — a leak injected by hand, and
+ * a demand that the verdict goes red.
+ *
+ * The specificity half matters as much. A judge that fails everything also has a
+ * perfect catch rate, so the legitimate shapes — a cache that fills and stops, a
+ * counter that wobbles without trending — are asserted to stay GREEN.
+ */
+import { describe, it, expect, beforeAll } from 'vitest';
+import { App } from '../../src/app/app';
+import { Transform, defineComponent } from '../../src/ecs/component';
+import { Emitter } from '../../src/ecs/emitter';
+import { addTrackedListener } from '../../src/util/listeners';
+import {
+ takeCensus, analyzeCensusSeries, counter, collectGarbage,
+ type Census, type CensusEntry, type CensusTier,
+} from '../../src/diagnostics';
+import type { CppRegistry, ESEngineModule } from '../../src/wasm';
+import { loadWasmModule, HAS_WASM } from '../helpers/loadWasm';
+
+const SabotageMarker = defineComponent('SabotageMarker', { n: 0 });
+
+// =============================================================================
+// The judge, on synthetic series — sensitivity and specificity, no engine
+// =============================================================================
+
+function series(key: string, tier: CensusTier, values: number[], unit: CensusEntry['unit'] = 'count'): Census[] {
+ return values.map((v) => ({
+ atMs: 0,
+ entries: new Map([[key, counter(key, v, tier, unit)]]),
+ failedProbes: [],
+ }));
+}
+
+const leakedKeys = (samples: Census[], opts = {}): string[] =>
+ analyzeCensusSeries(samples, opts).leaks.map((v) => v.key);
+
+describe('the census judge', () => {
+ it('catches a conserved counter that climbs', () => {
+ const climbing = Array.from({ length: 20 }, (_, i) => 100 + i);
+ expect(leakedKeys(series('x', 'conserved', climbing))).toEqual(['x']);
+ });
+
+ it('catches a conserved counter that spikes and returns', () => {
+ // Ends where it began, so a slope test alone calls this steady. It is not:
+ // a cycle cleaned up one cycle late, and a longer run overlaps two.
+ const wobble = Array.from({ length: 20 }, (_, i) => (i % 2 === 0 ? 100 : 140));
+ expect(leakedKeys(series('x', 'conserved', wobble))).toEqual(['x']);
+ });
+
+ it('passes a conserved counter that holds exactly', () => {
+ expect(leakedKeys(series('x', 'conserved', Array(20).fill(100)))).toEqual([]);
+ });
+
+ it('catches a bounded counter that never plateaus', () => {
+ const climbing = Array.from({ length: 30 }, (_, i) => 10 + i * 3);
+ expect(leakedKeys(series('x', 'bounded', climbing))).toEqual(['x']);
+ });
+
+ it('passes a bounded counter that fills a cache and stops', () => {
+ // The shape every pool and cache makes. Judging it as a leak is what
+ // makes a soak suite get switched off.
+ const filling = Array.from({ length: 30 }, (_, i) => Math.min(64, 10 + i * 8));
+ expect(leakedKeys(series('x', 'bounded', filling))).toEqual([]);
+ });
+
+ it('passes a pool that steps ONCE mid-run, which a line fits as a ramp', () => {
+ // From a real run: one 4 KB sparse page at cycle 30 of 80, flat either
+ // side. Least squares alone called that a confident 947 KB leak. One rise
+ // in 79 transitions is what saves it.
+ const staircase = Array.from({ length: 80 }, (_, i) => (i < 30 ? 0 : 4104));
+ expect(leakedKeys(series('m', 'bounded', staircase, 'bytes'))).toEqual([]);
+ });
+
+ it('still catches a counter that steps a page every few cycles', () => {
+ // The other side of that rule: repeated steps are a leak however chunky,
+ // so the discrimination cannot just be "ignore anything step-shaped".
+ const repeating = Array.from({ length: 80 }, (_, i) => Math.floor(i / 2) * 4104);
+ expect(leakedKeys(series('m', 'bounded', repeating, 'bytes'))).toEqual(['m']);
+ });
+
+ it('catches a heap that grows past its budget', () => {
+ const MB = 1024 * 1024;
+ const growing = Array.from({ length: 30 }, (_, i) => 40 * MB + i * 512 * 1024);
+ expect(leakedKeys(series('h', 'trend', growing, 'bytes'))).toEqual(['h']);
+ });
+
+ it('passes a heap that wanders without trending', () => {
+ // Real V8 numbers move like this. The confidence bound is what keeps the
+ // verdict from depending on which cycle sampling happened to stop at.
+ const noise = [0, 3, -2, 5, -4, 1, 6, -3, 2, -1, 4, 0, -5, 3, 1, -2, 4, -1, 2, 0];
+ const wandering = noise.map((n) => 40 * 1024 * 1024 + n * 8 * 1024);
+ expect(leakedKeys(series('h', 'trend', wandering, 'bytes'))).toEqual([]);
+ });
+
+ it('never judges an info counter', () => {
+ const climbing = Array.from({ length: 20 }, (_, i) => i * 1000);
+ expect(leakedKeys(series('x', 'info', climbing))).toEqual([]);
+ });
+
+ it('reports a probe that stopped answering rather than passing without it', () => {
+ const samples = series('x', 'conserved', Array(10).fill(1));
+ const broken: Census[] = samples.map((s) => ({ ...s, failedProbes: ['render: boom'] }));
+ expect(analyzeCensusSeries(broken).failedProbes).toEqual(['render: boom']);
+ });
+});
+
+// =============================================================================
+// The probes, against a real engine — proving they are wired to anything
+// =============================================================================
+
+describe.skipIf(!HAS_WASM)('census probes catch a real injected leak', () => {
+ let module: ESEngineModule;
+
+ beforeAll(async () => {
+ module = await loadWasmModule();
+ });
+
+ function freshApp(): App {
+ const app = App.new();
+ const registry = new (module as unknown as { Registry: new () => CppRegistry }).Registry();
+ app.connectCpp(registry, module);
+ return app;
+ }
+
+ /** Run `sabotage` once per cycle and report which counters went red. */
+ function soak(sabotage: (app: App, cycle: number) => void, cycles = 16): string[] {
+ const app = freshApp();
+ const samples: Census[] = [];
+ try {
+ for (let cycle = 0; cycle < cycles; cycle++) {
+ sabotage(app, cycle);
+ collectGarbage();
+ samples.push(takeCensus({ app, module }));
+ }
+ return analyzeCensusSeries(samples).leaks.map((v) => v.key);
+ } finally {
+ app.quit({ keepRenderer: true });
+ }
+ }
+
+ it('sees entities that were never despawned', () => {
+ expect(soak((app) => { app.world.spawn('leaked'); })).toContain('ecs.entities');
+ });
+
+ it('sees script component rows left behind', () => {
+ const leaked = soak((app) => {
+ const e = app.world.spawn('holder');
+ app.world.insert(e, SabotageMarker, { n: 1 });
+ // The entity goes; only the storage row is meant to be at issue, but a
+ // despawn that forgot ScriptStorage would show here and not in entities.
+ });
+ expect(leaked).toContain('ecs.scriptRows');
+ });
+
+ it('sees an emitter subscription that was never released', () => {
+ const emitter = new Emitter<{ tick: [] }>();
+ expect(soak(() => { emitter.on('tick', () => {}); })).toContain('events.emitterHandlers');
+ });
+
+ it('sees a host listener that was never detached', () => {
+ const target = new EventTarget();
+ expect(soak(() => { addTrackedListener(target, 'ping', () => {}); }))
+ .toContain('events.domListeners');
+ });
+
+ it('sees a world spawn callback that was never unsubscribed', () => {
+ expect(soak((app) => { app.world.onSpawn(() => {}); })).toContain('ecs.spawnCallbacks');
+ });
+
+ it('sees C++ memory that was malloced and never freed', () => {
+ const held: number[] = [];
+ const leaked = soak(() => { held.push(module._malloc(64 * 1024)); });
+ for (const ptr of held) module._free(ptr);
+ expect(leaked).toContain('wasm.mallocBytes');
+ });
+
+ it('sees retained objects on the JS heap, at the real budget', () => {
+ // 24 cycles of a few hundred KB — the retained-scene class the loose byte
+ // budget is set for. If this stops failing, that budget has drifted useless.
+ const held: object[][] = [];
+ const leaked = soak(() => {
+ held.push(Array.from({ length: 8000 }, (_, i) => ({ i, tag: `held-${i}` })));
+ }, 24);
+ expect(held.length).toBe(24);
+ expect(leaked).toContain('js.heapUsed');
+ });
+
+ it('sees retained buffers, which land OUTSIDE the JS heap', () => {
+ // A leaked texture, audio buffer or mesh is a typed array, and its bytes
+ // are external memory — `js.heapUsed` does not move for them at all. Both
+ // counters exist because either alone is blind to half of what leaks.
+ const held: Uint8Array[] = [];
+ const leaked = soak(() => { held.push(new Uint8Array(256 * 1024).fill(7)); }, 24);
+ expect(held.length).toBe(24);
+ expect(leaked).toContain('js.external');
+ });
+
+ it('stays green on the same workload without the sabotage', () => {
+ // The control. Every case above churns an App the same way; if this one
+ // went red too, none of them would have proven anything.
+ const leaked = soak((app) => {
+ const e = app.world.spawn('clean');
+ app.world.insert(e, Transform, {});
+ app.world.insert(e, SabotageMarker, { n: 1 });
+ app.world.despawn(e);
+ });
+ expect(leaked).toEqual([]);
+ });
+});
diff --git a/sdk/vitest.config.ts b/sdk/vitest.config.ts
index 81a1d9eb8..4a1f43158 100644
--- a/sdk/vitest.config.ts
+++ b/sdk/vitest.config.ts
@@ -9,6 +9,13 @@ export default defineConfig({
environment: 'happy-dom',
setupFiles: ['tests/setup.ts'],
include: ['tests/**/*.test.ts'],
+ // The soak suite judges heap growth, and a heap nobody can collect only
+ // climbs. Without this the census downgrades its heap counters to
+ // unasserted `info` rather than reporting every run as a leak.
+ poolOptions: {
+ forks: { execArgv: ['--expose-gc'] },
+ threads: { execArgv: ['--expose-gc'] },
+ },
benchmark: {
include: ['benchmarks/**/*.bench.ts'],
},