Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

test(torture): generate the interleavings nobody thinks to write

esengine16af5e681f20
diff --git a/package.json b/package.json
index e54d436e2..b59797de6 100644
--- a/package.json
+++ b/package.json
@@ -20,7 +20,8 @@
"verify": "pnpm --filter ./sdk exec tsc --noEmit && node tools/api-surface.mjs --check && node tools/check-cycles.mjs && node tools/check-layers.mjs && node tools/check-project-settings.mjs && node tools/check-workflows.mjs && node tools/check-tool-calls.mjs && node tools/check-inspector-door.mjs && node tools/check-dirty-source.mjs && node tools/check-path-sandbox.mjs && node tools/check-comment-style.mjs && node tools/check-release-metadata.mjs && node tools/component-reference.mjs --check",
"perf": "node tools/perf-guard.mjs --check",
"prepare": "git config core.hooksPath .githooks || true",
- "soak": "pnpm --filter ./sdk exec vitest run tests/soak"
+ "soak": "pnpm --filter ./sdk exec vitest run tests/soak",
+ "torture": "pnpm --filter ./sdk exec vitest run tests/torture"
},
"dependencies": {
"chalk": "^5.3.0",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 92b7b8574..14550a371 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -133,6 +133,9 @@ importers:
'@vitest/ui':
specifier: ^2.1.0
version: 2.1.9([email protected])
+ fast-check:
+ specifier: ^4.9.0
+ version: 4.9.0
happy-dom:
specifier: ^15.11.0
version: 15.11.7
@@ -1931,6 +1934,10 @@ packages:
resolution: {integrity: sha512-LmDxfWXwcTArk8fUEnOfSZpHOJ6zOMUJKOtFLFqJLoKJetuQG874Uc7/Kki7zFLzYybmZhp1M7+98pfMqeX8yA==}
+ resolution: {integrity: sha512-7ms6T7SybUev/PQITciI0yLM2pOSFy5zpG8Ty7tQofcVaQUvrMXp6CBwqF6fThLCLOrfBtuHAtwq6Yu4XPCllg==}
+ engines: {node: '>=12.17.0'}
+
resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
@@ -2861,6 +2868,9 @@ packages:
resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
engines: {node: '>=6'}
+ resolution: {integrity: sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==}
+
resolution: {integrity: sha512-PLgQXQ6H2FWCaeRak8vvk1GW462lMxB5s3Jm673N82zI4vqtVUPuZdffdZbPDFRoU8kAhItWFtPCWiPpp4/EDg==}
@@ -5372,6 +5382,10 @@ snapshots:
+ dependencies:
+ pure-rand: 8.4.2
+
@@ -6312,6 +6326,8 @@ snapshots:
optional: true
+
dependencies:
tslib: 2.8.1
diff --git a/sdk/package.json b/sdk/package.json
index 9f30737e1..d596a0bf8 100644
--- a/sdk/package.json
+++ b/sdk/package.json
@@ -82,6 +82,7 @@
"@types/ws": "^8.18.1",
"@vitest/coverage-v8": "^2.1.9",
"@vitest/ui": "^2.1.0",
+ "fast-check": "^4.9.0",
"happy-dom": "^15.11.0",
"minigame-api-typings": "^3.8.16",
"rimraf": "^5.0.5",
diff --git a/sdk/tests/torture/asset-lifecycle-guards.test.ts b/sdk/tests/torture/asset-lifecycle-guards.test.ts
new file mode 100644
index 000000000..b2ecac79d
--- /dev/null
+++ b/sdk/tests/torture/asset-lifecycle-guards.test.ts
@@ -0,0 +1,255 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file Proof that the torture harness tortures, and that its claims can fail.
+ *
+ * A generated property that passes in eight milliseconds is either watching an
+ * engine with no bugs or watching nothing, and the two look identical from the
+ * outside. So: the harness is asserted to actually drive loads and observe the
+ * pool, the invariants are asserted to go red on a leak injected by hand, and
+ * the counterexample the property found the first time it ran is kept here as
+ * an ordinary regression test — spelled out, so it fails for one reason.
+ */
+import { describe, it, expect, vi } from 'vitest';
+import { makeAssetHarness, withoutLoadTimeout, textureRefs } from './assetHarness';
+
+const KEY = 'tex/a.png';
+
+/** A loader whose settling this test controls, without a fast-check scheduler. */
+function manualHarness() {
+ const gates: Array<() => void> = [];
+ const h = makeAssetHarness(
+ (key) => new Promise((resolve) => { gates.push(() => resolve(key as never)); }),
+ );
+ return {
+ ...h,
+ /** Let the Nth outstanding load through (0 = oldest). */
+ settle(index = 0) {
+ const gate = gates.splice(index, 1)[0];
+ if (!gate) throw new Error(`no outstanding load at index ${index}`);
+ gate();
+ return new Promise<void>((r) => setTimeout(r, 0));
+ },
+ outstanding: () => gates.length,
+ };
+}
+
+describe('the torture harness', () => {
+ it('actually drives the loader and sees the pool', async () => {
+ await withoutLoadTimeout(async () => {
+ const h = manualHarness();
+ try {
+ const load = h.assets.loadTexture(KEY);
+ expect(h.loaderCalls, 'the patched loader was never called — I1 would be vacuous').toHaveLength(1);
+ expect(h.outstanding(), 'the load settled on its own; nothing is schedulable').toBe(1);
+
+ await h.settle();
+ const result = await load;
+ expect(h.gpu.created, 'no handle was minted — the GPU ledger sees nothing').toContain(result.handle);
+
+ h.assets.releaseTexture(KEY);
+ expect(h.gpu.refs.get(result.handle), 'release never reached the ResourceManager').toBe(0);
+ } finally {
+ h.dispose();
+ }
+ });
+ });
+
+ it('joins a second request to the load already in flight', async () => {
+ // I1's premise. If two concurrent loads produced two fetches, the
+ // invariant's budget arithmetic would be measuring the wrong thing.
+ await withoutLoadTimeout(async () => {
+ const h = manualHarness();
+ try {
+ const a = h.assets.loadTexture(KEY);
+ const b = h.assets.loadTexture(KEY);
+ expect(h.loaderCalls).toHaveLength(1);
+ await h.settle();
+ expect((await a).handle).toBe((await b).handle);
+ } finally {
+ h.dispose();
+ }
+ });
+ });
+
+ it('revives a released texture from the pool instead of re-fetching', async () => {
+ // I4's premise. With no revive there is nothing that could hand back a
+ // destroyed handle, and "a disposed resource never comes back" would be
+ // a claim about a path the property never takes.
+ await withoutLoadTimeout(async () => {
+ const h = manualHarness();
+ try {
+ const first = h.assets.loadTexture(KEY);
+ await h.settle();
+ const handle = (await first).handle;
+ h.assets.releaseTexture(KEY);
+ expect(h.gpu.freed, 'the pool destroyed it instead of retaining it').not.toContain(handle);
+
+ const second = h.assets.loadTexture(KEY);
+ await new Promise((r) => setTimeout(r, 0));
+ expect((await second).handle, 'the reload did not revive').toBe(handle);
+ expect(h.gpu.revived).toContain(handle);
+ expect(h.loaderCalls, 'a revive must not re-fetch').toHaveLength(1);
+ } finally {
+ h.dispose();
+ }
+ });
+ });
+
+ it('never revives across an invalidate', async () => {
+ await withoutLoadTimeout(async () => {
+ const h = manualHarness();
+ try {
+ const first = h.assets.loadTexture(KEY);
+ await h.settle();
+ const stale = (await first).handle;
+ h.assets.releaseTexture(KEY); // retained as evictable
+ h.assets.invalidate(KEY); // bytes changed -> residency severed
+
+ expect(h.gpu.freed, 'severing must destroy the retained texture').toContain(stale);
+
+ const second = h.assets.loadTexture(KEY);
+ await h.settle();
+ expect((await second).handle, 'a severed key served stale bytes').not.toBe(stale);
+ expect(h.gpu.revived).toEqual([]);
+ } finally {
+ h.dispose();
+ }
+ });
+ });
+
+ it('goes red when a leak is injected', async () => {
+ // The sabotage: a pool that quietly ignores releases. If the teardown
+ // claim cannot fail here, it cannot fail anywhere.
+ await withoutLoadTimeout(async () => {
+ const h = manualHarness();
+ try {
+ const load = h.assets.loadTexture(KEY);
+ await h.settle();
+ const { handle } = await load;
+
+ const swallow = vi.spyOn(h.gpu.refs, 'set').mockReturnValue(h.gpu.refs);
+ h.assets.releaseTexture(KEY);
+ swallow.mockRestore();
+
+ const owed = h.gpu.created.filter((c) => (h.gpu.refs.get(c) ?? 0) > 0);
+ expect(owed, 'a swallowed release left no trace — the leak check is blind').toEqual([handle]);
+ } finally {
+ h.dispose();
+ }
+ });
+ });
+});
+
+describe('regression: the interleaving the property found', () => {
+ /**
+ * load(k) → invalidate(k) → the superseded load resolves → release(k).
+ *
+ * One ledger slot per key put the resolving load's acquire on the generation
+ * invalidate had just minted, and release() — finding no cache entry — freed
+ * nothing. One texture stranded per hot reload.
+ */
+ it('releases the handle a superseded load handed out', async () => {
+ await withoutLoadTimeout(async () => {
+ const h = manualHarness();
+ try {
+ const load = h.assets.loadTexture(KEY);
+ h.assets.invalidate(KEY);
+ await h.settle();
+ const { handle } = await load;
+
+ h.assets.releaseTexture(KEY);
+
+ expect(h.gpu.refs.get(handle), `handle ${handle} was stranded by the invalidate`).toBe(0);
+ expect(h.gpu.freed, 'a severed key must destroy its texture, not retain it').toContain(handle);
+ expect([...textureRefs(h.assets).keys()], 'an orphan refcount survived').toEqual([]);
+ } finally {
+ h.dispose();
+ }
+ });
+ });
+
+ it('releases both generations when a reload happens under a live holder', async () => {
+ await withoutLoadTimeout(async () => {
+ const h = manualHarness();
+ try {
+ const first = h.assets.loadTexture(KEY);
+ await h.settle();
+ const older = (await first).handle;
+
+ h.assets.invalidate(KEY); // bytes changed on disk
+
+ const second = h.assets.loadTexture(KEY);
+ await h.settle();
+ const newer = (await second).handle;
+ expect(newer).not.toBe(older);
+
+ h.assets.releaseTexture(KEY); // the first holder lets go
+ h.assets.releaseTexture(KEY); // the second holder lets go
+
+ expect(h.gpu.refs.get(older)).toBe(0);
+ expect(h.gpu.refs.get(newer)).toBe(0);
+ expect(h.gpu.doubleReleased).toEqual([]);
+ expect([...textureRefs(h.assets).keys()]).toEqual([]);
+ } finally {
+ h.dispose();
+ }
+ });
+ });
+});
+
+describe('findings the property surfaced but does not assert', () => {
+ /**
+ * A CALLER bug the engine survives: no count goes negative, nothing is
+ * freed twice. It does consume the reference an undelivered load holds, so
+ * that caller's `await` resolves with a destroyed handle.
+ * @remarks Recorded, not asserted away; a safer engine would ignore it.
+ */
+ it('an over-release consumes an undelivered load\'s reservation', async () => {
+ await withoutLoadTimeout(async () => {
+ const h = manualHarness();
+ try {
+ const load = h.assets.loadTexture(KEY);
+ await h.settle();
+ await new Promise((r) => setTimeout(r, 0)); // the SDK has its ref
+
+ h.assets.releaseTexture(KEY); // nobody was handed anything yet
+ const { handle } = await load;
+
+ expect(h.gpu.refs.get(handle), 'the reservation was consumed').toBe(0);
+ expect(h.gpu.doubleReleased, 'but never below zero — I3 holds').toEqual([]);
+ expect([...textureRefs(h.assets).keys()], 'and the ledger is clean').toEqual([]);
+ } finally {
+ h.dispose();
+ }
+ });
+ });
+
+ /**
+ * One release, two references: the flipped and unflipped variants are
+ * separate cache keys and releaseTexture drains one from each. Two loads of
+ * the same file therefore need only ONE release between them — an asymmetry
+ * with load*() that is easy to write a leak against.
+ */
+ it('one releaseTexture drops a reference from BOTH flip variants', async () => {
+ await withoutLoadTimeout(async () => {
+ const h = manualHarness();
+ try {
+ const flipped = h.assets.loadTexture(KEY);
+ await h.settle();
+ const raw = h.assets.loadTextureRaw(KEY);
+ await h.settle();
+ const a = (await flipped).handle;
+ const b = (await raw).handle;
+ expect(a).not.toBe(b);
+
+ h.assets.releaseTexture(KEY);
+
+ expect(h.gpu.refs.get(a), 'the flipped variant was released').toBe(0);
+ expect(h.gpu.refs.get(b), 'and so was the unflipped one, by the same call').toBe(0);
+ } finally {
+ h.dispose();
+ }
+ });
+ });
+});
diff --git a/sdk/tests/torture/asset-lifecycle.test.ts b/sdk/tests/torture/asset-lifecycle.test.ts
new file mode 100644
index 000000000..dd34e6f7e
--- /dev/null
+++ b/sdk/tests/torture/asset-lifecycle.test.ts
@@ -0,0 +1,406 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file The asset lifecycle, tortured.
+ *
+ * Every function below is individually correct and has the unit test to prove
+ * it. What none of them prove is that they are correct TOGETHER: a hot reload
+ * landing between a fetch resolving and the cache write, a release running
+ * while a superseded load is still in flight, a scene unloading on top of a
+ * load it started. The failures live in the order, and the order is what a
+ * hand-written test fixes to whatever its author already suspected.
+ *
+ * So the order is generated. fast-check picks the commands AND decides which
+ * pending load settles next, then shrinks both to the smallest sequence that
+ * still breaks an invariant.
+ *
+ * The invariants are the specification — the four the engine must never
+ * violate no matter the interleaving, stated once here and checked after every
+ * single command rather than at the end, so a failure names the command that
+ * caused it instead of the one that noticed.
+ *
+ * TORTURE_RUNS=5000 pnpm --filter ./sdk exec vitest run tests/torture
+ * TORTURE_SEED=1234 … replay one failure
+ */
+import { describe, it, expect } from 'vitest';
+import fc from 'fast-check';
+import {
+ makeAssetHarness, withoutLoadTimeout, textureRefs,
+ type AssetHarness,
+} from './assetHarness';
+
+const KEYS = ['tex/a.png', 'tex/b.png'] as const;
+const RUNS = Number(process.env.TORTURE_RUNS ?? 200);
+const SEED = process.env.TORTURE_SEED ? Number(process.env.TORTURE_SEED) : undefined;
+
+/** One completed load, as the test saw it — which generation asked, what it got. */
+interface Outcome {
+ key: string;
+ generation: number;
+ handle?: number;
+ failed?: boolean;
+ /** The unflipped variant lives under its own cache key. */
+ raw?: boolean;
+}
+
+/** What the test believes about the world; the real system must not contradict it. */
+interface Model {
+ /** Bumped by every invalidate: the generation a later resolution belongs to. */
+ generation: Map<string, number>;
+ /**
+ * Loads a NEW request could join: issued in the current generation and not
+ * yet settled. Superseded loads do not count — invalidate took the key from
+ * them, so the next caller must fetch rather than wait on stale bytes.
+ */
+ joinable: Map<string, number>;
+ /** Loader invocations the test can justify: one per key per generation, plus retries after failure. */
+ allowedLoads: Map<string, number>;
+ /**
+ * References the test has actually been HANDED and not yet given back.
+ *
+ * Release is gated on this, so the property models a well-behaved caller.
+ * Over-release is a caller bug with its own finding in the guards file;
+ * mixing it in here would mask every other failure behind it.
+ */
+ held: Map<string, number>;
+ /**
+ * releaseAll() calls so far. A load that spans one has no owner by the time
+ * it lands: the engine fails it when the reset beats its internal
+ * completion, and cannot un-resolve a promise that already settled. Either
+ * way the result is void by contract, so its handle is not asserted alive.
+ */
+ resets: number;
+ /** Held references to the UNFLIPPED variant, which has its own cache key. */
+ heldRaw: Map<string, number>;
+ /**
+ * The run's scheduler. It lives on the model because a command's `check`
+ * only ever sees the model, and "is there anything left to settle" has to be
+ * answerable there or the shrinker cannot drop settle points.
+ */
+ scheduler: fc.Scheduler;
+}
+
+interface Real {
+ harness: AssetHarness;
+ outcomes: Outcome[];
+ settled: Promise<unknown>[];
+}
+
+const freshModel = (scheduler: fc.Scheduler): Model => ({
+ generation: new Map(KEYS.map((k) => [k, 0])),
+ joinable: new Map(KEYS.map((k) => [k, 0])),
+ allowedLoads: new Map(KEYS.map((k) => [k, 0])),
+ held: new Map(KEYS.map((k) => [k, 0])),
+ heldRaw: new Map(KEYS.map((k) => [k, 0])),
+ resets: 0,
+ scheduler,
+});
+
+const bump = (m: Map<string, number>, k: string, by = 1): void => {
+ m.set(k, (m.get(k) ?? 0) + by);
+};
+
+// =============================================================================
+// The invariants — the specification
+// =============================================================================
+
+/**
+ * Checked after every command. Each throws with the sentence a failure should
+ * print; fast-check shrinks the command list until only the cause remains.
+ */
+function checkInvariants(model: Model, real: Real): void {
+ const { assets, gpu } = real.harness;
+
+ // I3 — a reference count is a count of references. Nothing may drive it
+ // below zero, and no key may keep a count once its entry is gone.
+ for (const [key, generations] of textureRefs(assets)) {
+ for (const generation of generations) {
+ if (generation.count < 0) {
+ throw new Error(`refCount(${key}, handle ${generation.handle}) = ${generation.count} — a release outran its acquire`);
+ }
+ if (generation.count === 0) {
+ throw new Error(`refCount(${key}, handle ${generation.handle}) = 0 is recorded rather than removed — a generation nothing can drain`);
+ }
+ }
+ if (generations.length === 0) {
+ throw new Error(`${key} keeps an empty generation list — a key nothing will ever clean up`);
+ }
+ }
+
+ // I4 — a disposed GPU resource must never be reachable again. Reviving one
+ // hands the renderer a freed handle, which draws garbage or crashes the driver.
+ for (const key of KEYS) {
+ const live = assets.getTexture(key);
+ if (live && gpu.freed.includes(live.handle)) {
+ throw new Error(`getTexture(${key}) returns handle ${live.handle}, which the pool already destroyed`);
+ }
+ }
+
+ // The pool must never be told to free the same handle twice.
+ if (gpu.doubleReleased.length > 0) {
+ throw new Error(`handle(s) released more times than acquired: ${gpu.doubleReleased.join(', ')}`);
+ }
+ // Handles the pool destroyed stay destroyed: none may be sitting in the
+ // resident set waiting to be revived into a future load.
+ for (const [key, handle] of gpu.resident) {
+ if (gpu.freed.includes(handle)) {
+ throw new Error(`handle ${handle} is resident under ${key} after the pool destroyed it`);
+ }
+ }
+
+ // I1 — at most one authoritative request per key per generation. A load
+ // that finished after losing the key must not leave a record that makes the
+ // next request start a duplicate fetch of bytes already in flight.
+ for (const key of KEYS) {
+ const calls = real.harness.loaderCalls.filter((c) => c === key).length;
+ const allowed = model.allowedLoads.get(key) ?? 0;
+ if (calls > allowed) {
+ throw new Error(
+ `${key} was fetched ${calls} times where ${allowed} request(s) were asked for — `
+ + 'a superseded load evicted its successor\'s record and the next caller started a duplicate',
+ );
+ }
+ }
+
+ // I2 — a resolution from a superseded generation must not touch the current
+ // one. It may still reach its own caller; it may not become the cached value.
+ for (const outcome of real.outcomes) {
+ if (outcome.failed || outcome.raw || outcome.handle === undefined) continue;
+ const currentGeneration = model.generation.get(outcome.key) ?? 0;
+ if (outcome.generation === currentGeneration) continue;
+ const live = assets.getTexture(outcome.key);
+ if (live?.handle === outcome.handle) {
+ throw new Error(
+ `${outcome.key} is cached as handle ${outcome.handle}, produced by generation `
+ + `${outcome.generation} while the current generation is ${currentGeneration} — stale bytes are live`,
+ );
+ }
+ }
+}
+
+/**
+ * The end-of-run claim, and the one that finds leaks: after the test releases
+ * every reference it holds, every handle the engine minted must have been given
+ * back. A handle nobody released is a texture resident for the life of the
+ * process, which is what "the editor got slower" is made of.
+ */
+function checkFullTeardown(model: Model, real: Real): void {
+ const { assets, gpu } = real.harness;
+ for (const key of KEYS) {
+ const owed = Math.max(model.held.get(key) ?? 0, model.heldRaw.get(key) ?? 0);
+ for (let i = 0; i < owed; i++) assets.releaseTexture(key);
+ model.held.set(key, 0);
+ model.heldRaw.set(key, 0);
+ }
+
+ // Every minted handle back at zero C++ references; one above zero is
+ // resident for the life of the process. Zero-ref handles the pool retains
+ // as evictable are its business, not a leak.
+ const owed = gpu.created.filter((h) => (gpu.refs.get(h) ?? 0) > 0);
+ if (owed.length > 0) {
+ throw new Error(
+ `${owed.length} texture handle(s) still referenced after every reference was dropped: `
+ + owed.map((h) => `${h}(refs ${gpu.refs.get(h)})`).join(', '),
+ );
+ }
+ const refs = textureRefs(assets);
+ if (refs.size > 0) {
+ throw new Error(
+ 'refcount ledger is not empty after full teardown: '
+ + [...refs].map(([k, g]) => `${k}=[${g.map((x) => `h${x.handle}x${x.count}`).join(',')}]`).join(', '),
+ );
+ }
+}
+
+// =============================================================================
+// The commands
+// =============================================================================
+
+type Cmd = fc.AsyncCommand<Model, Real>;
+
+class Load implements Cmd {
+ constructor(readonly key: string) {}
+ check = (): boolean => true;
+ async run(model: Model, real: Real): Promise<void> {
+ const generation = model.generation.get(this.key) ?? 0;
+ const resetsAtIssue = model.resets;
+ // A request that cannot join an in-flight or cached one is entitled to
+ // its own fetch; that is what makes a THIRD fetch a bug rather than work.
+ if ((model.joinable.get(this.key) ?? 0) === 0 && !real.harness.assets.getTexture(this.key)) {
+ bump(model.allowedLoads, this.key);
+ }
+ bump(model.joinable, this.key);
+
+ const settled = real.harness.assets.loadTexture(this.key).then(
+ (result) => {
+ // I4, at the only moment it can be judged: ordering matters, and
+ // a handle destroyed LATER is not evidence of anything.
+ if (model.resets === resetsAtIssue && real.harness.gpu.freed.includes(result.handle)) {
+ throw new Error(`load(${this.key}) resolved with handle ${result.handle}, which the pool had already destroyed`);
+ }
+ real.outcomes.push({ key: this.key, generation, handle: result.handle });
+ bump(model.held, this.key);
+ if (model.generation.get(this.key) === generation) bump(model.joinable, this.key, -1);
+ },
+ () => {
+ real.outcomes.push({ key: this.key, generation, failed: true });
+ if (model.generation.get(this.key) === generation) bump(model.joinable, this.key, -1);
+ },
+ );
+ real.settled.push(settled);
+ checkInvariants(model, real);
+ }
+ toString = (): string => `load(${this.key})`;
+}
+
+class Release implements Cmd {
+ constructor(readonly key: string) {}
+ /** Only what the caller was actually handed — see Model.held. */
+ check = (model: Model): boolean =>
+ (model.held.get(this.key) ?? 0) > 0 || (model.heldRaw.get(this.key) ?? 0) > 0;
+ async run(model: Model, real: Real): Promise<void> {
+ real.harness.assets.releaseTexture(this.key);
+ // ONE call, TWO references: the variants are separate cache keys and
+ // releaseTexture drains one from each, so two loads of the same file
+ // need only one release between them.
+ if ((model.held.get(this.key) ?? 0) > 0) bump(model.held, this.key, -1);
+ if ((model.heldRaw.get(this.key) ?? 0) > 0) bump(model.heldRaw, this.key, -1);
+ checkInvariants(model, real);
+ }
+ toString = (): string => `release(${this.key})`;
+}
+
+/** The raw (unflipped) variant — a second cache key for the same file, which
+ * releaseTexture drains in the same call. Two keys per path is where a fix for
+ * one variant quietly skips the other. */
+class LoadRaw implements Cmd {
+ constructor(readonly key: string) {}
+ check = (): boolean => true;
+ async run(model: Model, real: Real): Promise<void> {
+ const generation = model.generation.get(this.key) ?? 0;
+ const resetsAtIssue = model.resets;
+ bump(model.allowedLoads, this.key);
+ const settled = real.harness.assets.loadTextureRaw(this.key).then(
+ (result) => {
+ if (model.resets === resetsAtIssue && real.harness.gpu.freed.includes(result.handle)) {
+ throw new Error(`loadRaw(${this.key}) resolved with handle ${result.handle}, which the pool had already destroyed`);
+ }
+ real.outcomes.push({ key: this.key, generation, handle: result.handle, raw: true });
+ bump(model.heldRaw, this.key);
+ },
+ () => { real.outcomes.push({ key: this.key, generation, failed: true, raw: true }); },
+ );
+ real.settled.push(settled);
+ checkInvariants(model, real);
+ }
+ toString = (): string => `loadRaw(${this.key})`;
+}
+
+/** Scene unload: everything goes at once, including loads still in flight. */
+class ReleaseAll implements Cmd {
+ check = (): boolean => true;
+ async run(model: Model, real: Real): Promise<void> {
+ real.harness.assets.releaseAll();
+ model.resets++;
+ for (const key of KEYS) {
+ model.held.set(key, 0);
+ model.heldRaw.set(key, 0);
+ }
+ // NOT a new generation: dropping references does not stale the bytes, so
+ // a revive may hand the same handle back. Only invalidate ages an asset.
+ // Pending records are gone though, so nothing is joinable.
+ for (const key of KEYS) {
+ model.joinable.set(key, 0);
+ bump(model.allowedLoads, key);
+ }
+ checkInvariants(model, real);
+ }
+ toString = (): string => 'releaseAll()';
+}
+
+class Invalidate implements Cmd {
+ constructor(readonly key: string) {}
+ check = (): boolean => true;
+ async run(model: Model, real: Real): Promise<void> {
+ real.harness.assets.invalidate(this.key);
+ bump(model.generation, this.key);
+ // Everything still in flight belonged to the outgoing generation, so
+ // nothing is joinable any more: the next request is entitled to fetch.
+ model.joinable.set(this.key, 0);
+ // The bytes changed on disk, so the next request must fetch again — a new
+ // generation is entitled to a fetch even while the old one is in flight.
+ bump(model.allowedLoads, this.key);
+ checkInvariants(model, real);
+ }
+ toString = (): string => `invalidate(${this.key})`;
+}
+
+class FailNext implements Cmd {
+ constructor(readonly key: string) {}
+ check = (): boolean => true;
+ async run(model: Model, real: Real): Promise<void> {
+ real.harness.failNext(this.key);
+ // A failed load leaves nothing cached, so the retry is a fetch we asked for.
+ bump(model.allowedLoads, this.key);
+ checkInvariants(model, real);
+ }
+ toString = (): string => `failNext(${this.key})`;
+}
+
+/** Let exactly one pending load settle — the scheduler picks which. */
+class SettleOne implements Cmd {
+ check = (model: Model): boolean => model.scheduler.count() > 0;
+ async run(model: Model, real: Real): Promise<void> {
+ await model.scheduler.waitOne();
+ // A macrotask, not one microtask: a completion runs through several
+ // awaits inside AsyncCache, and a model that resumed early would hold
+ // fewer references than the engine has already recorded.
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ checkInvariants(model, real);
+ }
+ toString = (): string => 'settleOne()';
+}
+
+// =============================================================================
+// The property
+// =============================================================================
+
+describe('asset lifecycle under generated interleavings', () => {
+ it('holds every invariant no matter the order', async () => {
+ const key = fc.constantFrom(...KEYS);
+
+ await withoutLoadTimeout(() => fc.assert(
+ fc.asyncProperty(
+ fc.scheduler(),
+ fc.commands(
+ [
+ key.map((k) => new Load(k)),
+ key.map((k) => new Release(k)),
+ key.map((k) => new Invalidate(k)),
+ key.map((k) => new FailNext(k)),
+ key.map((k) => new LoadRaw(k)),
+ fc.constant(new ReleaseAll()),
+ fc.constant(new SettleOne()),
+ ],
+ { maxCommands: 24 },
+ ),
+ async (scheduler, commands) => {
+ const harness = makeAssetHarness(scheduler.scheduleFunction(async (k: string) => k));
+ const real: Real = { harness, outcomes: [], settled: [] };
+ const model = freshModel(scheduler);
+ try {
+ await fc.asyncModelRun(() => ({ model, real }), commands);
+ // Drain whatever the commands left in flight, in an order
+ // the scheduler chose and can shrink.
+ await scheduler.waitAll();
+ await Promise.all(real.settled);
+ checkFullTeardown(model, real);
+ } finally {
+ harness.dispose();
+ }
+ },
+ ),
+ { numRuns: RUNS, seed: SEED, verbose: true },
+ ));
+ }, 300_000);
+});
diff --git a/sdk/tests/torture/assetHarness.ts b/sdk/tests/torture/assetHarness.ts
new file mode 100644
index 000000000..ce7f260cf
--- /dev/null
+++ b/sdk/tests/torture/assetHarness.ts
@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file assetHarness — one asset system whose every async edge is schedulable.
+ *
+ * The bugs this suite hunts are not in a function, they are in an ORDER: a hot
+ * reload that lands between a fetch resolving and the cache write, a release
+ * that runs while a superseded load is still in flight. Reproducing one by hand
+ * means writing the interleaving you already suspect, which is why the ones
+ * nobody suspected survive.
+ *
+ * So nothing here settles on its own. The texture loader is a scheduled
+ * function; fast-check decides which pending load resolves next, and shrinks
+ * that decision along with the commands.
+ */
+import type * as fc from 'fast-check';
+import { Assets } from '../../src/asset/Assets';
+import type { Backend } from '../../src/asset/Backend';
+import type { TextureResult } from '../../src/asset/Assets';
+import { initResourceManager, shutdownResourceManager } from '../../src/wasm/resourceManager';
+import { textureResidencyKey } from '../../src/asset/loaders/TextureLoader';
+import { RuntimeConfig } from '../../src/defaults';
+
+/**
+ * The C++ texture pool, modelled rather than stubbed.
+ *
+ * A stub where release means gone leaves "a disposed resource never comes
+ * back" vacuous — with no residency there is nothing to revive. The real pool
+ * retains an unreferenced texture under its residency key, and revives it.
+ */
+export interface FakeGpu {
+ /** Handles minted by a loader. */
+ readonly created: number[];
+ /** Live C++ reference count per handle. Zero = evictable or freed. */
+ readonly refs: Map<number, number>;
+ /** Handles the pool actually destroyed. These must never be seen again. */
+ readonly freed: number[];
+ /** A release that drove a handle's count below zero — a double free. */
+ readonly doubleReleased: number[];
+ /** Unreferenced but retained, keyed by residency key; a load revives these. */
+ readonly resident: Map<string, number>;
+ /** Residency keys severed by invalidate. */
+ readonly severed: Set<string>;
+ /** Revives that actually happened — proof the path is exercised at all. */
+ readonly revived: number[];
+}
+
+export interface AssetHarness {
+ readonly assets: Assets;
+ readonly gpu: FakeGpu;
+ /** Loader invocations, in order, as `key` — how request dedup is observed. */
+ readonly loaderCalls: string[];
+ /** Make the next load of `key` reject instead of resolving. */
+ failNext(key: string): void;
+ dispose(): void;
+}
+
+const mockModule = {
+ _malloc: () => 0,
+ _free: () => {},
+ HEAPU8: new Uint8Array(64),
+ GL: null,
+ FS: null,
+} as never;
+
+/**
+ * Build the system under test.
+ *
+ * `scheduleLoad` wraps each loader call so the scheduler owns when it settles;
+ * pass `s.scheduleFunction(fn)` from a fast-check scheduler.
+ */
+export function makeAssetHarness(
+ scheduleLoad: (key: string) => Promise<TextureResult>,
+): AssetHarness {
+ const gpu: FakeGpu = {
+ created: [], refs: new Map(), freed: [], doubleReleased: [],
+ resident: new Map(), severed: new Set(), revived: [],
+ };
+ const residencyOf = new Map<number, string>();
+ // Revive is gated on the pool knowing the texture's size: a fake that
+ // answered null made every revive bail into a re-fetch, so the path the
+ // "never resurrect a destroyed handle" invariant guards went untested.
+ const dimsOf = new Map<number, { width: number; height: number }>();
+ const loaderCalls: string[] = [];
+ const failing = new Set<string>();
+ let nextHandle = 1;
+
+ const rm = {
+ registerTextureWithPath(handle: number, key: string): void {
+ residencyOf.set(handle, key);
+ },
+ releaseTexture(handle: number): void {
+ const next = (gpu.refs.get(handle) ?? 0) - 1;
+ gpu.refs.set(handle, next);
+ if (next < 0) {
+ gpu.doubleReleased.push(handle);
+ return;
+ }
+ if (next > 0) return;
+ // Unreferenced: retained under its residency key unless that key was
+ // severed, in which case the bytes are stale and the texture goes.
+ const key = residencyOf.get(handle);
+ if (key !== undefined && !gpu.severed.has(key)) gpu.resident.set(key, handle);
+ else gpu.freed.push(handle);
+ },
+ getTextureDimensions: (handle: number) => dimsOf.get(handle) ?? null,
+ setTextureMetadata: () => {},
+ acquireTextureByPath(key: string): number {
+ if (gpu.severed.has(key)) return 0;
+ const handle = gpu.resident.get(key);
+ if (handle === undefined) return 0;
+ gpu.resident.delete(key);
+ gpu.refs.set(handle, (gpu.refs.get(handle) ?? 0) + 1);
+ gpu.revived.push(handle);
+ return handle;
+ },
+ invalidateTexturePath(key: string): boolean {
+ gpu.severed.add(key);
+ const handle = gpu.resident.get(key);
+ if (handle === undefined) return false;
+ gpu.resident.delete(key);
+ gpu.freed.push(handle);
+ return true;
+ },
+ setTextureBudget: () => {},
+ trimTextureCache: () => 0,
+ } as never;
+
+ initResourceManager(rm);
+
+ const backend = {
+ fetchBinary: async () => new ArrayBuffer(8),
+ fetchText: async () => '{}',
+ resolveUrl: (p: string) => `http://torture/${p}`,
+ } as unknown as Backend;
+
+ const assets = Assets.create({ backend, module: mockModule });
+
+ // Every texture load becomes a scheduled task. The handle is minted here, at
+ // the moment the loader is CALLED, so a value produced by a superseded load
+ // is distinguishable from its successor's — which is the whole question.
+ const loader = assets.getTextureLoader() as unknown as {
+ load: (path: string) => Promise<TextureResult>;
+ loadRaw: (path: string) => Promise<TextureResult>;
+ };
+ // The real loader registers its handle with the pool before returning — that
+ // registration is what makes a later revive possible, so a harness that
+ // skipped it would leave the revive path untested.
+ const run = (flip: boolean) => async (path: string): Promise<TextureResult> => {
+ loaderCalls.push(path);
+ const shouldFail = failing.delete(path);
+ await scheduleLoad(path);
+ if (shouldFail) throw new Error(`torture: load failed for ${path}`);
+ const handle = nextHandle++;
+ gpu.created.push(handle);
+ gpu.refs.set(handle, 1);
+ dimsOf.set(handle, { width: 4, height: 4 });
+ rm.registerTextureWithPath(handle, textureResidencyKey(path, flip));
+ return { handle, width: 4, height: 4 } as TextureResult;
+ };
+ loader.load = run(true);
+ loader.loadRaw = run(false);
+
+ return {
+ assets,
+ gpu,
+ loaderCalls,
+ failNext: (key) => failing.add(key),
+ dispose: () => shutdownResourceManager(),
+ };
+}
+
+/**
+ * Timeouts are their own property, not background noise: the timer path here
+ * would fire on wall-clock time and settle loads the scheduler was still
+ * holding, which turns every interleaving into a different one than the seed
+ * says. Disabled per-run and restored on dispose.
+ */
+export function withoutLoadTimeout<T>(body: () => T): T {
+ const previous = RuntimeConfig.assetLoadTimeout;
+ RuntimeConfig.assetLoadTimeout = 0;
+ try {
+ return body();
+ } finally {
+ RuntimeConfig.assetLoadTimeout = previous;
+ }
+}
+
+/** The scheduler type, without making every caller import fast-check's namespace. */
+export type Scheduler = fc.Scheduler;
+
+/** One live generation of a texture: the handle its holders were given, and how many. */
+export interface TextureGeneration {
+ handle: number;
+ count: number;
+}
+
+/**
+ * The reference ledger, read through the private field on purpose: "a refcount
+ * is never negative" is an invariant about the counter itself, and routing it
+ * through a public accessor would test the accessor. Nothing else reaches in.
+ */
+export function textureRefs(assets: Assets): ReadonlyMap<string, readonly TextureGeneration[]> {
+ return (assets as unknown as { textureRefs_: Map<string, TextureGeneration[]> }).textureRefs_;
+}