A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
test(torture): generate the interleavings nobody thinks to write
esengine16af5e681f20
diff --git a/package.json b/package.jsonindex e54d436e2..b59797de6 100644--- a/package.json+++ b/package.json@@ -20,7 +20,8 @@"verify": "pnpm --filter ./sdk exec tsc --noEmit && node tools/api-surface.mjs --check && node tools/check-cycles.mjs && node tools/check-layers.mjs && node tools/check-project-settings.mjs && node tools/check-workflows.mjs && node tools/check-tool-calls.mjs && node tools/check-inspector-door.mjs && node tools/check-dirty-source.mjs && node tools/check-path-sandbox.mjs && node tools/check-comment-style.mjs && node tools/check-release-metadata.mjs && node tools/component-reference.mjs --check","perf": "node tools/perf-guard.mjs --check","prepare": "git config core.hooksPath .githooks || true",- "soak": "pnpm --filter ./sdk exec vitest run tests/soak"+ "soak": "pnpm --filter ./sdk exec vitest run tests/soak",+ "torture": "pnpm --filter ./sdk exec vitest run tests/torture"},"dependencies": {"chalk": "^5.3.0",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 92b7b8574..14550a371 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -133,6 +133,9 @@ importers:'@vitest/ui':specifier: ^2.1.0version: 2.1.9([email protected])+ fast-check:+ specifier: ^4.9.0+ version: 4.9.0happy-dom:specifier: ^15.11.0version: 15.11.7@@ -1931,6 +1934,10 @@ packages:resolution: {integrity: sha512-LmDxfWXwcTArk8fUEnOfSZpHOJ6zOMUJKOtFLFqJLoKJetuQG874Uc7/Kki7zFLzYybmZhp1M7+98pfMqeX8yA==}+ resolution: {integrity: sha512-7ms6T7SybUev/PQITciI0yLM2pOSFy5zpG8Ty7tQofcVaQUvrMXp6CBwqF6fThLCLOrfBtuHAtwq6Yu4XPCllg==}+ engines: {node: '>=12.17.0'}+resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}@@ -2861,6 +2868,9 @@ packages:resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}engines: {node: '>=6'}+ resolution: {integrity: sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==}+resolution: {integrity: sha512-PLgQXQ6H2FWCaeRak8vvk1GW462lMxB5s3Jm673N82zI4vqtVUPuZdffdZbPDFRoU8kAhItWFtPCWiPpp4/EDg==}@@ -5372,6 +5382,10 @@ snapshots:+ dependencies:+ pure-rand: 8.4.2+@@ -6312,6 +6326,8 @@ snapshots:optional: true+ [email protected]: {}+dependencies:tslib: 2.8.1diff --git a/sdk/package.json b/sdk/package.jsonindex 9f30737e1..d596a0bf8 100644--- a/sdk/package.json+++ b/sdk/package.json@@ -82,6 +82,7 @@"@types/ws": "^8.18.1","@vitest/coverage-v8": "^2.1.9","@vitest/ui": "^2.1.0",+ "fast-check": "^4.9.0","happy-dom": "^15.11.0","minigame-api-typings": "^3.8.16","rimraf": "^5.0.5",diff --git a/sdk/tests/torture/asset-lifecycle-guards.test.ts b/sdk/tests/torture/asset-lifecycle-guards.test.tsnew file mode 100644index 000000000..b2ecac79d--- /dev/null+++ b/sdk/tests/torture/asset-lifecycle-guards.test.ts@@ -0,0 +1,255 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file Proof that the torture harness tortures, and that its claims can fail.+ *+ * A generated property that passes in eight milliseconds is either watching an+ * engine with no bugs or watching nothing, and the two look identical from the+ * outside. So: the harness is asserted to actually drive loads and observe the+ * pool, the invariants are asserted to go red on a leak injected by hand, and+ * the counterexample the property found the first time it ran is kept here as+ * an ordinary regression test — spelled out, so it fails for one reason.+ */+import { describe, it, expect, vi } from 'vitest';+import { makeAssetHarness, withoutLoadTimeout, textureRefs } from './assetHarness';++const KEY = 'tex/a.png';++/** A loader whose settling this test controls, without a fast-check scheduler. */+function manualHarness() {+ const gates: Array<() => void> = [];+ const h = makeAssetHarness(+ (key) => new Promise((resolve) => { gates.push(() => resolve(key as never)); }),+ );+ return {+ ...h,+ /** Let the Nth outstanding load through (0 = oldest). */+ settle(index = 0) {+ const gate = gates.splice(index, 1)[0];+ if (!gate) throw new Error(`no outstanding load at index ${index}`);+ gate();+ return new Promise<void>((r) => setTimeout(r, 0));+ },+ outstanding: () => gates.length,+ };+}++describe('the torture harness', () => {+ it('actually drives the loader and sees the pool', async () => {+ await withoutLoadTimeout(async () => {+ const h = manualHarness();+ try {+ const load = h.assets.loadTexture(KEY);+ expect(h.loaderCalls, 'the patched loader was never called — I1 would be vacuous').toHaveLength(1);+ expect(h.outstanding(), 'the load settled on its own; nothing is schedulable').toBe(1);++ await h.settle();+ const result = await load;+ expect(h.gpu.created, 'no handle was minted — the GPU ledger sees nothing').toContain(result.handle);++ h.assets.releaseTexture(KEY);+ expect(h.gpu.refs.get(result.handle), 'release never reached the ResourceManager').toBe(0);+ } finally {+ h.dispose();+ }+ });+ });++ it('joins a second request to the load already in flight', async () => {+ // I1's premise. If two concurrent loads produced two fetches, the+ // invariant's budget arithmetic would be measuring the wrong thing.+ await withoutLoadTimeout(async () => {+ const h = manualHarness();+ try {+ const a = h.assets.loadTexture(KEY);+ const b = h.assets.loadTexture(KEY);+ expect(h.loaderCalls).toHaveLength(1);+ await h.settle();+ expect((await a).handle).toBe((await b).handle);+ } finally {+ h.dispose();+ }+ });+ });++ it('revives a released texture from the pool instead of re-fetching', async () => {+ // I4's premise. With no revive there is nothing that could hand back a+ // destroyed handle, and "a disposed resource never comes back" would be+ // a claim about a path the property never takes.+ await withoutLoadTimeout(async () => {+ const h = manualHarness();+ try {+ const first = h.assets.loadTexture(KEY);+ await h.settle();+ const handle = (await first).handle;+ h.assets.releaseTexture(KEY);+ expect(h.gpu.freed, 'the pool destroyed it instead of retaining it').not.toContain(handle);++ const second = h.assets.loadTexture(KEY);+ await new Promise((r) => setTimeout(r, 0));+ expect((await second).handle, 'the reload did not revive').toBe(handle);+ expect(h.gpu.revived).toContain(handle);+ expect(h.loaderCalls, 'a revive must not re-fetch').toHaveLength(1);+ } finally {+ h.dispose();+ }+ });+ });++ it('never revives across an invalidate', async () => {+ await withoutLoadTimeout(async () => {+ const h = manualHarness();+ try {+ const first = h.assets.loadTexture(KEY);+ await h.settle();+ const stale = (await first).handle;+ h.assets.releaseTexture(KEY); // retained as evictable+ h.assets.invalidate(KEY); // bytes changed -> residency severed++ expect(h.gpu.freed, 'severing must destroy the retained texture').toContain(stale);++ const second = h.assets.loadTexture(KEY);+ await h.settle();+ expect((await second).handle, 'a severed key served stale bytes').not.toBe(stale);+ expect(h.gpu.revived).toEqual([]);+ } finally {+ h.dispose();+ }+ });+ });++ it('goes red when a leak is injected', async () => {+ // The sabotage: a pool that quietly ignores releases. If the teardown+ // claim cannot fail here, it cannot fail anywhere.+ await withoutLoadTimeout(async () => {+ const h = manualHarness();+ try {+ const load = h.assets.loadTexture(KEY);+ await h.settle();+ const { handle } = await load;++ const swallow = vi.spyOn(h.gpu.refs, 'set').mockReturnValue(h.gpu.refs);+ h.assets.releaseTexture(KEY);+ swallow.mockRestore();++ const owed = h.gpu.created.filter((c) => (h.gpu.refs.get(c) ?? 0) > 0);+ expect(owed, 'a swallowed release left no trace — the leak check is blind').toEqual([handle]);+ } finally {+ h.dispose();+ }+ });+ });+});++describe('regression: the interleaving the property found', () => {+ /**+ * load(k) → invalidate(k) → the superseded load resolves → release(k).+ *+ * One ledger slot per key put the resolving load's acquire on the generation+ * invalidate had just minted, and release() — finding no cache entry — freed+ * nothing. One texture stranded per hot reload.+ */+ it('releases the handle a superseded load handed out', async () => {+ await withoutLoadTimeout(async () => {+ const h = manualHarness();+ try {+ const load = h.assets.loadTexture(KEY);+ h.assets.invalidate(KEY);+ await h.settle();+ const { handle } = await load;++ h.assets.releaseTexture(KEY);++ expect(h.gpu.refs.get(handle), `handle ${handle} was stranded by the invalidate`).toBe(0);+ expect(h.gpu.freed, 'a severed key must destroy its texture, not retain it').toContain(handle);+ expect([...textureRefs(h.assets).keys()], 'an orphan refcount survived').toEqual([]);+ } finally {+ h.dispose();+ }+ });+ });++ it('releases both generations when a reload happens under a live holder', async () => {+ await withoutLoadTimeout(async () => {+ const h = manualHarness();+ try {+ const first = h.assets.loadTexture(KEY);+ await h.settle();+ const older = (await first).handle;++ h.assets.invalidate(KEY); // bytes changed on disk++ const second = h.assets.loadTexture(KEY);+ await h.settle();+ const newer = (await second).handle;+ expect(newer).not.toBe(older);++ h.assets.releaseTexture(KEY); // the first holder lets go+ h.assets.releaseTexture(KEY); // the second holder lets go++ expect(h.gpu.refs.get(older)).toBe(0);+ expect(h.gpu.refs.get(newer)).toBe(0);+ expect(h.gpu.doubleReleased).toEqual([]);+ expect([...textureRefs(h.assets).keys()]).toEqual([]);+ } finally {+ h.dispose();+ }+ });+ });+});++describe('findings the property surfaced but does not assert', () => {+ /**+ * A CALLER bug the engine survives: no count goes negative, nothing is+ * freed twice. It does consume the reference an undelivered load holds, so+ * that caller's `await` resolves with a destroyed handle.+ * @remarks Recorded, not asserted away; a safer engine would ignore it.+ */+ it('an over-release consumes an undelivered load\'s reservation', async () => {+ await withoutLoadTimeout(async () => {+ const h = manualHarness();+ try {+ const load = h.assets.loadTexture(KEY);+ await h.settle();+ await new Promise((r) => setTimeout(r, 0)); // the SDK has its ref++ h.assets.releaseTexture(KEY); // nobody was handed anything yet+ const { handle } = await load;++ expect(h.gpu.refs.get(handle), 'the reservation was consumed').toBe(0);+ expect(h.gpu.doubleReleased, 'but never below zero — I3 holds').toEqual([]);+ expect([...textureRefs(h.assets).keys()], 'and the ledger is clean').toEqual([]);+ } finally {+ h.dispose();+ }+ });+ });++ /**+ * One release, two references: the flipped and unflipped variants are+ * separate cache keys and releaseTexture drains one from each. Two loads of+ * the same file therefore need only ONE release between them — an asymmetry+ * with load*() that is easy to write a leak against.+ */+ it('one releaseTexture drops a reference from BOTH flip variants', async () => {+ await withoutLoadTimeout(async () => {+ const h = manualHarness();+ try {+ const flipped = h.assets.loadTexture(KEY);+ await h.settle();+ const raw = h.assets.loadTextureRaw(KEY);+ await h.settle();+ const a = (await flipped).handle;+ const b = (await raw).handle;+ expect(a).not.toBe(b);++ h.assets.releaseTexture(KEY);++ expect(h.gpu.refs.get(a), 'the flipped variant was released').toBe(0);+ expect(h.gpu.refs.get(b), 'and so was the unflipped one, by the same call').toBe(0);+ } finally {+ h.dispose();+ }+ });+ });+});diff --git a/sdk/tests/torture/asset-lifecycle.test.ts b/sdk/tests/torture/asset-lifecycle.test.tsnew file mode 100644index 000000000..dd34e6f7e--- /dev/null+++ b/sdk/tests/torture/asset-lifecycle.test.ts@@ -0,0 +1,406 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file The asset lifecycle, tortured.+ *+ * Every function below is individually correct and has the unit test to prove+ * it. What none of them prove is that they are correct TOGETHER: a hot reload+ * landing between a fetch resolving and the cache write, a release running+ * while a superseded load is still in flight, a scene unloading on top of a+ * load it started. The failures live in the order, and the order is what a+ * hand-written test fixes to whatever its author already suspected.+ *+ * So the order is generated. fast-check picks the commands AND decides which+ * pending load settles next, then shrinks both to the smallest sequence that+ * still breaks an invariant.+ *+ * The invariants are the specification — the four the engine must never+ * violate no matter the interleaving, stated once here and checked after every+ * single command rather than at the end, so a failure names the command that+ * caused it instead of the one that noticed.+ *+ * TORTURE_RUNS=5000 pnpm --filter ./sdk exec vitest run tests/torture+ * TORTURE_SEED=1234 … replay one failure+ */+import { describe, it, expect } from 'vitest';+import fc from 'fast-check';+import {+ makeAssetHarness, withoutLoadTimeout, textureRefs,+ type AssetHarness,+} from './assetHarness';++const KEYS = ['tex/a.png', 'tex/b.png'] as const;+const RUNS = Number(process.env.TORTURE_RUNS ?? 200);+const SEED = process.env.TORTURE_SEED ? Number(process.env.TORTURE_SEED) : undefined;++/** One completed load, as the test saw it — which generation asked, what it got. */+interface Outcome {+ key: string;+ generation: number;+ handle?: number;+ failed?: boolean;+ /** The unflipped variant lives under its own cache key. */+ raw?: boolean;+}++/** What the test believes about the world; the real system must not contradict it. */+interface Model {+ /** Bumped by every invalidate: the generation a later resolution belongs to. */+ generation: Map<string, number>;+ /**+ * Loads a NEW request could join: issued in the current generation and not+ * yet settled. Superseded loads do not count — invalidate took the key from+ * them, so the next caller must fetch rather than wait on stale bytes.+ */+ joinable: Map<string, number>;+ /** Loader invocations the test can justify: one per key per generation, plus retries after failure. */+ allowedLoads: Map<string, number>;+ /**+ * References the test has actually been HANDED and not yet given back.+ *+ * Release is gated on this, so the property models a well-behaved caller.+ * Over-release is a caller bug with its own finding in the guards file;+ * mixing it in here would mask every other failure behind it.+ */+ held: Map<string, number>;+ /**+ * releaseAll() calls so far. A load that spans one has no owner by the time+ * it lands: the engine fails it when the reset beats its internal+ * completion, and cannot un-resolve a promise that already settled. Either+ * way the result is void by contract, so its handle is not asserted alive.+ */+ resets: number;+ /** Held references to the UNFLIPPED variant, which has its own cache key. */+ heldRaw: Map<string, number>;+ /**+ * The run's scheduler. It lives on the model because a command's `check`+ * only ever sees the model, and "is there anything left to settle" has to be+ * answerable there or the shrinker cannot drop settle points.+ */+ scheduler: fc.Scheduler;+}++interface Real {+ harness: AssetHarness;+ outcomes: Outcome[];+ settled: Promise<unknown>[];+}++const freshModel = (scheduler: fc.Scheduler): Model => ({+ generation: new Map(KEYS.map((k) => [k, 0])),+ joinable: new Map(KEYS.map((k) => [k, 0])),+ allowedLoads: new Map(KEYS.map((k) => [k, 0])),+ held: new Map(KEYS.map((k) => [k, 0])),+ heldRaw: new Map(KEYS.map((k) => [k, 0])),+ resets: 0,+ scheduler,+});++const bump = (m: Map<string, number>, k: string, by = 1): void => {+ m.set(k, (m.get(k) ?? 0) + by);+};++// =============================================================================+// The invariants — the specification+// =============================================================================++/**+ * Checked after every command. Each throws with the sentence a failure should+ * print; fast-check shrinks the command list until only the cause remains.+ */+function checkInvariants(model: Model, real: Real): void {+ const { assets, gpu } = real.harness;++ // I3 — a reference count is a count of references. Nothing may drive it+ // below zero, and no key may keep a count once its entry is gone.+ for (const [key, generations] of textureRefs(assets)) {+ for (const generation of generations) {+ if (generation.count < 0) {+ throw new Error(`refCount(${key}, handle ${generation.handle}) = ${generation.count} — a release outran its acquire`);+ }+ if (generation.count === 0) {+ throw new Error(`refCount(${key}, handle ${generation.handle}) = 0 is recorded rather than removed — a generation nothing can drain`);+ }+ }+ if (generations.length === 0) {+ throw new Error(`${key} keeps an empty generation list — a key nothing will ever clean up`);+ }+ }++ // I4 — a disposed GPU resource must never be reachable again. Reviving one+ // hands the renderer a freed handle, which draws garbage or crashes the driver.+ for (const key of KEYS) {+ const live = assets.getTexture(key);+ if (live && gpu.freed.includes(live.handle)) {+ throw new Error(`getTexture(${key}) returns handle ${live.handle}, which the pool already destroyed`);+ }+ }++ // The pool must never be told to free the same handle twice.+ if (gpu.doubleReleased.length > 0) {+ throw new Error(`handle(s) released more times than acquired: ${gpu.doubleReleased.join(', ')}`);+ }+ // Handles the pool destroyed stay destroyed: none may be sitting in the+ // resident set waiting to be revived into a future load.+ for (const [key, handle] of gpu.resident) {+ if (gpu.freed.includes(handle)) {+ throw new Error(`handle ${handle} is resident under ${key} after the pool destroyed it`);+ }+ }++ // I1 — at most one authoritative request per key per generation. A load+ // that finished after losing the key must not leave a record that makes the+ // next request start a duplicate fetch of bytes already in flight.+ for (const key of KEYS) {+ const calls = real.harness.loaderCalls.filter((c) => c === key).length;+ const allowed = model.allowedLoads.get(key) ?? 0;+ if (calls > allowed) {+ throw new Error(+ `${key} was fetched ${calls} times where ${allowed} request(s) were asked for — `+ + 'a superseded load evicted its successor\'s record and the next caller started a duplicate',+ );+ }+ }++ // I2 — a resolution from a superseded generation must not touch the current+ // one. It may still reach its own caller; it may not become the cached value.+ for (const outcome of real.outcomes) {+ if (outcome.failed || outcome.raw || outcome.handle === undefined) continue;+ const currentGeneration = model.generation.get(outcome.key) ?? 0;+ if (outcome.generation === currentGeneration) continue;+ const live = assets.getTexture(outcome.key);+ if (live?.handle === outcome.handle) {+ throw new Error(+ `${outcome.key} is cached as handle ${outcome.handle}, produced by generation `+ + `${outcome.generation} while the current generation is ${currentGeneration} — stale bytes are live`,+ );+ }+ }+}++/**+ * The end-of-run claim, and the one that finds leaks: after the test releases+ * every reference it holds, every handle the engine minted must have been given+ * back. A handle nobody released is a texture resident for the life of the+ * process, which is what "the editor got slower" is made of.+ */+function checkFullTeardown(model: Model, real: Real): void {+ const { assets, gpu } = real.harness;+ for (const key of KEYS) {+ const owed = Math.max(model.held.get(key) ?? 0, model.heldRaw.get(key) ?? 0);+ for (let i = 0; i < owed; i++) assets.releaseTexture(key);+ model.held.set(key, 0);+ model.heldRaw.set(key, 0);+ }++ // Every minted handle back at zero C++ references; one above zero is+ // resident for the life of the process. Zero-ref handles the pool retains+ // as evictable are its business, not a leak.+ const owed = gpu.created.filter((h) => (gpu.refs.get(h) ?? 0) > 0);+ if (owed.length > 0) {+ throw new Error(+ `${owed.length} texture handle(s) still referenced after every reference was dropped: `+ + owed.map((h) => `${h}(refs ${gpu.refs.get(h)})`).join(', '),+ );+ }+ const refs = textureRefs(assets);+ if (refs.size > 0) {+ throw new Error(+ 'refcount ledger is not empty after full teardown: '+ + [...refs].map(([k, g]) => `${k}=[${g.map((x) => `h${x.handle}x${x.count}`).join(',')}]`).join(', '),+ );+ }+}++// =============================================================================+// The commands+// =============================================================================++type Cmd = fc.AsyncCommand<Model, Real>;++class Load implements Cmd {+ constructor(readonly key: string) {}+ check = (): boolean => true;+ async run(model: Model, real: Real): Promise<void> {+ const generation = model.generation.get(this.key) ?? 0;+ const resetsAtIssue = model.resets;+ // A request that cannot join an in-flight or cached one is entitled to+ // its own fetch; that is what makes a THIRD fetch a bug rather than work.+ if ((model.joinable.get(this.key) ?? 0) === 0 && !real.harness.assets.getTexture(this.key)) {+ bump(model.allowedLoads, this.key);+ }+ bump(model.joinable, this.key);++ const settled = real.harness.assets.loadTexture(this.key).then(+ (result) => {+ // I4, at the only moment it can be judged: ordering matters, and+ // a handle destroyed LATER is not evidence of anything.+ if (model.resets === resetsAtIssue && real.harness.gpu.freed.includes(result.handle)) {+ throw new Error(`load(${this.key}) resolved with handle ${result.handle}, which the pool had already destroyed`);+ }+ real.outcomes.push({ key: this.key, generation, handle: result.handle });+ bump(model.held, this.key);+ if (model.generation.get(this.key) === generation) bump(model.joinable, this.key, -1);+ },+ () => {+ real.outcomes.push({ key: this.key, generation, failed: true });+ if (model.generation.get(this.key) === generation) bump(model.joinable, this.key, -1);+ },+ );+ real.settled.push(settled);+ checkInvariants(model, real);+ }+ toString = (): string => `load(${this.key})`;+}++class Release implements Cmd {+ constructor(readonly key: string) {}+ /** Only what the caller was actually handed — see Model.held. */+ check = (model: Model): boolean =>+ (model.held.get(this.key) ?? 0) > 0 || (model.heldRaw.get(this.key) ?? 0) > 0;+ async run(model: Model, real: Real): Promise<void> {+ real.harness.assets.releaseTexture(this.key);+ // ONE call, TWO references: the variants are separate cache keys and+ // releaseTexture drains one from each, so two loads of the same file+ // need only one release between them.+ if ((model.held.get(this.key) ?? 0) > 0) bump(model.held, this.key, -1);+ if ((model.heldRaw.get(this.key) ?? 0) > 0) bump(model.heldRaw, this.key, -1);+ checkInvariants(model, real);+ }+ toString = (): string => `release(${this.key})`;+}++/** The raw (unflipped) variant — a second cache key for the same file, which+ * releaseTexture drains in the same call. Two keys per path is where a fix for+ * one variant quietly skips the other. */+class LoadRaw implements Cmd {+ constructor(readonly key: string) {}+ check = (): boolean => true;+ async run(model: Model, real: Real): Promise<void> {+ const generation = model.generation.get(this.key) ?? 0;+ const resetsAtIssue = model.resets;+ bump(model.allowedLoads, this.key);+ const settled = real.harness.assets.loadTextureRaw(this.key).then(+ (result) => {+ if (model.resets === resetsAtIssue && real.harness.gpu.freed.includes(result.handle)) {+ throw new Error(`loadRaw(${this.key}) resolved with handle ${result.handle}, which the pool had already destroyed`);+ }+ real.outcomes.push({ key: this.key, generation, handle: result.handle, raw: true });+ bump(model.heldRaw, this.key);+ },+ () => { real.outcomes.push({ key: this.key, generation, failed: true, raw: true }); },+ );+ real.settled.push(settled);+ checkInvariants(model, real);+ }+ toString = (): string => `loadRaw(${this.key})`;+}++/** Scene unload: everything goes at once, including loads still in flight. */+class ReleaseAll implements Cmd {+ check = (): boolean => true;+ async run(model: Model, real: Real): Promise<void> {+ real.harness.assets.releaseAll();+ model.resets++;+ for (const key of KEYS) {+ model.held.set(key, 0);+ model.heldRaw.set(key, 0);+ }+ // NOT a new generation: dropping references does not stale the bytes, so+ // a revive may hand the same handle back. Only invalidate ages an asset.+ // Pending records are gone though, so nothing is joinable.+ for (const key of KEYS) {+ model.joinable.set(key, 0);+ bump(model.allowedLoads, key);+ }+ checkInvariants(model, real);+ }+ toString = (): string => 'releaseAll()';+}++class Invalidate implements Cmd {+ constructor(readonly key: string) {}+ check = (): boolean => true;+ async run(model: Model, real: Real): Promise<void> {+ real.harness.assets.invalidate(this.key);+ bump(model.generation, this.key);+ // Everything still in flight belonged to the outgoing generation, so+ // nothing is joinable any more: the next request is entitled to fetch.+ model.joinable.set(this.key, 0);+ // The bytes changed on disk, so the next request must fetch again — a new+ // generation is entitled to a fetch even while the old one is in flight.+ bump(model.allowedLoads, this.key);+ checkInvariants(model, real);+ }+ toString = (): string => `invalidate(${this.key})`;+}++class FailNext implements Cmd {+ constructor(readonly key: string) {}+ check = (): boolean => true;+ async run(model: Model, real: Real): Promise<void> {+ real.harness.failNext(this.key);+ // A failed load leaves nothing cached, so the retry is a fetch we asked for.+ bump(model.allowedLoads, this.key);+ checkInvariants(model, real);+ }+ toString = (): string => `failNext(${this.key})`;+}++/** Let exactly one pending load settle — the scheduler picks which. */+class SettleOne implements Cmd {+ check = (model: Model): boolean => model.scheduler.count() > 0;+ async run(model: Model, real: Real): Promise<void> {+ await model.scheduler.waitOne();+ // A macrotask, not one microtask: a completion runs through several+ // awaits inside AsyncCache, and a model that resumed early would hold+ // fewer references than the engine has already recorded.+ await new Promise((resolve) => setTimeout(resolve, 0));+ checkInvariants(model, real);+ }+ toString = (): string => 'settleOne()';+}++// =============================================================================+// The property+// =============================================================================++describe('asset lifecycle under generated interleavings', () => {+ it('holds every invariant no matter the order', async () => {+ const key = fc.constantFrom(...KEYS);++ await withoutLoadTimeout(() => fc.assert(+ fc.asyncProperty(+ fc.scheduler(),+ fc.commands(+ [+ key.map((k) => new Load(k)),+ key.map((k) => new Release(k)),+ key.map((k) => new Invalidate(k)),+ key.map((k) => new FailNext(k)),+ key.map((k) => new LoadRaw(k)),+ fc.constant(new ReleaseAll()),+ fc.constant(new SettleOne()),+ ],+ { maxCommands: 24 },+ ),+ async (scheduler, commands) => {+ const harness = makeAssetHarness(scheduler.scheduleFunction(async (k: string) => k));+ const real: Real = { harness, outcomes: [], settled: [] };+ const model = freshModel(scheduler);+ try {+ await fc.asyncModelRun(() => ({ model, real }), commands);+ // Drain whatever the commands left in flight, in an order+ // the scheduler chose and can shrink.+ await scheduler.waitAll();+ await Promise.all(real.settled);+ checkFullTeardown(model, real);+ } finally {+ harness.dispose();+ }+ },+ ),+ { numRuns: RUNS, seed: SEED, verbose: true },+ ));+ }, 300_000);+});diff --git a/sdk/tests/torture/assetHarness.ts b/sdk/tests/torture/assetHarness.tsnew file mode 100644index 000000000..ce7f260cf--- /dev/null+++ b/sdk/tests/torture/assetHarness.ts@@ -0,0 +1,205 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file assetHarness — one asset system whose every async edge is schedulable.+ *+ * The bugs this suite hunts are not in a function, they are in an ORDER: a hot+ * reload that lands between a fetch resolving and the cache write, a release+ * that runs while a superseded load is still in flight. Reproducing one by hand+ * means writing the interleaving you already suspect, which is why the ones+ * nobody suspected survive.+ *+ * So nothing here settles on its own. The texture loader is a scheduled+ * function; fast-check decides which pending load resolves next, and shrinks+ * that decision along with the commands.+ */+import type * as fc from 'fast-check';+import { Assets } from '../../src/asset/Assets';+import type { Backend } from '../../src/asset/Backend';+import type { TextureResult } from '../../src/asset/Assets';+import { initResourceManager, shutdownResourceManager } from '../../src/wasm/resourceManager';+import { textureResidencyKey } from '../../src/asset/loaders/TextureLoader';+import { RuntimeConfig } from '../../src/defaults';++/**+ * The C++ texture pool, modelled rather than stubbed.+ *+ * A stub where release means gone leaves "a disposed resource never comes+ * back" vacuous — with no residency there is nothing to revive. The real pool+ * retains an unreferenced texture under its residency key, and revives it.+ */+export interface FakeGpu {+ /** Handles minted by a loader. */+ readonly created: number[];+ /** Live C++ reference count per handle. Zero = evictable or freed. */+ readonly refs: Map<number, number>;+ /** Handles the pool actually destroyed. These must never be seen again. */+ readonly freed: number[];+ /** A release that drove a handle's count below zero — a double free. */+ readonly doubleReleased: number[];+ /** Unreferenced but retained, keyed by residency key; a load revives these. */+ readonly resident: Map<string, number>;+ /** Residency keys severed by invalidate. */+ readonly severed: Set<string>;+ /** Revives that actually happened — proof the path is exercised at all. */+ readonly revived: number[];+}++export interface AssetHarness {+ readonly assets: Assets;+ readonly gpu: FakeGpu;+ /** Loader invocations, in order, as `key` — how request dedup is observed. */+ readonly loaderCalls: string[];+ /** Make the next load of `key` reject instead of resolving. */+ failNext(key: string): void;+ dispose(): void;+}++const mockModule = {+ _malloc: () => 0,+ _free: () => {},+ HEAPU8: new Uint8Array(64),+ GL: null,+ FS: null,+} as never;++/**+ * Build the system under test.+ *+ * `scheduleLoad` wraps each loader call so the scheduler owns when it settles;+ * pass `s.scheduleFunction(fn)` from a fast-check scheduler.+ */+export function makeAssetHarness(+ scheduleLoad: (key: string) => Promise<TextureResult>,+): AssetHarness {+ const gpu: FakeGpu = {+ created: [], refs: new Map(), freed: [], doubleReleased: [],+ resident: new Map(), severed: new Set(), revived: [],+ };+ const residencyOf = new Map<number, string>();+ // Revive is gated on the pool knowing the texture's size: a fake that+ // answered null made every revive bail into a re-fetch, so the path the+ // "never resurrect a destroyed handle" invariant guards went untested.+ const dimsOf = new Map<number, { width: number; height: number }>();+ const loaderCalls: string[] = [];+ const failing = new Set<string>();+ let nextHandle = 1;++ const rm = {+ registerTextureWithPath(handle: number, key: string): void {+ residencyOf.set(handle, key);+ },+ releaseTexture(handle: number): void {+ const next = (gpu.refs.get(handle) ?? 0) - 1;+ gpu.refs.set(handle, next);+ if (next < 0) {+ gpu.doubleReleased.push(handle);+ return;+ }+ if (next > 0) return;+ // Unreferenced: retained under its residency key unless that key was+ // severed, in which case the bytes are stale and the texture goes.+ const key = residencyOf.get(handle);+ if (key !== undefined && !gpu.severed.has(key)) gpu.resident.set(key, handle);+ else gpu.freed.push(handle);+ },+ getTextureDimensions: (handle: number) => dimsOf.get(handle) ?? null,+ setTextureMetadata: () => {},+ acquireTextureByPath(key: string): number {+ if (gpu.severed.has(key)) return 0;+ const handle = gpu.resident.get(key);+ if (handle === undefined) return 0;+ gpu.resident.delete(key);+ gpu.refs.set(handle, (gpu.refs.get(handle) ?? 0) + 1);+ gpu.revived.push(handle);+ return handle;+ },+ invalidateTexturePath(key: string): boolean {+ gpu.severed.add(key);+ const handle = gpu.resident.get(key);+ if (handle === undefined) return false;+ gpu.resident.delete(key);+ gpu.freed.push(handle);+ return true;+ },+ setTextureBudget: () => {},+ trimTextureCache: () => 0,+ } as never;++ initResourceManager(rm);++ const backend = {+ fetchBinary: async () => new ArrayBuffer(8),+ fetchText: async () => '{}',+ resolveUrl: (p: string) => `http://torture/${p}`,+ } as unknown as Backend;++ const assets = Assets.create({ backend, module: mockModule });++ // Every texture load becomes a scheduled task. The handle is minted here, at+ // the moment the loader is CALLED, so a value produced by a superseded load+ // is distinguishable from its successor's — which is the whole question.+ const loader = assets.getTextureLoader() as unknown as {+ load: (path: string) => Promise<TextureResult>;+ loadRaw: (path: string) => Promise<TextureResult>;+ };+ // The real loader registers its handle with the pool before returning — that+ // registration is what makes a later revive possible, so a harness that+ // skipped it would leave the revive path untested.+ const run = (flip: boolean) => async (path: string): Promise<TextureResult> => {+ loaderCalls.push(path);+ const shouldFail = failing.delete(path);+ await scheduleLoad(path);+ if (shouldFail) throw new Error(`torture: load failed for ${path}`);+ const handle = nextHandle++;+ gpu.created.push(handle);+ gpu.refs.set(handle, 1);+ dimsOf.set(handle, { width: 4, height: 4 });+ rm.registerTextureWithPath(handle, textureResidencyKey(path, flip));+ return { handle, width: 4, height: 4 } as TextureResult;+ };+ loader.load = run(true);+ loader.loadRaw = run(false);++ return {+ assets,+ gpu,+ loaderCalls,+ failNext: (key) => failing.add(key),+ dispose: () => shutdownResourceManager(),+ };+}++/**+ * Timeouts are their own property, not background noise: the timer path here+ * would fire on wall-clock time and settle loads the scheduler was still+ * holding, which turns every interleaving into a different one than the seed+ * says. Disabled per-run and restored on dispose.+ */+export function withoutLoadTimeout<T>(body: () => T): T {+ const previous = RuntimeConfig.assetLoadTimeout;+ RuntimeConfig.assetLoadTimeout = 0;+ try {+ return body();+ } finally {+ RuntimeConfig.assetLoadTimeout = previous;+ }+}++/** The scheduler type, without making every caller import fast-check's namespace. */+export type Scheduler = fc.Scheduler;++/** One live generation of a texture: the handle its holders were given, and how many. */+export interface TextureGeneration {+ handle: number;+ count: number;+}++/**+ * The reference ledger, read through the private field on purpose: "a refcount+ * is never negative" is an invariant about the counter itself, and routing it+ * through a public accessor would test the accessor. Nothing else reaches in.+ */+export function textureRefs(assets: Assets): ReadonlyMap<string, readonly TextureGeneration[]> {+ return (assets as unknown as { textureRefs_: Map<string, TextureGeneration[]> }).textureRefs_;+}