A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(export): a playable's engine travels compressed
esengineef4e89441c1e
diff --git a/pipeline/src/export/exportPlayable.ts b/pipeline/src/export/exportPlayable.tsindex b851df0b3..2fbe5529f 100644--- a/pipeline/src/export/exportPlayable.ts+++ b/pipeline/src/export/exportPlayable.ts@@ -22,6 +22,7 @@ import {} from '../project/runtimeConfig';import { writeFile, mkdir, readFile, stat, rm } from 'node:fs/promises';import { existsSync } from 'node:fs';+import { deflateRawSync } from 'node:zlib';import path from 'node:path';import { cookAssets } from '../assets/cookAssets';import type { OnExportProgress } from './exportProgress';@@ -72,6 +73,30 @@ const mimeOf = (p: string): string => MIME[path.extname(p).slice(1).toLowerCase(/** Escape `</script` so inlined content can't close the host <script> early. */const inlineSafe = (s: string): string => s.replace(/<\/script/gi, '<\\/script');+/**+ * A binary payload on its way into an HTML file: deflated, then base64.+ *+ * Base64 costs +33% over the bytes it carries, and the bytes are wasm — which+ * deflates to under a third. Compressing before encoding is worth more than any+ * denser encoding could be: the engine's own module goes 1.542MB → 2.056MB+ * encoded raw, against 0.597MB encoded this way.+ *+ * `n` is the original length, so the decoder allocates once and a stream that+ * does not fill it exactly is caught here rather than at some later use. Raw+ * deflate: `inflateRaw` is the only reader, and a wrapper's checksum would only+ * re-answer what instantiating the module answers immediately after.+ */+interface PackedBytes {+ /** base64 of the raw-deflated bytes. */+ z: string;+ /** Length before deflating. */+ n: number;+}++function pack(bytes: Uint8Array | Buffer): PackedBytes {+ return { z: deflateRawSync(bytes, { level: 9 }).toString('base64'), n: bytes.length };+}+/*** The playable page. Deliberately WITHOUT the web export's rotate-to-fit overlay:* inside an ad SDK the container's size is the SDK's business, so `@media@@ -129,14 +154,14 @@ async function collectSideModules(wasmDir: string,errors: string[],physicsEnabled: boolean,-): Promise<Record<string, { glueBase64: string; wasmBase64: string }>> {+): Promise<Record<string, { glue: PackedBytes; wasm: PackedBytes }>> {const ids = await scanSideModuleIds({root, includedPaths, cookEntries: manifestEntries, stagedDir: cookDir, physicsEnabled,});const { files, unknown } = sideModuleFiles(ids);for (const id of unknown) errors.push(`internal: no artifact mapping for side module "${id}"`);- const registry: Record<string, { glueBase64: string; wasmBase64: string }> = {};+ const registry: Record<string, { glue: PackedBytes; wasm: PackedBytes }> = {};for (const { id, file } of files) {const gluePath = path.join(wasmDir, `${file}.js`);const wasmPath = path.join(wasmDir, `${file}.wasm`);@@ -145,8 +170,8 @@ async function collectSideModules(continue;}registry[id] = {- glueBase64: (await readFile(gluePath)).toString('base64'),- wasmBase64: (await readFile(wasmPath)).toString('base64'),+ glue: pack(await readFile(gluePath)),+ wasm: pack(await readFile(wasmPath)),};}return registry;@@ -298,13 +323,13 @@ export async function exportPlayable(opts: {// inlined — the host loads the glue via a blob module + feeds the wasm (base64)// through instantiateWasm. No separate single-file build needed.progress({ phase: 'Inlining engine' });- let glue = '';- let wasmB64 = '';+ let glue: PackedBytes = { z: '', n: 0 };+ let wasm: PackedBytes = { z: '', n: 0 };const gluePath = path.join(opts.wasmDir, 'esengine.js');const wasmPath = path.join(opts.wasmDir, 'esengine.wasm');if (existsSync(gluePath) && existsSync(wasmPath)) {- glue = await readFile(gluePath, 'utf8');- wasmB64 = (await readFile(wasmPath)).toString('base64');+ glue = pack(await readFile(gluePath));+ wasm = pack(await readFile(wasmPath));} else {errors.push(`engine runtime not found in ${opts.wasmDir} (need esengine.js + esengine.wasm)`);}@@ -327,7 +352,7 @@ export async function exportPlayable(opts: {};const globals =span('wasm/esengine.js', `window.__ENGINE_GLUE__=${JSON.stringify(glue)};`) +- span('wasm/esengine.wasm', `window.__ENGINE_WASM__=${JSON.stringify(wasmB64)};`) ++ span('wasm/esengine.wasm', `window.__ENGINE_WASM__=${JSON.stringify(wasm)};`) +span('wasm/side-modules', `window.__SIDE_MODULES__=${JSON.stringify(sideModules)};`) +`window.__GAME_ASSETS__=${JSON.stringify(assets)};` +`window.__GAME_PATHMAP__=${JSON.stringify(pathMap)};` +diff --git a/pipeline/src/runtime/inflate.ts b/pipeline/src/runtime/inflate.tsnew file mode 100644index 000000000..5b7b0b5b6--- /dev/null+++ b/pipeline/src/runtime/inflate.ts@@ -0,0 +1,217 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file inflate.ts — raw DEFLATE, for the payloads a single-file build inlines.+ *+ * A playable ad is one HTML file under a hard byte cap, so its engine+ * arrives as text: base64 costs +33% over the binary, and the binary+ * itself is a 1.5MB wasm that deflates to 29% of that. Compressing before+ * encoding is worth more than any denser encoding could be.+ *+ * The browser has `DecompressionStream`, and this does not use it. It+ * landed in Safari 16.4, and a playable that white-screens on the older+ * WebViews still in an ad network's traffic is worse than one that is+ * larger — the impression is paid for either way. A fallback path would+ * be a second decoder that only some devices ever run, which is the shape+ * every "it works on the machine that tested it" bug comes in. So there+ * is one path, it runs everywhere, and every device exercises the code+ * the tests exercise.+ *+ * RAW deflate (no zlib or gzip wrapper): the producer is+ * `deflateRawSync`, the size is known from the manifest, and the checksum+ * a wrapper carries would only re-answer what `WebAssembly.instantiate`+ * answers immediately after.+ *+ * Structure follows zlib's `puff.c` — the canonical-Huffman decode walks+ * code lengths a bit at a time rather than building a lookup table.+ * Measured on the engine's own wasm, that is fast enough to not be worth+ * the bigger decoder (see inflate.test.ts, which holds the rate).+ */++/** Longest Huffman code DEFLATE permits. */+const MAX_BITS = 15;++/** RFC 1951 §3.2.5 — length codes 257..285. */+const LENGTH_BASE = [+ 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59,+ 67, 83, 99, 115, 131, 163, 195, 227, 258,+];+const LENGTH_EXTRA = [+ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,+ 4, 4, 4, 4, 5, 5, 5, 5, 0,+];+/** RFC 1951 §3.2.5 — distance codes 0..29. */+const DIST_BASE = [+ 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769,+ 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577,+];+const DIST_EXTRA = [+ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,+ 9, 9, 10, 10, 11, 11, 12, 12, 13, 13,+];+/** RFC 1951 §3.2.7 — the order code lengths are themselves written in. */+const CLEN_ORDER = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15];++/**+ * A canonical Huffman table as counts-per-length plus symbols in canonical+ * order — the two arrays the length-walking decode needs, and nothing else.+ */+interface Huffman {+ counts: Int32Array;+ symbols: Int32Array;+}++function buildHuffman(lengths: Uint8Array, n: number): Huffman {+ const counts = new Int32Array(MAX_BITS + 1);+ for (let i = 0; i < n; i++) counts[lengths[i]]++;+ counts[0] = 0;+ const offsets = new Int32Array(MAX_BITS + 1);+ for (let len = 1; len <= MAX_BITS; len++) offsets[len] = offsets[len - 1] + counts[len - 1];+ const symbols = new Int32Array(n);+ for (let i = 0; i < n; i++) if (lengths[i]) symbols[offsets[lengths[i]]++] = i;+ return { counts, symbols };+}++/** The fixed tables of RFC 1951 §3.2.6, built once. */+function fixedTables(): { lit: Huffman; dist: Huffman } {+ const litLengths = new Uint8Array(288);+ litLengths.fill(8, 0, 144);+ litLengths.fill(9, 144, 256);+ litLengths.fill(7, 256, 280);+ litLengths.fill(8, 280, 288);+ const distLengths = new Uint8Array(30).fill(5);+ return { lit: buildHuffman(litLengths, 288), dist: buildHuffman(distLengths, 30) };+}+let FIXED: { lit: Huffman; dist: Huffman } | null = null;++/**+ * Decompress raw DEFLATE.+ *+ * @param src the deflated bytes+ * @param expectedSize the original length, which the caller knows: the output+ * is allocated once at the right size rather than grown,+ * and a stream that does not fill it exactly is corrupt+ * and says so here rather than at some later use.+ */+export function inflateRaw(src: Uint8Array, expectedSize: number): Uint8Array {+ const out = new Uint8Array(expectedSize);+ let outPos = 0;+ let pos = 0;+ let bitBuf = 0;+ let bitCnt = 0;++ const need = (n: number): number => {+ while (bitCnt < n) {+ if (pos >= src.length) throw new Error('inflate: ran out of input');+ bitBuf |= src[pos++] << bitCnt;+ bitCnt += 8;+ }+ const v = bitBuf & ((1 << n) - 1);+ bitBuf >>>= n;+ bitCnt -= n;+ return v;+ };++ // Walk code lengths shortest-first, comparing against the canonical first code+ // of each length — no table, so nothing to build per block.+ const decode = (h: Huffman): number => {+ let code = 0;+ let first = 0;+ let index = 0;+ for (let len = 1; len <= MAX_BITS; len++) {+ code |= need(1);+ const count = h.counts[len];+ if (code - first < count) return h.symbols[index + (code - first)];+ index += count;+ first = (first + count) << 1;+ code <<= 1;+ }+ throw new Error('inflate: bad Huffman code');+ };++ for (;;) {+ const last = need(1);+ const type = need(2);++ if (type === 0) {+ // Stored: skip to the byte boundary, then a length and its complement.+ bitBuf = 0;+ bitCnt = 0;+ if (pos + 4 > src.length) throw new Error('inflate: truncated stored block');+ const len = src[pos] | (src[pos + 1] << 8);+ const nlen = src[pos + 2] | (src[pos + 3] << 8);+ if ((len ^ 0xffff) !== nlen) throw new Error('inflate: stored length mismatch');+ pos += 4;+ if (pos + len > src.length) throw new Error('inflate: truncated stored block');+ if (outPos + len > expectedSize) throw new Error('inflate: output longer than expected');+ out.set(src.subarray(pos, pos + len), outPos);+ pos += len;+ outPos += len;+ } else if (type === 1 || type === 2) {+ let lit: Huffman;+ let dist: Huffman;+ if (type === 1) {+ FIXED ??= fixedTables();+ ({ lit, dist } = FIXED);+ } else {+ const nlen = need(5) + 257;+ const ndist = need(5) + 1;+ const ncode = need(4) + 4;+ const clen = new Uint8Array(19);+ for (let i = 0; i < ncode; i++) clen[CLEN_ORDER[i]] = need(3);+ const clenHuff = buildHuffman(clen, 19);+ // Literal/length and distance lengths are one run, coded together.+ const lengths = new Uint8Array(nlen + ndist);+ for (let i = 0; i < nlen + ndist;) {+ const sym = decode(clenHuff);+ if (sym < 16) {+ lengths[i++] = sym;+ } else if (sym === 16) {+ if (i === 0) throw new Error('inflate: repeat with no previous length');+ const prev = lengths[i - 1];+ for (let r = need(2) + 3; r > 0; r--) lengths[i++] = prev;+ } else if (sym === 17) {+ i += need(3) + 3;+ } else {+ i += need(7) + 11;+ }+ }+ if (lengths.length < nlen + ndist) throw new Error('inflate: code length overflow');+ lit = buildHuffman(lengths.subarray(0, nlen), nlen);+ dist = buildHuffman(lengths.subarray(nlen), ndist);+ }++ for (;;) {+ const sym = decode(lit);+ if (sym === 256) break;+ if (sym < 256) {+ if (outPos >= expectedSize) throw new Error('inflate: output longer than expected');+ out[outPos++] = sym;+ } else {+ const li = sym - 257;+ if (li >= LENGTH_BASE.length) throw new Error('inflate: bad length code');+ const len = LENGTH_BASE[li] + need(LENGTH_EXTRA[li]);+ const di = decode(dist);+ if (di >= DIST_BASE.length) throw new Error('inflate: bad distance code');+ const d = DIST_BASE[di] + need(DIST_EXTRA[di]);+ if (d > outPos) throw new Error('inflate: distance before start of output');+ if (outPos + len > expectedSize) throw new Error('inflate: output longer than expected');+ // Byte at a time on purpose: DEFLATE allows the match to overlap what+ // it is still writing (a run is coded as distance 1), so a block copy+ // would read bytes that do not exist yet.+ let from = outPos - d;+ for (let k = 0; k < len; k++) out[outPos++] = out[from++];+ }+ }+ } else {+ throw new Error('inflate: reserved block type');+ }++ if (last) break;+ }++ if (outPos !== expectedSize) {+ throw new Error(`inflate: expected ${expectedSize} bytes, produced ${outPos}`);+ }+ return out;+}diff --git a/pipeline/src/runtime/playableHost.ts b/pipeline/src/runtime/playableHost.tsindex 8591b36be..1dc2274ae 100644--- a/pipeline/src/runtime/playableHost.ts+++ b/pipeline/src/runtime/playableHost.ts@@ -5,8 +5,10 @@* esbuilt to an IIFE by exportPlayable with esengine + the project scripts* INLINED. The engine is the shipped WEB glue, handed over as the inline* `__ENGINE_GLUE__` global and evaluated as a blob module (it is ESM), with- * the wasm arriving base64 in `__ENGINE_WASM__`; assets + scenes are inlined- * globals too. Boots the SAME shipping runtime via initPlayableRuntime+ * the wasm arriving in `__ENGINE_WASM__`; assets + scenes are inlined+ * globals too. Both engine payloads are deflate+base64 — see PackedBytes in+ * exportPlayable, and inflate.ts for why the decoder is ours rather than+ * the platform's. Boots the SAME shipping runtime via initPlayableRuntime* (play == ship). Nothing is fetched — the whole game is one self-contained* .html (ad-network ready).*/@@ -14,19 +16,22 @@ import {createWebApp, setEditorMode, setPlayMode, initPlayableRuntime, createEmbeddedSideModuleHost,packagedAppOptions, packagedRuntimeInit,} from 'esengine';-import type { SceneData, EmbeddedSideModuleRegistry } from 'esengine';+import type { SceneData, EmbeddedSideModuleRegistry, EmbeddedSideModuleEntry } from 'esengine';import type { ESEngineModule as EngineModule } from 'esengine/wasm';import type { PackagedRuntimeFields } from '../project/runtimeConfig';+import { inflateRaw } from './inflate';type EngineFactory = (opts?: Record<string, unknown>) => Promise<EngineModule>;// Inlined by exportPlayable as <script> globals (kept out of the bundle so the// large base64 blobs aren't re-parsed as code). The glue is the WEB esengine.js-// (ESM) text; the wasm is base64-encoded esengine.wasm. __SIDE_MODULES__ holds-// the base64 glue+wasm for exactly the optional modules (physics/spine) the scene-// uses — the exporter ran the runtime's gating scan to pick them.-declare const __ENGINE_GLUE__: string;-declare const __ENGINE_WASM__: string;-declare const __SIDE_MODULES__: EmbeddedSideModuleRegistry;+// (ESM) text; the wasm is esengine.wasm. __SIDE_MODULES__ holds glue+wasm for+// exactly the optional modules (physics/spine) the scene uses — the exporter ran+// the runtime's gating scan to pick them. Every one of them is `PackedBytes`:+// raw-deflated, then base64.+interface PackedBytes { z: string; n: number }+declare const __ENGINE_GLUE__: PackedBytes;+declare const __ENGINE_WASM__: PackedBytes;+declare const __SIDE_MODULES__: Record<string, { glue: PackedBytes; wasm: PackedBytes }>;declare const __GAME_ASSETS__: Record<string, string>;declare const __GAME_PATHMAP__: Record<string, string>;declare const __GAME_SCENES__: Array<{ name: string; data: SceneData }>;@@ -43,6 +48,40 @@ function decodeBase64(b64: string): Uint8Array {return bytes;}+/** One inlined payload, back to the bytes the exporter packed. */+function unpack(p: PackedBytes): Uint8Array {+ return inflateRaw(decodeBase64(p.z), p.n);+}++const unpackText = (p: PackedBytes): string => new TextDecoder().decode(unpack(p));++/**+ * The inlined side-module registry, decoded on first use.+ *+ * Lazily because the scan that chose these modules asks what the CONTENT could+ * reach, not what a given run does: a scene that can spawn a Spine skeleton+ * carries the runtime whether or not it spawns one. Inflating all of them at+ * boot would spend that cost on every play, and these are the big ones —+ * basis is 1MB before deflating.+ */+function unpackSideModules(): EmbeddedSideModuleRegistry {+ if (typeof __SIDE_MODULES__ === 'undefined') return {};+ const registry: Record<string, EmbeddedSideModuleEntry> = {};+ for (const [id, packed] of Object.entries(__SIDE_MODULES__)) {+ Object.defineProperty(registry, id, {+ enumerable: true,+ configurable: true,+ get(): EmbeddedSideModuleEntry {+ const entry = { glue: unpackText(packed.glue), wasm: unpack(packed.wasm) };+ // Overwrite the getter, so acquiring a module twice inflates it once.+ Object.defineProperty(registry, id, { value: entry, enumerable: true, configurable: true });+ return entry;+ },+ });+ }+ return registry as EmbeddedSideModuleRegistry;+}+async function boot(): Promise<void> {// An export built before this global simply has no settings to apply.const runtimeConfig: PackagedRuntimeFields =@@ -56,12 +95,12 @@ async function boot(): Promise<void> {window.addEventListener('resize', resize);resize();- if (!__ENGINE_GLUE__ || !__ENGINE_WASM__) throw new Error('Engine runtime not inlined — re-export the playable.');+ if (!__ENGINE_GLUE__?.n || !__ENGINE_WASM__?.n) throw new Error('Engine runtime not inlined — re-export the playable.');// The esengine.js glue is ESM, so run it via a blob module (own scope); feed the// embedded wasm through instantiateWasm so nothing is fetched (single-file).- const blobUrl = URL.createObjectURL(new Blob([__ENGINE_GLUE__], { type: 'text/javascript' }));+ const blobUrl = URL.createObjectURL(new Blob([unpackText(__ENGINE_GLUE__)], { type: 'text/javascript' }));const { default: createEngine } = (await import(/* @vite-ignore */ blobUrl)) as { default: EngineFactory };- const wasmBytes = decodeBase64(__ENGINE_WASM__);+ const wasmBytes = unpack(__ENGINE_WASM__);const module = await createEngine({canvas,instantiateWasm(imports: WebAssembly.Imports, cb: (inst: WebAssembly.Instance, mod?: WebAssembly.Module) => void) {@@ -95,8 +134,12 @@ async function boot(): Promise<void> {// Everything the config says an App must be BUILT with (see packagedAppOptions)....packagedAppOptions(runtimeConfig),getViewportSize: () => ({ width: canvas.width, height: canvas.height }),- // Physics + spine resolve from the inlined base64 registry (no fetch).- sideModules: createEmbeddedSideModuleHost(typeof __SIDE_MODULES__ !== 'undefined' ? __SIDE_MODULES__ : {}),+ // Physics + spine resolve from the inlined registry (no fetch). Unpacked+ // here rather than inside the SDK: how these bytes survived a trip inside an+ // HTML file is this page's business, and the SDK takes them decoded.+ // Lazily, because a module the scene never asks for should not be paid for+ // at boot — spine and basis are ~1MB each before deflating.+ sideModules: createEmbeddedSideModuleHost(unpackSideModules()),});setEditorMode(false);setPlayMode(true);diff --git a/pipeline/tests/export-playable.test.ts b/pipeline/tests/export-playable.test.tsindex ebd8f98fc..d7fe61c14 100644--- a/pipeline/tests/export-playable.test.ts+++ b/pipeline/tests/export-playable.test.ts@@ -15,6 +15,33 @@ import path from 'node:path';import { fileURLToPath } from 'node:url';import { execFileSync } from 'node:child_process';import { exportGame } from '../src/export/exportGame';+import { inflateRaw } from '../src/runtime/inflate';++/**+ * Read an inlined payload back out of the page, the way the host does.+ *+ * The engine and every side module travel deflated-then-base64 (see PackedBytes+ * in exportPlayable), so "the bytes are in the file" cannot be asked by looking+ * for their base64 — that only ever held while the transport was plain. Asking+ * it through the decoder is the stronger question anyway: it says the bytes+ * arrived AND that what shipped can be turned back into them.+ */+function inlinedText(html: string, global: string, pick?: (v: never) => { z: string; n: number }): string {+ const at = html.indexOf(`window.${global}=`);+ if (at < 0) throw new Error(`${global} is not in the page`);+ const json = html.slice(at + `window.${global}=`.length);+ // The globals are `window.X=<json>;` concatenated, so the value ends at the+ // `;` that leaves the JSON balanced — JSON.parse of the prefix finds it.+ for (let end = json.indexOf(';'); end > 0; end = json.indexOf(';', end + 1)) {+ try {+ const value = JSON.parse(json.slice(0, end));+ const packed = pick ? pick(value) : value;+ const raw = Uint8Array.from(atob(packed.z), (c) => c.charCodeAt(0));+ return new TextDecoder().decode(inflateRaw(raw, packed.n));+ } catch { /* that `;` was inside the JSON — try the next one */ }+ }+ throw new Error(`${global} did not parse as a packed payload`);+}const HERE = path.dirname(fileURLToPath(import.meta.url));const PLAYABLE_HOST = path.join(HERE, '..', '..', 'pipeline', 'src', 'runtime', 'playableHost.ts');@@ -88,7 +115,7 @@ describe('exportGame (playable)', () => {expect(existsSync(path.join(out, 'assets'))).toBe(false);const html = readFileSync(path.join(out, 'index.html'), 'utf8');- expect(html).toContain('WEB_GLUE'); // web glue text inlined+ expect(inlinedText(html, '__ENGINE_GLUE__')).toContain('WEB_GLUE'); // web glue text inlinedexpect(html).toContain('__ENGINE_GLUE__'); // glue + wasm inlined as globalsexpect(html).toContain('__ENGINE_WASM__');expect(html).toContain('createObjectURL'); // real host bundled (blob loader)@@ -328,7 +355,7 @@ export function packagedRuntimeInit(c){return c;}\n`);expect(res.ok).toBe(true);const html = readFileSync(path.join(o, 'index.html'), 'utf8');expect(html).toContain('"physics"');- expect(html).toContain(Buffer.from('PHYSWASM').toString('base64'));+ expect(inlinedText(html, '__SIDE_MODULES__', (v) => v['physics'].wasm)).toBe('PHYSWASM');// ...and the runtime is told to install it, plus the world it declared.expect(html).toMatch(/"physicsEnabled":true/);expect(html).toMatch(/"y":-20/);@@ -345,7 +372,7 @@ export function packagedRuntimeInit(c){return c;}\n`);const html = readFileSync(path.join(o, 'index.html'), 'utf8');expect(html).toContain('__SIDE_MODULES__');expect(html).toContain('"physics"');- expect(html).toContain(Buffer.from('PHYSWASM').toString('base64')); // physics.wasm inlined+ expect(inlinedText(html, '__SIDE_MODULES__', (v) => v['physics'].wasm)).toBe('PHYSWASM');} finally {rmSync(r, { recursive: true, force: true });}@@ -421,7 +448,7 @@ export function packagedRuntimeInit(c){return c;}\n`);const html = readFileSync(path.join(o, 'index.html'), 'utf8');expect(html).toContain('__SIDE_MODULES__');expect(html).toContain('"spine:4.2"');- expect(html).toContain(Buffer.from('SPINE42WASM').toString('base64')); // spine42.wasm inlined+ expect(inlinedText(html, '__SIDE_MODULES__', (v) => v['spine:4.2'].wasm)).toBe('SPINE42WASM');// The atlas page image is embedded + path-mapped (the dep the cook now follows).expect(html).toMatch(/"assets\/spine\/hero\.png":"@uuid:/);} finally {diff --git a/pipeline/tests/inflate.test.ts b/pipeline/tests/inflate.test.tsnew file mode 100644index 000000000..7589b46af--- /dev/null+++ b/pipeline/tests/inflate.test.ts@@ -0,0 +1,142 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file The playable's DEFLATE decoder, against the encoder that feeds it.+ *+ * Every byte of a playable's engine goes through `inflateRaw`, so a bug+ * in it is not a degraded playable, it is one that does not boot at all —+ * on every device, since there is deliberately no second path. The stakes+ * are why this holds it against `zlib` over shapes chosen to reach each+ * branch of the format rather than a few happy cases:+ *+ * - every compression level, 0 through 9 (0 is all stored blocks, and+ * the fixed-Huffman block only appears for tiny inputs);+ * - a run long enough to be coded as an overlapping match (distance 1),+ * which a block copy would get wrong;+ * - incompressible input, which deflate stores rather than shrinks;+ * - lengths either side of the 258-byte maximum match and the 64KB+ * stored-block boundary;+ * - the real payloads — the engine module and each side module — since+ * those are the bytes that actually ship.+ */+import { describe, it, expect } from 'vitest';+import { deflateRawSync } from 'node:zlib';+import { readFileSync, existsSync } from 'node:fs';+import path from 'node:path';+import { fileURLToPath } from 'node:url';+import { inflateRaw } from '../src/runtime/inflate';++const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');+const WASM_DIR = path.join(ROOT, 'build', 'wasm', 'web');++/**+ * Deflate with zlib, inflate with ours, and require the bytes back exactly.+ *+ * Compared by hand rather than with `toEqual` over the two arrays: these run to+ * hundreds of kilobytes, and building a structural diff of that takes long+ * enough to look like a hang — while saying far less than "byte 70123 differs".+ */+const roundTrip = (bytes: Uint8Array, level?: number): void => {+ const deflated = new Uint8Array(deflateRawSync(bytes, level != null ? { level } : {}));+ const got = inflateRaw(deflated, bytes.length);+ expect(got.length).toBe(bytes.length);+ let differsAt = -1;+ for (let i = 0; i < bytes.length; i++) {+ if (got[i] !== bytes[i]) { differsAt = i; break; }+ }+ expect(differsAt < 0 ? 'identical'+ : `byte ${differsAt} of ${bytes.length}: expected ${bytes[differsAt]}, got ${got[differsAt]}`,+ ).toBe('identical');+};++/** Deterministic pseudo-random, so a failure reproduces rather than haunts. */+function pseudoRandom(n: number): Uint8Array {+ const out = new Uint8Array(n);+ let seed = 12345;+ for (let i = 0; i < n; i++) {+ seed = (seed * 1103515245 + 12345) & 0x7fffffff;+ out[i] = seed & 255;+ }+ return out;+}++const patterned = (n: number): Uint8Array => {+ const out = new Uint8Array(n);+ for (let i = 0; i < n; i++) out[i] = (i * 7 + 3) & 255;+ return out;+};++describe('inflateRaw — against zlib', () => {+ it('round-trips the empty input and a single byte', () => {+ roundTrip(new Uint8Array(0));+ roundTrip(new Uint8Array([42]));+ });++ it('round-trips at every compression level', () => {+ const source = readFileSync(path.join(ROOT, 'pipeline', 'src', 'runtime', 'inflate.ts'));+ for (let level = 0; level <= 9; level++) roundTrip(new Uint8Array(source), level);+ });++ it('round-trips a run coded as an overlapping match', () => {+ // 70,000 identical bytes deflate to a match whose distance is 1 — it reads+ // bytes it is in the middle of writing, so copying block-wise gets it wrong.+ roundTrip(new Uint8Array(70_000).fill(0x61));+ roundTrip(new Uint8Array(100_000)); // all zeroes+ const abc = new Uint8Array(90_000);+ for (let i = 0; i < abc.length; i++) abc[i] = 0x61 + (i % 3);+ roundTrip(abc); // short-period repeat+ });++ it('round-trips incompressible input, which deflate stores', () => {+ for (const n of [100, 70_000, 200_000]) roundTrip(pseudoRandom(n));+ });++ it('round-trips either side of the format boundaries', () => {+ // 258 is the longest match a length code can name; 65535 the largest stored+ // block. Off-by-one at either shows up here and nowhere else.+ for (const n of [1, 2, 3, 255, 256, 257, 258, 259, 65_535, 65_536, 65_537]) {+ roundTrip(patterned(n));+ }+ });++ it('refuses a stream that does not produce what it promised', () => {+ const bytes = patterned(1000);+ const deflated = new Uint8Array(deflateRawSync(bytes));+ // A wrong length is how a mismatched export/host pair would show up, and it+ // has to fail here rather than hand back a half-filled buffer.+ expect(() => inflateRaw(deflated, 999)).toThrow(/longer than expected|expected 999/);+ expect(() => inflateRaw(deflated, 1001)).toThrow(/expected 1001/);+ expect(() => inflateRaw(deflated.subarray(0, deflated.length - 3), 1000)).toThrow();+ });++ it('refuses garbage rather than looping or returning it', () => {+ expect(() => inflateRaw(pseudoRandom(64), 4096)).toThrow();+ });+});++// The modules a playable actually inlines. Skipped rather than failed when the+// wasm tree is absent: this suite is about the decoder, and a checkout that has+// not built the engine should still run it.+const engine = path.join(WASM_DIR, 'esengine.wasm');+describe.skipIf(!existsSync(engine))('inflateRaw — the payloads that ship', () => {+ for (const file of ['esengine.wasm', 'physics.wasm', 'physics3d.wasm', 'basis.wasm', 'spine42.wasm']) {+ it(`round-trips ${file}`, () => {+ const p = path.join(WASM_DIR, file);+ if (!existsSync(p)) return;+ roundTrip(new Uint8Array(readFileSync(p)), 9);+ });+ }++ it('decodes the engine module fast enough to boot behind', () => {+ const bytes = new Uint8Array(readFileSync(engine));+ const deflated = new Uint8Array(deflateRawSync(bytes, { level: 9 }));+ const started = performance.now();+ inflateRaw(deflated, bytes.length);+ const ms = performance.now() - started;+ // Measured ~16ms for 1.5MB on a development machine (~95 MB/s). The ceiling+ // is deliberately an order of magnitude above that: what it guards against+ // is a rewrite that drops to something unusable on a phone, not the spread+ // between one machine and another — which is all a tight bound would time.+ expect(ms).toBeLessThan(500);+ });+});diff --git a/sdk/etc/index.api.md b/sdk/etc/index.api.mdindex 1a73662fc..e439d0b91 100644--- a/sdk/etc/index.api.md+++ b/sdk/etc/index.api.md@@ -2891,8 +2891,8 @@ step: number## EmbeddedSideModuleEntry — interface @experimental```-glueBase64: string-wasmBase64: string+glue: string+wasm: Uint8Array<ArrayBufferLike>```## EmbeddedSideModuleRegistry — type @experimentaldiff --git a/sdk/etc/index.native.api.md b/sdk/etc/index.native.api.mdindex a7a52c696..2a4b292ba 100644--- a/sdk/etc/index.native.api.md+++ b/sdk/etc/index.native.api.md@@ -2899,8 +2899,8 @@ step: number## EmbeddedSideModuleEntry — interface @experimental```-glueBase64: string-wasmBase64: string+glue: string+wasm: Uint8Array<ArrayBufferLike>```## EmbeddedSideModuleRegistry — type @experimentaldiff --git a/sdk/etc/index.node.api.md b/sdk/etc/index.node.api.mdindex 04dccb621..04b80ad34 100644--- a/sdk/etc/index.node.api.md+++ b/sdk/etc/index.node.api.md@@ -2886,8 +2886,8 @@ step: number## EmbeddedSideModuleEntry — interface @experimental```-glueBase64: string-wasmBase64: string+glue: string+wasm: Uint8Array<ArrayBufferLike>```## EmbeddedSideModuleRegistry — type @experimentaldiff --git a/sdk/etc/minigame.api.md b/sdk/etc/minigame.api.mdindex e54d578b7..a4d9a517e 100644--- a/sdk/etc/minigame.api.md+++ b/sdk/etc/minigame.api.md@@ -2886,8 +2886,8 @@ step: number## EmbeddedSideModuleEntry — interface @experimental```-glueBase64: string-wasmBase64: string+glue: string+wasm: Uint8Array<ArrayBufferLike>```## EmbeddedSideModuleRegistry — type @experimentaldiff --git a/sdk/etc/wechat.api.md b/sdk/etc/wechat.api.mdindex 1a2e25628..d1dc89eed 100644--- a/sdk/etc/wechat.api.md+++ b/sdk/etc/wechat.api.md@@ -2886,8 +2886,8 @@ step: number## EmbeddedSideModuleEntry — interface @experimental```-glueBase64: string-wasmBase64: string+glue: string+wasm: Uint8Array<ArrayBufferLike>```## EmbeddedSideModuleRegistry — type @experimentaldiff --git a/sdk/src/sideModules/embeddedHost.ts b/sdk/src/sideModules/embeddedHost.tsindex c1c75c9cc..28667fab4 100644--- a/sdk/src/sideModules/embeddedHost.ts+++ b/sdk/src/sideModules/embeddedHost.ts@@ -3,42 +3,36 @@/*** @file embeddedHost.ts* @brief The single-file (playable-ad) transport: every needed side module's- * glue + wasm is inlined as base64 by the exporter, so nothing is- * fetched (ad networks require a self-contained .html). The exporter- * decides *which* modules to embed from a content scan of the scene, so- * an embedded registry that lacks a module the scene needs is an export- * bug, not a silent runtime degrade.+ * glue + wasm is inlined by the exporter, so nothing is fetched (ad+ * networks require a self-contained .html). The exporter decides *which*+ * modules to embed from a content scan of the scene, so an embedded+ * registry that lacks a module the scene needs is an export bug, not a+ * silent runtime degrade.+ *+ * How those bytes were ENCODED to survive the trip inside an HTML file+ * is the host page's business, not this module's: they arrive already+ * decoded. They used to arrive base64, which put one page's transport+ * decision in the SDK and meant a change to it (they are deflated now,+ * which is worth ~1MB of an ad network's 2MB cap) could not be made+ * without changing this file too.*/import { createSideModuleHost, instantiateFromGlueText, type SideModuleHost } from './host';import type { SideModuleId } from './registry';export interface EmbeddedSideModuleEntry {- /** base64 of the emscripten `<file>.js` glue text. */- glueBase64: string;- /** base64 of the `<file>.wasm` binary. */- wasmBase64: string;+ /** The emscripten `<file>.js` glue, as source text. */+ glue: string;+ /** The `<file>.wasm` binary. */+ wasm: Uint8Array;}/** Inlined by the playable exporter as a window global, keyed by {@link SideModuleId}. */export type EmbeddedSideModuleRegistry = Partial<Record<SideModuleId, EmbeddedSideModuleEntry>>;-function decodeBase64ToText(b64: string): string {- return atob(b64);-}--function decodeBase64ToBytes(b64: string): Uint8Array {- const raw = atob(b64);- const bytes = new Uint8Array(raw.length);- for (let i = 0; i < raw.length; i++) bytes[i] = raw.charCodeAt(i);- return bytes;-}-export function createEmbeddedSideModuleHost(registry: EmbeddedSideModuleRegistry): SideModuleHost {return createSideModuleHost(async (descriptor, id) => {const entry = registry[id];if (!entry) throw new Error(`side module "${id}" (${descriptor.file}) not embedded in this playable`);- const glueText = decodeBase64ToText(entry.glueBase64);- const wasmBytes = decodeBase64ToBytes(entry.wasmBase64);- return instantiateFromGlueText(glueText, wasmBytes.buffer as ArrayBuffer, descriptor);+ return instantiateFromGlueText(entry.glue, entry.wasm.buffer as ArrayBuffer, descriptor);});}