Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(export): a playable's engine travels compressed

esengineef4e89441c1e
diff --git a/pipeline/src/export/exportPlayable.ts b/pipeline/src/export/exportPlayable.ts
index b851df0b3..2fbe5529f 100644
--- a/pipeline/src/export/exportPlayable.ts
+++ b/pipeline/src/export/exportPlayable.ts
@@ -22,6 +22,7 @@ import {
} from '../project/runtimeConfig';
import { writeFile, mkdir, readFile, stat, rm } from 'node:fs/promises';
import { existsSync } from 'node:fs';
+import { deflateRawSync } from 'node:zlib';
import path from 'node:path';
import { cookAssets } from '../assets/cookAssets';
import type { OnExportProgress } from './exportProgress';
@@ -72,6 +73,30 @@ const mimeOf = (p: string): string => MIME[path.extname(p).slice(1).toLowerCase(
/** Escape `</script` so inlined content can't close the host <script> early. */
const inlineSafe = (s: string): string => s.replace(/<\/script/gi, '<\\/script');
+/**
+ * A binary payload on its way into an HTML file: deflated, then base64.
+ *
+ * Base64 costs +33% over the bytes it carries, and the bytes are wasm — which
+ * deflates to under a third. Compressing before encoding is worth more than any
+ * denser encoding could be: the engine's own module goes 1.542MB → 2.056MB
+ * encoded raw, against 0.597MB encoded this way.
+ *
+ * `n` is the original length, so the decoder allocates once and a stream that
+ * does not fill it exactly is caught here rather than at some later use. Raw
+ * deflate: `inflateRaw` is the only reader, and a wrapper's checksum would only
+ * re-answer what instantiating the module answers immediately after.
+ */
+interface PackedBytes {
+ /** base64 of the raw-deflated bytes. */
+ z: string;
+ /** Length before deflating. */
+ n: number;
+}
+
+function pack(bytes: Uint8Array | Buffer): PackedBytes {
+ return { z: deflateRawSync(bytes, { level: 9 }).toString('base64'), n: bytes.length };
+}
+
/**
* The playable page. Deliberately WITHOUT the web export's rotate-to-fit overlay:
* inside an ad SDK the container's size is the SDK's business, so `@media
@@ -129,14 +154,14 @@ async function collectSideModules(
wasmDir: string,
errors: string[],
physicsEnabled: boolean,
-): Promise<Record<string, { glueBase64: string; wasmBase64: string }>> {
+): Promise<Record<string, { glue: PackedBytes; wasm: PackedBytes }>> {
const ids = await scanSideModuleIds({
root, includedPaths, cookEntries: manifestEntries, stagedDir: cookDir, physicsEnabled,
});
const { files, unknown } = sideModuleFiles(ids);
for (const id of unknown) errors.push(`internal: no artifact mapping for side module "${id}"`);
- const registry: Record<string, { glueBase64: string; wasmBase64: string }> = {};
+ const registry: Record<string, { glue: PackedBytes; wasm: PackedBytes }> = {};
for (const { id, file } of files) {
const gluePath = path.join(wasmDir, `${file}.js`);
const wasmPath = path.join(wasmDir, `${file}.wasm`);
@@ -145,8 +170,8 @@ async function collectSideModules(
continue;
}
registry[id] = {
- glueBase64: (await readFile(gluePath)).toString('base64'),
- wasmBase64: (await readFile(wasmPath)).toString('base64'),
+ glue: pack(await readFile(gluePath)),
+ wasm: pack(await readFile(wasmPath)),
};
}
return registry;
@@ -298,13 +323,13 @@ export async function exportPlayable(opts: {
// inlined — the host loads the glue via a blob module + feeds the wasm (base64)
// through instantiateWasm. No separate single-file build needed.
progress({ phase: 'Inlining engine' });
- let glue = '';
- let wasmB64 = '';
+ let glue: PackedBytes = { z: '', n: 0 };
+ let wasm: PackedBytes = { z: '', n: 0 };
const gluePath = path.join(opts.wasmDir, 'esengine.js');
const wasmPath = path.join(opts.wasmDir, 'esengine.wasm');
if (existsSync(gluePath) && existsSync(wasmPath)) {
- glue = await readFile(gluePath, 'utf8');
- wasmB64 = (await readFile(wasmPath)).toString('base64');
+ glue = pack(await readFile(gluePath));
+ wasm = pack(await readFile(wasmPath));
} else {
errors.push(`engine runtime not found in ${opts.wasmDir} (need esengine.js + esengine.wasm)`);
}
@@ -327,7 +352,7 @@ export async function exportPlayable(opts: {
};
const globals =
span('wasm/esengine.js', `window.__ENGINE_GLUE__=${JSON.stringify(glue)};`) +
- span('wasm/esengine.wasm', `window.__ENGINE_WASM__=${JSON.stringify(wasmB64)};`) +
+ span('wasm/esengine.wasm', `window.__ENGINE_WASM__=${JSON.stringify(wasm)};`) +
span('wasm/side-modules', `window.__SIDE_MODULES__=${JSON.stringify(sideModules)};`) +
`window.__GAME_ASSETS__=${JSON.stringify(assets)};` +
`window.__GAME_PATHMAP__=${JSON.stringify(pathMap)};` +
diff --git a/pipeline/src/runtime/inflate.ts b/pipeline/src/runtime/inflate.ts
new file mode 100644
index 000000000..5b7b0b5b6
--- /dev/null
+++ b/pipeline/src/runtime/inflate.ts
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file inflate.ts — raw DEFLATE, for the payloads a single-file build inlines.
+ *
+ * A playable ad is one HTML file under a hard byte cap, so its engine
+ * arrives as text: base64 costs +33% over the binary, and the binary
+ * itself is a 1.5MB wasm that deflates to 29% of that. Compressing before
+ * encoding is worth more than any denser encoding could be.
+ *
+ * The browser has `DecompressionStream`, and this does not use it. It
+ * landed in Safari 16.4, and a playable that white-screens on the older
+ * WebViews still in an ad network's traffic is worse than one that is
+ * larger — the impression is paid for either way. A fallback path would
+ * be a second decoder that only some devices ever run, which is the shape
+ * every "it works on the machine that tested it" bug comes in. So there
+ * is one path, it runs everywhere, and every device exercises the code
+ * the tests exercise.
+ *
+ * RAW deflate (no zlib or gzip wrapper): the producer is
+ * `deflateRawSync`, the size is known from the manifest, and the checksum
+ * a wrapper carries would only re-answer what `WebAssembly.instantiate`
+ * answers immediately after.
+ *
+ * Structure follows zlib's `puff.c` — the canonical-Huffman decode walks
+ * code lengths a bit at a time rather than building a lookup table.
+ * Measured on the engine's own wasm, that is fast enough to not be worth
+ * the bigger decoder (see inflate.test.ts, which holds the rate).
+ */
+
+/** Longest Huffman code DEFLATE permits. */
+const MAX_BITS = 15;
+
+/** RFC 1951 §3.2.5 — length codes 257..285. */
+const LENGTH_BASE = [
+ 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59,
+ 67, 83, 99, 115, 131, 163, 195, 227, 258,
+];
+const LENGTH_EXTRA = [
+ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
+ 4, 4, 4, 4, 5, 5, 5, 5, 0,
+];
+/** RFC 1951 §3.2.5 — distance codes 0..29. */
+const DIST_BASE = [
+ 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769,
+ 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577,
+];
+const DIST_EXTRA = [
+ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+ 9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
+];
+/** RFC 1951 §3.2.7 — the order code lengths are themselves written in. */
+const CLEN_ORDER = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15];
+
+/**
+ * A canonical Huffman table as counts-per-length plus symbols in canonical
+ * order — the two arrays the length-walking decode needs, and nothing else.
+ */
+interface Huffman {
+ counts: Int32Array;
+ symbols: Int32Array;
+}
+
+function buildHuffman(lengths: Uint8Array, n: number): Huffman {
+ const counts = new Int32Array(MAX_BITS + 1);
+ for (let i = 0; i < n; i++) counts[lengths[i]]++;
+ counts[0] = 0;
+ const offsets = new Int32Array(MAX_BITS + 1);
+ for (let len = 1; len <= MAX_BITS; len++) offsets[len] = offsets[len - 1] + counts[len - 1];
+ const symbols = new Int32Array(n);
+ for (let i = 0; i < n; i++) if (lengths[i]) symbols[offsets[lengths[i]]++] = i;
+ return { counts, symbols };
+}
+
+/** The fixed tables of RFC 1951 §3.2.6, built once. */
+function fixedTables(): { lit: Huffman; dist: Huffman } {
+ const litLengths = new Uint8Array(288);
+ litLengths.fill(8, 0, 144);
+ litLengths.fill(9, 144, 256);
+ litLengths.fill(7, 256, 280);
+ litLengths.fill(8, 280, 288);
+ const distLengths = new Uint8Array(30).fill(5);
+ return { lit: buildHuffman(litLengths, 288), dist: buildHuffman(distLengths, 30) };
+}
+let FIXED: { lit: Huffman; dist: Huffman } | null = null;
+
+/**
+ * Decompress raw DEFLATE.
+ *
+ * @param src the deflated bytes
+ * @param expectedSize the original length, which the caller knows: the output
+ * is allocated once at the right size rather than grown,
+ * and a stream that does not fill it exactly is corrupt
+ * and says so here rather than at some later use.
+ */
+export function inflateRaw(src: Uint8Array, expectedSize: number): Uint8Array {
+ const out = new Uint8Array(expectedSize);
+ let outPos = 0;
+ let pos = 0;
+ let bitBuf = 0;
+ let bitCnt = 0;
+
+ const need = (n: number): number => {
+ while (bitCnt < n) {
+ if (pos >= src.length) throw new Error('inflate: ran out of input');
+ bitBuf |= src[pos++] << bitCnt;
+ bitCnt += 8;
+ }
+ const v = bitBuf & ((1 << n) - 1);
+ bitBuf >>>= n;
+ bitCnt -= n;
+ return v;
+ };
+
+ // Walk code lengths shortest-first, comparing against the canonical first code
+ // of each length — no table, so nothing to build per block.
+ const decode = (h: Huffman): number => {
+ let code = 0;
+ let first = 0;
+ let index = 0;
+ for (let len = 1; len <= MAX_BITS; len++) {
+ code |= need(1);
+ const count = h.counts[len];
+ if (code - first < count) return h.symbols[index + (code - first)];
+ index += count;
+ first = (first + count) << 1;
+ code <<= 1;
+ }
+ throw new Error('inflate: bad Huffman code');
+ };
+
+ for (;;) {
+ const last = need(1);
+ const type = need(2);
+
+ if (type === 0) {
+ // Stored: skip to the byte boundary, then a length and its complement.
+ bitBuf = 0;
+ bitCnt = 0;
+ if (pos + 4 > src.length) throw new Error('inflate: truncated stored block');
+ const len = src[pos] | (src[pos + 1] << 8);
+ const nlen = src[pos + 2] | (src[pos + 3] << 8);
+ if ((len ^ 0xffff) !== nlen) throw new Error('inflate: stored length mismatch');
+ pos += 4;
+ if (pos + len > src.length) throw new Error('inflate: truncated stored block');
+ if (outPos + len > expectedSize) throw new Error('inflate: output longer than expected');
+ out.set(src.subarray(pos, pos + len), outPos);
+ pos += len;
+ outPos += len;
+ } else if (type === 1 || type === 2) {
+ let lit: Huffman;
+ let dist: Huffman;
+ if (type === 1) {
+ FIXED ??= fixedTables();
+ ({ lit, dist } = FIXED);
+ } else {
+ const nlen = need(5) + 257;
+ const ndist = need(5) + 1;
+ const ncode = need(4) + 4;
+ const clen = new Uint8Array(19);
+ for (let i = 0; i < ncode; i++) clen[CLEN_ORDER[i]] = need(3);
+ const clenHuff = buildHuffman(clen, 19);
+ // Literal/length and distance lengths are one run, coded together.
+ const lengths = new Uint8Array(nlen + ndist);
+ for (let i = 0; i < nlen + ndist;) {
+ const sym = decode(clenHuff);
+ if (sym < 16) {
+ lengths[i++] = sym;
+ } else if (sym === 16) {
+ if (i === 0) throw new Error('inflate: repeat with no previous length');
+ const prev = lengths[i - 1];
+ for (let r = need(2) + 3; r > 0; r--) lengths[i++] = prev;
+ } else if (sym === 17) {
+ i += need(3) + 3;
+ } else {
+ i += need(7) + 11;
+ }
+ }
+ if (lengths.length < nlen + ndist) throw new Error('inflate: code length overflow');
+ lit = buildHuffman(lengths.subarray(0, nlen), nlen);
+ dist = buildHuffman(lengths.subarray(nlen), ndist);
+ }
+
+ for (;;) {
+ const sym = decode(lit);
+ if (sym === 256) break;
+ if (sym < 256) {
+ if (outPos >= expectedSize) throw new Error('inflate: output longer than expected');
+ out[outPos++] = sym;
+ } else {
+ const li = sym - 257;
+ if (li >= LENGTH_BASE.length) throw new Error('inflate: bad length code');
+ const len = LENGTH_BASE[li] + need(LENGTH_EXTRA[li]);
+ const di = decode(dist);
+ if (di >= DIST_BASE.length) throw new Error('inflate: bad distance code');
+ const d = DIST_BASE[di] + need(DIST_EXTRA[di]);
+ if (d > outPos) throw new Error('inflate: distance before start of output');
+ if (outPos + len > expectedSize) throw new Error('inflate: output longer than expected');
+ // Byte at a time on purpose: DEFLATE allows the match to overlap what
+ // it is still writing (a run is coded as distance 1), so a block copy
+ // would read bytes that do not exist yet.
+ let from = outPos - d;
+ for (let k = 0; k < len; k++) out[outPos++] = out[from++];
+ }
+ }
+ } else {
+ throw new Error('inflate: reserved block type');
+ }
+
+ if (last) break;
+ }
+
+ if (outPos !== expectedSize) {
+ throw new Error(`inflate: expected ${expectedSize} bytes, produced ${outPos}`);
+ }
+ return out;
+}
diff --git a/pipeline/src/runtime/playableHost.ts b/pipeline/src/runtime/playableHost.ts
index 8591b36be..1dc2274ae 100644
--- a/pipeline/src/runtime/playableHost.ts
+++ b/pipeline/src/runtime/playableHost.ts
@@ -5,8 +5,10 @@
* esbuilt to an IIFE by exportPlayable with esengine + the project scripts
* INLINED. The engine is the shipped WEB glue, handed over as the inline
* `__ENGINE_GLUE__` global and evaluated as a blob module (it is ESM), with
- * the wasm arriving base64 in `__ENGINE_WASM__`; assets + scenes are inlined
- * globals too. Boots the SAME shipping runtime via initPlayableRuntime
+ * the wasm arriving in `__ENGINE_WASM__`; assets + scenes are inlined
+ * globals too. Both engine payloads are deflate+base64 — see PackedBytes in
+ * exportPlayable, and inflate.ts for why the decoder is ours rather than
+ * the platform's. Boots the SAME shipping runtime via initPlayableRuntime
* (play == ship). Nothing is fetched — the whole game is one self-contained
* .html (ad-network ready).
*/
@@ -14,19 +16,22 @@ import {
createWebApp, setEditorMode, setPlayMode, initPlayableRuntime, createEmbeddedSideModuleHost,
packagedAppOptions, packagedRuntimeInit,
} from 'esengine';
-import type { SceneData, EmbeddedSideModuleRegistry } from 'esengine';
+import type { SceneData, EmbeddedSideModuleRegistry, EmbeddedSideModuleEntry } from 'esengine';
import type { ESEngineModule as EngineModule } from 'esengine/wasm';
import type { PackagedRuntimeFields } from '../project/runtimeConfig';
+import { inflateRaw } from './inflate';
type EngineFactory = (opts?: Record<string, unknown>) => Promise<EngineModule>;
// Inlined by exportPlayable as <script> globals (kept out of the bundle so the
// large base64 blobs aren't re-parsed as code). The glue is the WEB esengine.js
-// (ESM) text; the wasm is base64-encoded esengine.wasm. __SIDE_MODULES__ holds
-// the base64 glue+wasm for exactly the optional modules (physics/spine) the scene
-// uses — the exporter ran the runtime's gating scan to pick them.
-declare const __ENGINE_GLUE__: string;
-declare const __ENGINE_WASM__: string;
-declare const __SIDE_MODULES__: EmbeddedSideModuleRegistry;
+// (ESM) text; the wasm is esengine.wasm. __SIDE_MODULES__ holds glue+wasm for
+// exactly the optional modules (physics/spine) the scene uses — the exporter ran
+// the runtime's gating scan to pick them. Every one of them is `PackedBytes`:
+// raw-deflated, then base64.
+interface PackedBytes { z: string; n: number }
+declare const __ENGINE_GLUE__: PackedBytes;
+declare const __ENGINE_WASM__: PackedBytes;
+declare const __SIDE_MODULES__: Record<string, { glue: PackedBytes; wasm: PackedBytes }>;
declare const __GAME_ASSETS__: Record<string, string>;
declare const __GAME_PATHMAP__: Record<string, string>;
declare const __GAME_SCENES__: Array<{ name: string; data: SceneData }>;
@@ -43,6 +48,40 @@ function decodeBase64(b64: string): Uint8Array {
return bytes;
}
+/** One inlined payload, back to the bytes the exporter packed. */
+function unpack(p: PackedBytes): Uint8Array {
+ return inflateRaw(decodeBase64(p.z), p.n);
+}
+
+const unpackText = (p: PackedBytes): string => new TextDecoder().decode(unpack(p));
+
+/**
+ * The inlined side-module registry, decoded on first use.
+ *
+ * Lazily because the scan that chose these modules asks what the CONTENT could
+ * reach, not what a given run does: a scene that can spawn a Spine skeleton
+ * carries the runtime whether or not it spawns one. Inflating all of them at
+ * boot would spend that cost on every play, and these are the big ones —
+ * basis is 1MB before deflating.
+ */
+function unpackSideModules(): EmbeddedSideModuleRegistry {
+ if (typeof __SIDE_MODULES__ === 'undefined') return {};
+ const registry: Record<string, EmbeddedSideModuleEntry> = {};
+ for (const [id, packed] of Object.entries(__SIDE_MODULES__)) {
+ Object.defineProperty(registry, id, {
+ enumerable: true,
+ configurable: true,
+ get(): EmbeddedSideModuleEntry {
+ const entry = { glue: unpackText(packed.glue), wasm: unpack(packed.wasm) };
+ // Overwrite the getter, so acquiring a module twice inflates it once.
+ Object.defineProperty(registry, id, { value: entry, enumerable: true, configurable: true });
+ return entry;
+ },
+ });
+ }
+ return registry as EmbeddedSideModuleRegistry;
+}
+
async function boot(): Promise<void> {
// An export built before this global simply has no settings to apply.
const runtimeConfig: PackagedRuntimeFields =
@@ -56,12 +95,12 @@ async function boot(): Promise<void> {
window.addEventListener('resize', resize);
resize();
- if (!__ENGINE_GLUE__ || !__ENGINE_WASM__) throw new Error('Engine runtime not inlined — re-export the playable.');
+ if (!__ENGINE_GLUE__?.n || !__ENGINE_WASM__?.n) throw new Error('Engine runtime not inlined — re-export the playable.');
// The esengine.js glue is ESM, so run it via a blob module (own scope); feed the
// embedded wasm through instantiateWasm so nothing is fetched (single-file).
- const blobUrl = URL.createObjectURL(new Blob([__ENGINE_GLUE__], { type: 'text/javascript' }));
+ const blobUrl = URL.createObjectURL(new Blob([unpackText(__ENGINE_GLUE__)], { type: 'text/javascript' }));
const { default: createEngine } = (await import(/* @vite-ignore */ blobUrl)) as { default: EngineFactory };
- const wasmBytes = decodeBase64(__ENGINE_WASM__);
+ const wasmBytes = unpack(__ENGINE_WASM__);
const module = await createEngine({
canvas,
instantiateWasm(imports: WebAssembly.Imports, cb: (inst: WebAssembly.Instance, mod?: WebAssembly.Module) => void) {
@@ -95,8 +134,12 @@ async function boot(): Promise<void> {
// Everything the config says an App must be BUILT with (see packagedAppOptions).
...packagedAppOptions(runtimeConfig),
getViewportSize: () => ({ width: canvas.width, height: canvas.height }),
- // Physics + spine resolve from the inlined base64 registry (no fetch).
- sideModules: createEmbeddedSideModuleHost(typeof __SIDE_MODULES__ !== 'undefined' ? __SIDE_MODULES__ : {}),
+ // Physics + spine resolve from the inlined registry (no fetch). Unpacked
+ // here rather than inside the SDK: how these bytes survived a trip inside an
+ // HTML file is this page's business, and the SDK takes them decoded.
+ // Lazily, because a module the scene never asks for should not be paid for
+ // at boot — spine and basis are ~1MB each before deflating.
+ sideModules: createEmbeddedSideModuleHost(unpackSideModules()),
});
setEditorMode(false);
setPlayMode(true);
diff --git a/pipeline/tests/export-playable.test.ts b/pipeline/tests/export-playable.test.ts
index ebd8f98fc..d7fe61c14 100644
--- a/pipeline/tests/export-playable.test.ts
+++ b/pipeline/tests/export-playable.test.ts
@@ -15,6 +15,33 @@ import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { execFileSync } from 'node:child_process';
import { exportGame } from '../src/export/exportGame';
+import { inflateRaw } from '../src/runtime/inflate';
+
+/**
+ * Read an inlined payload back out of the page, the way the host does.
+ *
+ * The engine and every side module travel deflated-then-base64 (see PackedBytes
+ * in exportPlayable), so "the bytes are in the file" cannot be asked by looking
+ * for their base64 — that only ever held while the transport was plain. Asking
+ * it through the decoder is the stronger question anyway: it says the bytes
+ * arrived AND that what shipped can be turned back into them.
+ */
+function inlinedText(html: string, global: string, pick?: (v: never) => { z: string; n: number }): string {
+ const at = html.indexOf(`window.${global}=`);
+ if (at < 0) throw new Error(`${global} is not in the page`);
+ const json = html.slice(at + `window.${global}=`.length);
+ // The globals are `window.X=<json>;` concatenated, so the value ends at the
+ // `;` that leaves the JSON balanced — JSON.parse of the prefix finds it.
+ for (let end = json.indexOf(';'); end > 0; end = json.indexOf(';', end + 1)) {
+ try {
+ const value = JSON.parse(json.slice(0, end));
+ const packed = pick ? pick(value) : value;
+ const raw = Uint8Array.from(atob(packed.z), (c) => c.charCodeAt(0));
+ return new TextDecoder().decode(inflateRaw(raw, packed.n));
+ } catch { /* that `;` was inside the JSON — try the next one */ }
+ }
+ throw new Error(`${global} did not parse as a packed payload`);
+}
const HERE = path.dirname(fileURLToPath(import.meta.url));
const PLAYABLE_HOST = path.join(HERE, '..', '..', 'pipeline', 'src', 'runtime', 'playableHost.ts');
@@ -88,7 +115,7 @@ describe('exportGame (playable)', () => {
expect(existsSync(path.join(out, 'assets'))).toBe(false);
const html = readFileSync(path.join(out, 'index.html'), 'utf8');
- expect(html).toContain('WEB_GLUE'); // web glue text inlined
+ expect(inlinedText(html, '__ENGINE_GLUE__')).toContain('WEB_GLUE'); // web glue text inlined
expect(html).toContain('__ENGINE_GLUE__'); // glue + wasm inlined as globals
expect(html).toContain('__ENGINE_WASM__');
expect(html).toContain('createObjectURL'); // real host bundled (blob loader)
@@ -328,7 +355,7 @@ export function packagedRuntimeInit(c){return c;}\n`);
expect(res.ok).toBe(true);
const html = readFileSync(path.join(o, 'index.html'), 'utf8');
expect(html).toContain('"physics"');
- expect(html).toContain(Buffer.from('PHYSWASM').toString('base64'));
+ expect(inlinedText(html, '__SIDE_MODULES__', (v) => v['physics'].wasm)).toBe('PHYSWASM');
// ...and the runtime is told to install it, plus the world it declared.
expect(html).toMatch(/"physicsEnabled":true/);
expect(html).toMatch(/"y":-20/);
@@ -345,7 +372,7 @@ export function packagedRuntimeInit(c){return c;}\n`);
const html = readFileSync(path.join(o, 'index.html'), 'utf8');
expect(html).toContain('__SIDE_MODULES__');
expect(html).toContain('"physics"');
- expect(html).toContain(Buffer.from('PHYSWASM').toString('base64')); // physics.wasm inlined
+ expect(inlinedText(html, '__SIDE_MODULES__', (v) => v['physics'].wasm)).toBe('PHYSWASM');
} finally {
rmSync(r, { recursive: true, force: true });
}
@@ -421,7 +448,7 @@ export function packagedRuntimeInit(c){return c;}\n`);
const html = readFileSync(path.join(o, 'index.html'), 'utf8');
expect(html).toContain('__SIDE_MODULES__');
expect(html).toContain('"spine:4.2"');
- expect(html).toContain(Buffer.from('SPINE42WASM').toString('base64')); // spine42.wasm inlined
+ expect(inlinedText(html, '__SIDE_MODULES__', (v) => v['spine:4.2'].wasm)).toBe('SPINE42WASM');
// The atlas page image is embedded + path-mapped (the dep the cook now follows).
expect(html).toMatch(/"assets\/spine\/hero\.png":"@uuid:/);
} finally {
diff --git a/pipeline/tests/inflate.test.ts b/pipeline/tests/inflate.test.ts
new file mode 100644
index 000000000..7589b46af
--- /dev/null
+++ b/pipeline/tests/inflate.test.ts
@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file The playable's DEFLATE decoder, against the encoder that feeds it.
+ *
+ * Every byte of a playable's engine goes through `inflateRaw`, so a bug
+ * in it is not a degraded playable, it is one that does not boot at all —
+ * on every device, since there is deliberately no second path. The stakes
+ * are why this holds it against `zlib` over shapes chosen to reach each
+ * branch of the format rather than a few happy cases:
+ *
+ * - every compression level, 0 through 9 (0 is all stored blocks, and
+ * the fixed-Huffman block only appears for tiny inputs);
+ * - a run long enough to be coded as an overlapping match (distance 1),
+ * which a block copy would get wrong;
+ * - incompressible input, which deflate stores rather than shrinks;
+ * - lengths either side of the 258-byte maximum match and the 64KB
+ * stored-block boundary;
+ * - the real payloads — the engine module and each side module — since
+ * those are the bytes that actually ship.
+ */
+import { describe, it, expect } from 'vitest';
+import { deflateRawSync } from 'node:zlib';
+import { readFileSync, existsSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { inflateRaw } from '../src/runtime/inflate';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
+const WASM_DIR = path.join(ROOT, 'build', 'wasm', 'web');
+
+/**
+ * Deflate with zlib, inflate with ours, and require the bytes back exactly.
+ *
+ * Compared by hand rather than with `toEqual` over the two arrays: these run to
+ * hundreds of kilobytes, and building a structural diff of that takes long
+ * enough to look like a hang — while saying far less than "byte 70123 differs".
+ */
+const roundTrip = (bytes: Uint8Array, level?: number): void => {
+ const deflated = new Uint8Array(deflateRawSync(bytes, level != null ? { level } : {}));
+ const got = inflateRaw(deflated, bytes.length);
+ expect(got.length).toBe(bytes.length);
+ let differsAt = -1;
+ for (let i = 0; i < bytes.length; i++) {
+ if (got[i] !== bytes[i]) { differsAt = i; break; }
+ }
+ expect(differsAt < 0 ? 'identical'
+ : `byte ${differsAt} of ${bytes.length}: expected ${bytes[differsAt]}, got ${got[differsAt]}`,
+ ).toBe('identical');
+};
+
+/** Deterministic pseudo-random, so a failure reproduces rather than haunts. */
+function pseudoRandom(n: number): Uint8Array {
+ const out = new Uint8Array(n);
+ let seed = 12345;
+ for (let i = 0; i < n; i++) {
+ seed = (seed * 1103515245 + 12345) & 0x7fffffff;
+ out[i] = seed & 255;
+ }
+ return out;
+}
+
+const patterned = (n: number): Uint8Array => {
+ const out = new Uint8Array(n);
+ for (let i = 0; i < n; i++) out[i] = (i * 7 + 3) & 255;
+ return out;
+};
+
+describe('inflateRaw — against zlib', () => {
+ it('round-trips the empty input and a single byte', () => {
+ roundTrip(new Uint8Array(0));
+ roundTrip(new Uint8Array([42]));
+ });
+
+ it('round-trips at every compression level', () => {
+ const source = readFileSync(path.join(ROOT, 'pipeline', 'src', 'runtime', 'inflate.ts'));
+ for (let level = 0; level <= 9; level++) roundTrip(new Uint8Array(source), level);
+ });
+
+ it('round-trips a run coded as an overlapping match', () => {
+ // 70,000 identical bytes deflate to a match whose distance is 1 — it reads
+ // bytes it is in the middle of writing, so copying block-wise gets it wrong.
+ roundTrip(new Uint8Array(70_000).fill(0x61));
+ roundTrip(new Uint8Array(100_000)); // all zeroes
+ const abc = new Uint8Array(90_000);
+ for (let i = 0; i < abc.length; i++) abc[i] = 0x61 + (i % 3);
+ roundTrip(abc); // short-period repeat
+ });
+
+ it('round-trips incompressible input, which deflate stores', () => {
+ for (const n of [100, 70_000, 200_000]) roundTrip(pseudoRandom(n));
+ });
+
+ it('round-trips either side of the format boundaries', () => {
+ // 258 is the longest match a length code can name; 65535 the largest stored
+ // block. Off-by-one at either shows up here and nowhere else.
+ for (const n of [1, 2, 3, 255, 256, 257, 258, 259, 65_535, 65_536, 65_537]) {
+ roundTrip(patterned(n));
+ }
+ });
+
+ it('refuses a stream that does not produce what it promised', () => {
+ const bytes = patterned(1000);
+ const deflated = new Uint8Array(deflateRawSync(bytes));
+ // A wrong length is how a mismatched export/host pair would show up, and it
+ // has to fail here rather than hand back a half-filled buffer.
+ expect(() => inflateRaw(deflated, 999)).toThrow(/longer than expected|expected 999/);
+ expect(() => inflateRaw(deflated, 1001)).toThrow(/expected 1001/);
+ expect(() => inflateRaw(deflated.subarray(0, deflated.length - 3), 1000)).toThrow();
+ });
+
+ it('refuses garbage rather than looping or returning it', () => {
+ expect(() => inflateRaw(pseudoRandom(64), 4096)).toThrow();
+ });
+});
+
+// The modules a playable actually inlines. Skipped rather than failed when the
+// wasm tree is absent: this suite is about the decoder, and a checkout that has
+// not built the engine should still run it.
+const engine = path.join(WASM_DIR, 'esengine.wasm');
+describe.skipIf(!existsSync(engine))('inflateRaw — the payloads that ship', () => {
+ for (const file of ['esengine.wasm', 'physics.wasm', 'physics3d.wasm', 'basis.wasm', 'spine42.wasm']) {
+ it(`round-trips ${file}`, () => {
+ const p = path.join(WASM_DIR, file);
+ if (!existsSync(p)) return;
+ roundTrip(new Uint8Array(readFileSync(p)), 9);
+ });
+ }
+
+ it('decodes the engine module fast enough to boot behind', () => {
+ const bytes = new Uint8Array(readFileSync(engine));
+ const deflated = new Uint8Array(deflateRawSync(bytes, { level: 9 }));
+ const started = performance.now();
+ inflateRaw(deflated, bytes.length);
+ const ms = performance.now() - started;
+ // Measured ~16ms for 1.5MB on a development machine (~95 MB/s). The ceiling
+ // is deliberately an order of magnitude above that: what it guards against
+ // is a rewrite that drops to something unusable on a phone, not the spread
+ // between one machine and another — which is all a tight bound would time.
+ expect(ms).toBeLessThan(500);
+ });
+});
diff --git a/sdk/etc/index.api.md b/sdk/etc/index.api.md
index 1a73662fc..e439d0b91 100644
--- a/sdk/etc/index.api.md
+++ b/sdk/etc/index.api.md
@@ -2891,8 +2891,8 @@ step: number
## EmbeddedSideModuleEntry — interface @experimental
```
-glueBase64: string
-wasmBase64: string
+glue: string
+wasm: Uint8Array<ArrayBufferLike>
```
## EmbeddedSideModuleRegistry — type @experimental
diff --git a/sdk/etc/index.native.api.md b/sdk/etc/index.native.api.md
index a7a52c696..2a4b292ba 100644
--- a/sdk/etc/index.native.api.md
+++ b/sdk/etc/index.native.api.md
@@ -2899,8 +2899,8 @@ step: number
## EmbeddedSideModuleEntry — interface @experimental
```
-glueBase64: string
-wasmBase64: string
+glue: string
+wasm: Uint8Array<ArrayBufferLike>
```
## EmbeddedSideModuleRegistry — type @experimental
diff --git a/sdk/etc/index.node.api.md b/sdk/etc/index.node.api.md
index 04dccb621..04b80ad34 100644
--- a/sdk/etc/index.node.api.md
+++ b/sdk/etc/index.node.api.md
@@ -2886,8 +2886,8 @@ step: number
## EmbeddedSideModuleEntry — interface @experimental
```
-glueBase64: string
-wasmBase64: string
+glue: string
+wasm: Uint8Array<ArrayBufferLike>
```
## EmbeddedSideModuleRegistry — type @experimental
diff --git a/sdk/etc/minigame.api.md b/sdk/etc/minigame.api.md
index e54d578b7..a4d9a517e 100644
--- a/sdk/etc/minigame.api.md
+++ b/sdk/etc/minigame.api.md
@@ -2886,8 +2886,8 @@ step: number
## EmbeddedSideModuleEntry — interface @experimental
```
-glueBase64: string
-wasmBase64: string
+glue: string
+wasm: Uint8Array<ArrayBufferLike>
```
## EmbeddedSideModuleRegistry — type @experimental
diff --git a/sdk/etc/wechat.api.md b/sdk/etc/wechat.api.md
index 1a2e25628..d1dc89eed 100644
--- a/sdk/etc/wechat.api.md
+++ b/sdk/etc/wechat.api.md
@@ -2886,8 +2886,8 @@ step: number
## EmbeddedSideModuleEntry — interface @experimental
```
-glueBase64: string
-wasmBase64: string
+glue: string
+wasm: Uint8Array<ArrayBufferLike>
```
## EmbeddedSideModuleRegistry — type @experimental
diff --git a/sdk/src/sideModules/embeddedHost.ts b/sdk/src/sideModules/embeddedHost.ts
index c1c75c9cc..28667fab4 100644
--- a/sdk/src/sideModules/embeddedHost.ts
+++ b/sdk/src/sideModules/embeddedHost.ts
@@ -3,42 +3,36 @@
/**
* @file embeddedHost.ts
* @brief The single-file (playable-ad) transport: every needed side module's
- * glue + wasm is inlined as base64 by the exporter, so nothing is
- * fetched (ad networks require a self-contained .html). The exporter
- * decides *which* modules to embed from a content scan of the scene, so
- * an embedded registry that lacks a module the scene needs is an export
- * bug, not a silent runtime degrade.
+ * glue + wasm is inlined by the exporter, so nothing is fetched (ad
+ * networks require a self-contained .html). The exporter decides *which*
+ * modules to embed from a content scan of the scene, so an embedded
+ * registry that lacks a module the scene needs is an export bug, not a
+ * silent runtime degrade.
+ *
+ * How those bytes were ENCODED to survive the trip inside an HTML file
+ * is the host page's business, not this module's: they arrive already
+ * decoded. They used to arrive base64, which put one page's transport
+ * decision in the SDK and meant a change to it (they are deflated now,
+ * which is worth ~1MB of an ad network's 2MB cap) could not be made
+ * without changing this file too.
*/
import { createSideModuleHost, instantiateFromGlueText, type SideModuleHost } from './host';
import type { SideModuleId } from './registry';
export interface EmbeddedSideModuleEntry {
- /** base64 of the emscripten `<file>.js` glue text. */
- glueBase64: string;
- /** base64 of the `<file>.wasm` binary. */
- wasmBase64: string;
+ /** The emscripten `<file>.js` glue, as source text. */
+ glue: string;
+ /** The `<file>.wasm` binary. */
+ wasm: Uint8Array;
}
/** Inlined by the playable exporter as a window global, keyed by {@link SideModuleId}. */
export type EmbeddedSideModuleRegistry = Partial<Record<SideModuleId, EmbeddedSideModuleEntry>>;
-function decodeBase64ToText(b64: string): string {
- return atob(b64);
-}
-
-function decodeBase64ToBytes(b64: string): Uint8Array {
- const raw = atob(b64);
- const bytes = new Uint8Array(raw.length);
- for (let i = 0; i < raw.length; i++) bytes[i] = raw.charCodeAt(i);
- return bytes;
-}
-
export function createEmbeddedSideModuleHost(registry: EmbeddedSideModuleRegistry): SideModuleHost {
return createSideModuleHost(async (descriptor, id) => {
const entry = registry[id];
if (!entry) throw new Error(`side module "${id}" (${descriptor.file}) not embedded in this playable`);
- const glueText = decodeBase64ToText(entry.glueBase64);
- const wasmBytes = decodeBase64ToBytes(entry.wasmBase64);
- return instantiateFromGlueText(glueText, wasmBytes.buffer as ArrayBuffer, descriptor);
+ return instantiateFromGlueText(entry.glue, entry.wasm.buffer as ArrayBuffer, descriptor);
});
}