A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
fix(sdk): UTF-8 conversion that does not assume a browser
yhh02b1e2a3fa90
diff --git a/sdk/src/dragonbones/DragonBonesController.ts b/sdk/src/dragonbones/DragonBonesController.tsindex 3b0629fdb..04f75322b 100644--- a/sdk/src/dragonbones/DragonBonesController.ts+++ b/sdk/src/dragonbones/DragonBonesController.ts@@ -19,6 +19,7 @@ import { forEachMeshBatch, type MeshBatchVisitor } from '../skeletal/meshBatchesimport type { SkeletalBounds, SkeletalController } from '../skeletal/types';import type { DragonBonesWasmModule, DragonBonesWrappedAPI } from './DragonBonesModuleLoader';import { withMalloc } from '../wasmScratch';+import { encodeUtf8 } from '../utf8';import { log } from '../logger';/** Parse a `["a","b"]` the module published; a malformed one is no names, not a throw. */@@ -49,9 +50,9 @@ export class DragonBonesModuleController implements SkeletalController {*/loadSkeleton(skeletonData: Uint8Array | string, atlasJson: string): number {const skeleton = typeof skeletonData === 'string'- ? new TextEncoder().encode(skeletonData)+ ? encodeUtf8(skeletonData): skeletonData;- const atlas = new TextEncoder().encode(atlasJson);+ const atlas = encodeUtf8(atlasJson);return withMalloc(this.raw_, skeleton.length + atlas.length, basePtr => {const skeletonPtr = basePtr;diff --git a/sdk/src/utf8.ts b/sdk/src/utf8.tsnew file mode 100644index 000000000..51a530d79--- /dev/null+++ b/sdk/src/utf8.ts@@ -0,0 +1,104 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file utf8.ts+ * @brief UTF-8 conversion that does not assume a browser.+ *+ * @details `TextEncoder` is a web platform API, not a JavaScript one. The engine+ * runs on a device inside QuickJS, which does not have it — so SDK code+ * that reaches for the global works everywhere it was tested and throws+ * on the one target nobody can test in a unit test. That is how a+ * skeleton loader shipped in an APK that failed at boot.+ *+ * So: use the platform's encoder when it exists, because it is faster and+ * correct, and fall back to doing the arithmetic. The fallback is not a+ * nicety — on a native host it is the only path.+ */++const nativeEncoder: TextEncoder | null =+ typeof TextEncoder !== 'undefined' ? new TextEncoder() : null;+const nativeDecoder: TextDecoder | null =+ typeof TextDecoder !== 'undefined' ? new TextDecoder() : null;++/** `str` as UTF-8 bytes. */+export function encodeUtf8(str: string): Uint8Array {+ if (nativeEncoder) return nativeEncoder.encode(str);++ // Two passes: measure, then fill. One allocation of the right size beats+ // growing a buffer for what is usually a whole asset file.+ let bytes = 0;+ for (let i = 0; i < str.length; i++) {+ const c = str.codePointAt(i)!;+ if (c > 0xffff) i++; // a surrogate pair is one code point, two units+ bytes += c < 0x80 ? 1 : c < 0x800 ? 2 : c < 0x10000 ? 3 : 4;+ }++ const out = new Uint8Array(bytes);+ let at = 0;+ for (let i = 0; i < str.length; i++) {+ const c = str.codePointAt(i)!;+ if (c > 0xffff) i++;+ if (c < 0x80) {+ out[at++] = c;+ } else if (c < 0x800) {+ out[at++] = 0xc0 | (c >> 6);+ out[at++] = 0x80 | (c & 0x3f);+ } else if (c < 0x10000) {+ out[at++] = 0xe0 | (c >> 12);+ out[at++] = 0x80 | ((c >> 6) & 0x3f);+ out[at++] = 0x80 | (c & 0x3f);+ } else {+ out[at++] = 0xf0 | (c >> 18);+ out[at++] = 0x80 | ((c >> 12) & 0x3f);+ out[at++] = 0x80 | ((c >> 6) & 0x3f);+ out[at++] = 0x80 | (c & 0x3f);+ }+ }+ return out;+}++/** UTF-8 `bytes` as a string. Malformed input yields U+FFFD, as the platform's+ * decoder does in its non-fatal mode. */+export function decodeUtf8(bytes: Uint8Array): string {+ if (nativeDecoder) return nativeDecoder.decode(bytes);++ // Built in chunks: one big String.fromCharCode(...spread) blows the argument+ // limit on an asset-sized buffer.+ const parts: string[] = [];+ let units: number[] = [];+ for (let i = 0; i < bytes.length;) {+ const b = bytes[i];+ let code: number;+ let len: number;+ if (b < 0x80) { code = b; len = 1; }+ else if ((b & 0xe0) === 0xc0) { code = b & 0x1f; len = 2; }+ else if ((b & 0xf0) === 0xe0) { code = b & 0x0f; len = 3; }+ else if ((b & 0xf8) === 0xf0) { code = b & 0x07; len = 4; }+ else { code = 0xfffd; len = 1; }++ if (len > 1) {+ if (i + len > bytes.length) { code = 0xfffd; len = 1; }+ else {+ for (let k = 1; k < len; k++) {+ const cont = bytes[i + k];+ if ((cont & 0xc0) !== 0x80) { code = 0xfffd; len = 1; break; }+ code = (code << 6) | (cont & 0x3f);+ }+ }+ }+ i += len;++ if (code > 0xffff) {+ const v = code - 0x10000;+ units.push(0xd800 | (v >> 10), 0xdc00 | (v & 0x3ff));+ } else {+ units.push(code);+ }+ if (units.length >= 4096) {+ parts.push(String.fromCharCode(...units));+ units = [];+ }+ }+ if (units.length) parts.push(String.fromCharCode(...units));+ return parts.join('');+}diff --git a/sdk/tests/utf8.test.ts b/sdk/tests/utf8.test.tsnew file mode 100644index 000000000..0b54b6395--- /dev/null+++ b/sdk/tests/utf8.test.ts@@ -0,0 +1,79 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+//+// The fallback is the whole point of this module, and it is the path that never+// runs here: node has TextEncoder, so importing utf8.ts normally exercises the+// platform's implementation and proves nothing about the device. So each case+// runs BOTH — the module imported with the globals deleted (the QuickJS shape)+// checked against the platform's own answer.+import { describe, it, expect, beforeAll, vi } from 'vitest';++type Utf8 = typeof import('../src/utf8');++/** utf8.ts as a host with no TextEncoder/TextDecoder sees it. */+async function importWithoutGlobals(): Promise<Utf8> {+ const encoder = globalThis.TextEncoder;+ const decoder = globalThis.TextDecoder;+ // The module caches the globals at load, so they must be gone BEFORE it is+ // evaluated — hence resetModules + a fresh import rather than a spy.+ delete (globalThis as { TextEncoder?: unknown }).TextEncoder;+ delete (globalThis as { TextDecoder?: unknown }).TextDecoder;+ try {+ vi.resetModules();+ return await import('../src/utf8?fallback');+ } finally {+ globalThis.TextEncoder = encoder;+ globalThis.TextDecoder = decoder;+ }+}++const CASES = [+ ['ascii', 'DragonBoy_tex.png'],+ ['empty', ''],+ ['latin-1 supplement (2 bytes)', 'café — naïve'],+ ['CJK (3 bytes)', '龙骨动画 · 忍野'],+ ['astral / surrogate pair (4 bytes)', 'a🐉b👨👩👧c'],+ ['mixed widths', 'xé中\u{1F409}y'],+ ['a whole JSON-ish blob', JSON.stringify({ name: '龙', frames: [1, 2, 3], note: 'ünïcödé 🐲' })],+] as const;++describe('utf8 without the web globals (the device path)', () => {+ let fallback: Utf8;+ beforeAll(async () => { fallback = await importWithoutGlobals(); });++ it('the fallback really is the fallback (no globals at load time)', () => {+ // A canary: if this file ever stops removing the globals, every assertion+ // below would compare the platform against itself and pass vacuously.+ expect(fallback.encodeUtf8).not.toBe((globalThis.TextEncoder.prototype.encode as unknown));+ expect(globalThis.TextEncoder).toBeDefined(); // restored for everyone else+ });++ for (const [label, input] of CASES) {+ it(`encodes ${label} exactly as TextEncoder does`, () => {+ expect(Array.from(fallback.encodeUtf8(input)))+ .toEqual(Array.from(new TextEncoder().encode(input)));+ });++ it(`round-trips ${label}`, () => {+ expect(fallback.decodeUtf8(fallback.encodeUtf8(input))).toBe(input);+ });++ it(`decodes ${label} exactly as TextDecoder does`, () => {+ const bytes = new TextEncoder().encode(input);+ expect(fallback.decodeUtf8(bytes)).toBe(new TextDecoder().decode(bytes));+ });+ }++ it('decodes a buffer larger than one chunk', () => {+ // The decoder emits in 4096-unit chunks; a skeleton file is far bigger, and+ // an off-by-one at the seam would corrupt exactly one character in 4096.+ const input = '龙'.repeat(5000) + 'tail';+ expect(fallback.decodeUtf8(new TextEncoder().encode(input))).toBe(input);+ });++ it('answers U+FFFD for malformed bytes rather than throwing', () => {+ // A truncated 3-byte sequence and a stray continuation byte.+ expect(fallback.decodeUtf8(new Uint8Array([0xe4, 0xb8]))).toBe('��');+ expect(fallback.decodeUtf8(new Uint8Array([0x41, 0x80, 0x42]))).toBe('A�B');+ });+});