Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

fix(sdk): UTF-8 conversion that does not assume a browser

yhh02b1e2a3fa90
diff --git a/sdk/src/dragonbones/DragonBonesController.ts b/sdk/src/dragonbones/DragonBonesController.ts
index 3b0629fdb..04f75322b 100644
--- a/sdk/src/dragonbones/DragonBonesController.ts
+++ b/sdk/src/dragonbones/DragonBonesController.ts
@@ -19,6 +19,7 @@ import { forEachMeshBatch, type MeshBatchVisitor } from '../skeletal/meshBatches
import type { SkeletalBounds, SkeletalController } from '../skeletal/types';
import type { DragonBonesWasmModule, DragonBonesWrappedAPI } from './DragonBonesModuleLoader';
import { withMalloc } from '../wasmScratch';
+import { encodeUtf8 } from '../utf8';
import { log } from '../logger';
/** Parse a `["a","b"]` the module published; a malformed one is no names, not a throw. */
@@ -49,9 +50,9 @@ export class DragonBonesModuleController implements SkeletalController {
*/
loadSkeleton(skeletonData: Uint8Array | string, atlasJson: string): number {
const skeleton = typeof skeletonData === 'string'
- ? new TextEncoder().encode(skeletonData)
+ ? encodeUtf8(skeletonData)
: skeletonData;
- const atlas = new TextEncoder().encode(atlasJson);
+ const atlas = encodeUtf8(atlasJson);
return withMalloc(this.raw_, skeleton.length + atlas.length, basePtr => {
const skeletonPtr = basePtr;
diff --git a/sdk/src/utf8.ts b/sdk/src/utf8.ts
new file mode 100644
index 000000000..51a530d79
--- /dev/null
+++ b/sdk/src/utf8.ts
@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file utf8.ts
+ * @brief UTF-8 conversion that does not assume a browser.
+ *
+ * @details `TextEncoder` is a web platform API, not a JavaScript one. The engine
+ * runs on a device inside QuickJS, which does not have it — so SDK code
+ * that reaches for the global works everywhere it was tested and throws
+ * on the one target nobody can test in a unit test. That is how a
+ * skeleton loader shipped in an APK that failed at boot.
+ *
+ * So: use the platform's encoder when it exists, because it is faster and
+ * correct, and fall back to doing the arithmetic. The fallback is not a
+ * nicety — on a native host it is the only path.
+ */
+
+const nativeEncoder: TextEncoder | null =
+ typeof TextEncoder !== 'undefined' ? new TextEncoder() : null;
+const nativeDecoder: TextDecoder | null =
+ typeof TextDecoder !== 'undefined' ? new TextDecoder() : null;
+
+/** `str` as UTF-8 bytes. */
+export function encodeUtf8(str: string): Uint8Array {
+ if (nativeEncoder) return nativeEncoder.encode(str);
+
+ // Two passes: measure, then fill. One allocation of the right size beats
+ // growing a buffer for what is usually a whole asset file.
+ let bytes = 0;
+ for (let i = 0; i < str.length; i++) {
+ const c = str.codePointAt(i)!;
+ if (c > 0xffff) i++; // a surrogate pair is one code point, two units
+ bytes += c < 0x80 ? 1 : c < 0x800 ? 2 : c < 0x10000 ? 3 : 4;
+ }
+
+ const out = new Uint8Array(bytes);
+ let at = 0;
+ for (let i = 0; i < str.length; i++) {
+ const c = str.codePointAt(i)!;
+ if (c > 0xffff) i++;
+ if (c < 0x80) {
+ out[at++] = c;
+ } else if (c < 0x800) {
+ out[at++] = 0xc0 | (c >> 6);
+ out[at++] = 0x80 | (c & 0x3f);
+ } else if (c < 0x10000) {
+ out[at++] = 0xe0 | (c >> 12);
+ out[at++] = 0x80 | ((c >> 6) & 0x3f);
+ out[at++] = 0x80 | (c & 0x3f);
+ } else {
+ out[at++] = 0xf0 | (c >> 18);
+ out[at++] = 0x80 | ((c >> 12) & 0x3f);
+ out[at++] = 0x80 | ((c >> 6) & 0x3f);
+ out[at++] = 0x80 | (c & 0x3f);
+ }
+ }
+ return out;
+}
+
+/** UTF-8 `bytes` as a string. Malformed input yields U+FFFD, as the platform's
+ * decoder does in its non-fatal mode. */
+export function decodeUtf8(bytes: Uint8Array): string {
+ if (nativeDecoder) return nativeDecoder.decode(bytes);
+
+ // Built in chunks: one big String.fromCharCode(...spread) blows the argument
+ // limit on an asset-sized buffer.
+ const parts: string[] = [];
+ let units: number[] = [];
+ for (let i = 0; i < bytes.length;) {
+ const b = bytes[i];
+ let code: number;
+ let len: number;
+ if (b < 0x80) { code = b; len = 1; }
+ else if ((b & 0xe0) === 0xc0) { code = b & 0x1f; len = 2; }
+ else if ((b & 0xf0) === 0xe0) { code = b & 0x0f; len = 3; }
+ else if ((b & 0xf8) === 0xf0) { code = b & 0x07; len = 4; }
+ else { code = 0xfffd; len = 1; }
+
+ if (len > 1) {
+ if (i + len > bytes.length) { code = 0xfffd; len = 1; }
+ else {
+ for (let k = 1; k < len; k++) {
+ const cont = bytes[i + k];
+ if ((cont & 0xc0) !== 0x80) { code = 0xfffd; len = 1; break; }
+ code = (code << 6) | (cont & 0x3f);
+ }
+ }
+ }
+ i += len;
+
+ if (code > 0xffff) {
+ const v = code - 0x10000;
+ units.push(0xd800 | (v >> 10), 0xdc00 | (v & 0x3ff));
+ } else {
+ units.push(code);
+ }
+ if (units.length >= 4096) {
+ parts.push(String.fromCharCode(...units));
+ units = [];
+ }
+ }
+ if (units.length) parts.push(String.fromCharCode(...units));
+ return parts.join('');
+}
diff --git a/sdk/tests/utf8.test.ts b/sdk/tests/utf8.test.ts
new file mode 100644
index 000000000..0b54b6395
--- /dev/null
+++ b/sdk/tests/utf8.test.ts
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+//
+// The fallback is the whole point of this module, and it is the path that never
+// runs here: node has TextEncoder, so importing utf8.ts normally exercises the
+// platform's implementation and proves nothing about the device. So each case
+// runs BOTH — the module imported with the globals deleted (the QuickJS shape)
+// checked against the platform's own answer.
+import { describe, it, expect, beforeAll, vi } from 'vitest';
+
+type Utf8 = typeof import('../src/utf8');
+
+/** utf8.ts as a host with no TextEncoder/TextDecoder sees it. */
+async function importWithoutGlobals(): Promise<Utf8> {
+ const encoder = globalThis.TextEncoder;
+ const decoder = globalThis.TextDecoder;
+ // The module caches the globals at load, so they must be gone BEFORE it is
+ // evaluated — hence resetModules + a fresh import rather than a spy.
+ delete (globalThis as { TextEncoder?: unknown }).TextEncoder;
+ delete (globalThis as { TextDecoder?: unknown }).TextDecoder;
+ try {
+ vi.resetModules();
+ return await import('../src/utf8?fallback');
+ } finally {
+ globalThis.TextEncoder = encoder;
+ globalThis.TextDecoder = decoder;
+ }
+}
+
+const CASES = [
+ ['ascii', 'DragonBoy_tex.png'],
+ ['empty', ''],
+ ['latin-1 supplement (2 bytes)', 'café — naïve'],
+ ['CJK (3 bytes)', '龙骨动画 · 忍野'],
+ ['astral / surrogate pair (4 bytes)', 'a🐉b👨‍👩‍👧c'],
+ ['mixed widths', 'xé中\u{1F409}y'],
+ ['a whole JSON-ish blob', JSON.stringify({ name: '龙', frames: [1, 2, 3], note: 'ünïcödé 🐲' })],
+] as const;
+
+describe('utf8 without the web globals (the device path)', () => {
+ let fallback: Utf8;
+ beforeAll(async () => { fallback = await importWithoutGlobals(); });
+
+ it('the fallback really is the fallback (no globals at load time)', () => {
+ // A canary: if this file ever stops removing the globals, every assertion
+ // below would compare the platform against itself and pass vacuously.
+ expect(fallback.encodeUtf8).not.toBe((globalThis.TextEncoder.prototype.encode as unknown));
+ expect(globalThis.TextEncoder).toBeDefined(); // restored for everyone else
+ });
+
+ for (const [label, input] of CASES) {
+ it(`encodes ${label} exactly as TextEncoder does`, () => {
+ expect(Array.from(fallback.encodeUtf8(input)))
+ .toEqual(Array.from(new TextEncoder().encode(input)));
+ });
+
+ it(`round-trips ${label}`, () => {
+ expect(fallback.decodeUtf8(fallback.encodeUtf8(input))).toBe(input);
+ });
+
+ it(`decodes ${label} exactly as TextDecoder does`, () => {
+ const bytes = new TextEncoder().encode(input);
+ expect(fallback.decodeUtf8(bytes)).toBe(new TextDecoder().decode(bytes));
+ });
+ }
+
+ it('decodes a buffer larger than one chunk', () => {
+ // The decoder emits in 4096-unit chunks; a skeleton file is far bigger, and
+ // an off-by-one at the seam would corrupt exactly one character in 4096.
+ const input = '龙'.repeat(5000) + 'tail';
+ expect(fallback.decodeUtf8(new TextEncoder().encode(input))).toBe(input);
+ });
+
+ it('answers U+FFFD for malformed bytes rather than throwing', () => {
+ // A truncated 3-byte sequence and a stray continuation byte.
+ expect(fallback.decodeUtf8(new Uint8Array([0xe4, 0xb8]))).toBe('��');
+ expect(fallback.decodeUtf8(new Uint8Array([0x41, 0x80, 0x42]))).toBe('A�B');
+ });
+});