A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(native): the engine's video decoder on the device, and an ownership bug it exposed
yhha79912c0432d
diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.jsonindex 90c267214..c07300e0a 100644--- a/.vscode/c_cpp_properties.json+++ b/.vscode/c_cpp_properties.json@@ -19,7 +19,8 @@"${env:ESTELLA_DAWN_DIR}/include","${env:ESTELLA_DAWN_BUILD}/gen/include","${env:ESTELLA_QUICKJS_DIR}",- "${workspaceFolder}/build/native_intellisense"+ "${workspaceFolder}/build/native_intellisense",+ "${workspaceFolder}/third_party/pl_mpeg"],"defines": ["_DEBUG",@@ -47,7 +48,8 @@"${workspaceFolder}/third_party/nlohmann","${workspaceFolder}/third_party/box2d/include","${workspaceFolder}/build/quickjs_headers",- "${workspaceFolder}/build/native_intellisense"+ "${workspaceFolder}/build/native_intellisense",+ "${workspaceFolder}/third_party/pl_mpeg"],"defines": ["_DEBUG",@@ -71,7 +73,8 @@"${workspaceFolder}/third_party/stb","${workspaceFolder}/third_party/spine-runtimes-4.2/spine-c/spine-c/include","${workspaceFolder}/third_party/nlohmann",- "${workspaceFolder}/third_party/box2d/include"+ "${workspaceFolder}/third_party/box2d/include",+ "${workspaceFolder}/third_party/pl_mpeg"],"defines": ["ES_PLATFORM_WEB",@@ -97,10 +100,18 @@"${env:ESTELLA_QUICKJS_DIR}","${workspaceFolder}/build/quickjs_headers","${workspaceFolder}/build-native/gen",- "${workspaceFolder}/build/native_intellisense"+ "${workspaceFolder}/build/native_intellisense",+ "${workspaceFolder}/third_party/box2d/include",+ "${workspaceFolder}/third_party/pl_mpeg",+ "${workspaceFolder}/third_party/basis_universal/transcoder",+ "${workspaceFolder}/third_party/miniaudio"],"defines": [- "ES_ENABLE_WEBGPU"+ "ES_ENABLE_WEBGPU",+ "ES_ENABLE_BITMAP_TEXT",+ "ES_ENABLE_TILEMAP",+ "ES_ENABLE_PARTICLES",+ "ES_ENABLE_POSTPROCESS"],"cStandard": "c17","cppStandard": "c++20",@@ -122,10 +133,18 @@"${env:ESTELLA_QUICKJS_DIR}","${workspaceFolder}/build/quickjs_headers","${workspaceFolder}/build-native-ios/gen",- "${workspaceFolder}/build/native_intellisense"+ "${workspaceFolder}/build/native_intellisense",+ "${workspaceFolder}/third_party/box2d/include",+ "${workspaceFolder}/third_party/pl_mpeg",+ "${workspaceFolder}/third_party/basis_universal/transcoder",+ "${workspaceFolder}/third_party/miniaudio"],"defines": [- "ES_ENABLE_WEBGPU"+ "ES_ENABLE_WEBGPU",+ "ES_ENABLE_BITMAP_TEXT",+ "ES_ENABLE_TILEMAP",+ "ES_ENABLE_PARTICLES",+ "ES_ENABLE_POSTPROCESS"],"cStandard": "c17","cppStandard": "c++20",diff --git a/desktop/src/project/targetSupport.ts b/desktop/src/project/targetSupport.tsindex 05e9ee874..650a0e69f 100644--- a/desktop/src/project/targetSupport.ts+++ b/desktop/src/project/targetSupport.ts@@ -6,9 +6,10 @@* The editor authors every subsystem the engine has; a target does not* necessarily compile all of them. The native app builds a subset: the flags* `native/CMakeLists.txt` sets decide what `cmake/ESEngineSources.cmake` compiles- * in, and the optional emscripten side modules (Box2D, Spine, video) have no- * native counterpart yet. iOS and Android build from that same CMakeLists, so- * their gaps are one list.+ * in, and of the optional emscripten side modules Box2D and the video decoder are+ * compiled into the host binary (there is no dynamic-linking story on a device) while+ * the Spine runtime is not there yet. iOS and Android build from that same+ * CMakeLists, so their gaps are one list.** Without this, an export writes a package that is quietly missing half a* scene. This is the one declaration of that gap, so the export can name it@@ -76,9 +77,7 @@ export interface SubsystemGap {/** What the native app cannot render. ONE list: iOS and Android compile the same* `native/CMakeLists.txt`, so a flag flipped there closes the gap on both. */const NATIVE_GAPS: readonly SubsystemGap[] = [- { subsystem: 'physics', why: 'Box2D ships as an emscripten side module; the native host has no counterpart yet' },{ subsystem: 'spine', why: 'the Spine runtime ships as an emscripten side module; the native host has no counterpart yet' },- { subsystem: 'video', why: 'the video decoder ships as an emscripten side module; the native host has no counterpart yet' },];/**diff --git a/desktop/tests/target-support.test.ts b/desktop/tests/target-support.test.tsindex 9b538f189..e2beede11 100644--- a/desktop/tests/target-support.test.ts+++ b/desktop/tests/target-support.test.ts@@ -53,12 +53,12 @@ describe('collectSubsystems', () => {});describe('subsystemGapWarnings', () => {- const usage = new Map<Subsystem, string[]>([['physics', ['scenes/b.esscene', 'scenes/a.esscene']]]);+ const usage = new Map<Subsystem, string[]>([['spine', ['scenes/b.esscene', 'scenes/a.esscene']]]);it('names the subsystem, the reason and the files', () => {const [warning, ...rest] = subsystemGapWarnings('android', usage);expect(rest).toEqual([]);- expect(warning).toContain('Physics (Box2D) will not render');+ expect(warning).toContain('Spine animation will not render');expect(warning).toContain('scenes/a.esscene, scenes/b.esscene');});@@ -137,17 +137,17 @@ describe('exportGame warns about content the target cannot render', () => {expect(res.errors).toEqual([]);expect(res.ok).toBe(true);const gaps = res.warnings.filter((w) => w.startsWith('android:'));- expect(gaps).toHaveLength(2);- expect(gaps.join('\n')).toContain('Physics (Box2D) will not render');+ expect(gaps).toHaveLength(1);expect(gaps.join('\n')).toContain('Spine animation will not render');expect(gaps.join('\n')).toContain('scenes/main.esscene');- // What the native core DOES compile is not warned about: text (the host- // rasterizes glyphs for the same atlas), tilemaps, particles and- // post-processing. The gaps are what the build really cannot do, not- // everything that happens to be optional.+ // What the native build DOES have is not warned about: text (the host+ // rasterizes glyphs for the same atlas), tilemaps, particles,+ // post-processing, and physics (Box2D compiles into the host binary). The+ // gaps are what the build really cannot do, not everything optional.expect(gaps.join('\n')).not.toContain('Text will not render');expect(gaps.join('\n')).not.toContain('Tilemaps will not render');expect(gaps.join('\n')).not.toContain('Particles will not render');+ expect(gaps.join('\n')).not.toContain('Physics (Box2D) will not render');}, 60_000);it('stays quiet for a target that renders all of it', async () => {diff --git a/sdk/src/video/PlatformVideoBackend.ts b/sdk/src/video/PlatformVideoBackend.tsindex fe918b56e..ed6c81542 100644--- a/sdk/src/video/PlatformVideoBackend.ts+++ b/sdk/src/video/PlatformVideoBackend.ts@@ -26,7 +26,9 @@ export interface VideoStreamHandle {setPlaybackRate(rate: number): void;/** Upload the current frame if a new one arrived; called once per frame. */- pump(module: ESEngineModule): void;+ /** @param module the wasm engine module, or null on a core with no heap —+ * the texture upload takes bytes directly there. */+ pump(module: ESEngineModule | null): void;onReady?: () => void;onEnded?: () => void;diff --git a/sdk/src/video/VideoAPI.ts b/sdk/src/video/VideoAPI.tsindex c538fb362..0b000c2a7 100644--- a/sdk/src/video/VideoAPI.ts+++ b/sdk/src/video/VideoAPI.ts@@ -57,7 +57,7 @@ export class VideoAPI {return resolved !== siblingRef ? resolved : undefined;}- update(module: ESEngineModule): void {+ update(module: ESEngineModule | null): void {if (this.disposed_) return;for (const handle of this.handles_) {// One stream's decode failure (wasm abort, detached heap) must notdiff --git a/sdk/src/video/VideoPlugin.ts b/sdk/src/video/VideoPlugin.tsindex d10894ece..e14b84338 100644--- a/sdk/src/video/VideoPlugin.ts+++ b/sdk/src/video/VideoPlugin.ts@@ -100,8 +100,9 @@ export class VideoPlugin implements Plugin {(_time: TimeData, videoAPI: VideoAPI) => {if (isEditor() && !isPlayMode()) return;const world = app.world;+ // Null on a native core: the frame upload goes through the+ // ResourceManager's byte path instead of a wasm heap.const module = world.getWasmModule();- if (!module) return;const entities = world.getEntitiesWithComponents([Video]);live.clear();diff --git a/sdk/src/video/WasmVideoBackend.ts b/sdk/src/video/WasmVideoBackend.tsindex 0d02864ee..3d024f364 100644--- a/sdk/src/video/WasmVideoBackend.ts+++ b/sdk/src/video/WasmVideoBackend.ts@@ -132,8 +132,15 @@ class WasmVideoStreamHandle implements VideoStreamHandle {const ptr = video._malloc(data.length);if (!ptr) throw new Error('videodec out of memory');video.HEAPU8.set(data, ptr);- // _es_video_open owns `ptr` from here (freed on close or on failure).- const decoder = video._es_video_open(ptr, data.length);+ // The decoder copies the bytes into its own memory, so this buffer is+ // ours to release either way — ownership does not cross the boundary+ // (the two sides share an allocator on the web and not on a device).+ let decoder = 0;+ try {+ decoder = video._es_video_open(ptr, data.length);+ } finally {+ video._free(ptr);+ }if (!decoder) {throw new Error(`not a decodable MPEG-1 stream (or no free decoder slot): ${url} — ` +@@ -266,7 +273,7 @@ class WasmVideoStreamHandle implements VideoStreamHandle {// === texture pump =======================================================- pump(module: ESEngineModule): void {+ pump(module: ESEngineModule | null): void {const video = this.mod_;if (this.disposed_ || !video) return;diff --git a/sdk/src/video/WebVideoBackend.ts b/sdk/src/video/WebVideoBackend.tsindex 7142b4919..4cb73dc4b 100644--- a/sdk/src/video/WebVideoBackend.ts+++ b/sdk/src/video/WebVideoBackend.ts@@ -109,7 +109,10 @@ class WebVideoStreamHandle implements VideoStreamHandle {// === texture pump =======================================================- pump(module: ESEngineModule): void {+ pump(module: ESEngineModule | null): void {+ // This backend IS the browser's <video> element, so it only ever runs on a+ // realm that has the wasm module (it samples through its GL context).+ if (!module) return;if (this.disposed_ || this.width_ <= 0 || this.height_ <= 0) return;if (this.video_.readyState < 2) return; // HAVE_CURRENT_DATA — a frame to samplediff --git a/src/esengine/bindings/VideoBindings.hpp b/src/esengine/bindings/VideoBindings.hppnew file mode 100644index 000000000..4ac32badc--- /dev/null+++ b/src/esengine/bindings/VideoBindings.hpp@@ -0,0 +1,51 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file VideoBindings.hpp+ * @brief The MPEG-1 decoder's entry points, declared once for both platforms.+ * @details The engine owns its video decode (pl_mpeg) rather than depending on a+ * platform's media stack, which is what makes a cooked `.esv` play the same+ * way everywhere. On the web it links as a wasm side module; a native app+ * compiles the same TU into the host binary and reaches it through the+ * QuickJS wrappers EHT generates from these declarations.+ *+ * Bulk data crosses as an offset into the caller's heap (wasm linear memory+ * on the web, the host arena on a device): the source bytes in, the decoded+ * RGBA frame out. Nothing owns anything across the boundary — see+ * `es_video_open`.+ *+ * @author ESEngine Team+ * @date 2026+ *+ * @copyright Copyright (c) 2026 ESEngine Team+ * Licensed under the Apache License, Version 2.0.+ */+#pragma once++#include <cstdint>++extern "C" {++/** Open a stream from `size` bytes at `dataPtr`; the bytes are copied, so the caller+ * keeps its buffer. Returns a handle >= 1, or 0. */+int es_video_open(uintptr_t dataPtr, uint32_t size);+/** Release the instance and everything it decoded. Safe on a stale handle. */+void es_video_close(int handle);++uint32_t es_video_width(int handle);+uint32_t es_video_height(int handle);+double es_video_duration(int handle);+double es_video_framerate(int handle);+/** Presentation time of the last decoded frame, in seconds. */+double es_video_time(int handle);+void es_video_set_loop(int handle, int loop);+int es_video_has_ended(int handle);++/** Decode forward by `dt` seconds; returns 1 when a new frame became available. */+int es_video_advance(int handle, double dt);+/** Write the pending frame as RGBA into `outPtr` (>= width*height*4 bytes). */+int es_video_frame_rgba(int handle, uintptr_t outPtr, uint32_t outSize);+/** Exact-seek to `time` seconds, leaving the target frame pending. */+int es_video_seek(int handle, double time);++} // extern "C"diff --git a/src/esengine/bindings/VideoModuleEntry.cpp b/src/esengine/bindings/VideoModuleEntry.cppindex 90af98fbf..02c1cfeae 100644--- a/src/esengine/bindings/VideoModuleEntry.cpp+++ b/src/esengine/bindings/VideoModuleEntry.cpp@@ -51,22 +51,27 @@ void onVideoFrame(plm_t*, plm_frame_t* frame, void* user) {extern "C" {/**- * Open an MPEG-PS / MPEG-1 stream. Takes ownership of `data` (allocated in this- * module's heap via _malloc); it is freed on close — and on failure — so the- * caller never frees it. Returns a handle >= 1, or 0 on failure.+ * Open an MPEG-PS / MPEG-1 stream from `dataPtr` (`size` bytes in the caller's heap).+ * The bytes are COPIED into memory this module owns, so the caller keeps and frees+ * its own buffer: ownership must not cross the boundary, because the two sides do not+ * necessarily share an allocator. They do in the wasm build — one emscripten heap —+ * and they do not in a native app, where the caller writes into the host's arena+ * (host/heap.hpp) and this module's `free` is libc's. Returns a handle >= 1, or 0.*/-int es_video_open(uint8_t* data, uint32_t size) {+int es_video_open(uintptr_t dataPtr, uint32_t size) {+ const auto* source = reinterpret_cast<const uint8_t*>(dataPtr);+ if (!source || size == 0) return 0;int slot = -1;for (int i = 0; i < kMaxInstances; i++) {if (!g_instances[i].plm) { slot = i; break; }}- if (slot < 0) {- free(data);- return 0;- }- plm_t* plm = plm_create_with_memory(data, size, /* free_when_done */ TRUE);+ if (slot < 0) return 0;+ auto* owned = static_cast<uint8_t*>(malloc(size));+ if (!owned) return 0;+ memcpy(owned, source, size);+ plm_t* plm = plm_create_with_memory(owned, size, /* free_when_done */ TRUE);if (!plm) {- free(data);+ free(owned);return 0;}// Validate via the parsed sequence header, not plm_probe(): probe scans@@ -138,11 +143,13 @@ int es_video_advance(int handle, double dt) {/*** Convert the latest decoded frame into `out` as bottom-first RGBA rows — the* orientation the engine's flipY-off texture upload samples upright. `outSize`- * must be at least width*height*4. Returns 1 and clears the new-frame flag on- * success.+ * must be at least width*height*4. `outPtr` is an offset the caller wrote — wasm+ * linear memory on the web, the host arena on a device — so the pixels land where+ * the caller reads them. Returns 1 and clears the new-frame flag on success.*/-int es_video_frame_rgba(int handle, uint8_t* out, uint32_t outSize) {+int es_video_frame_rgba(int handle, uintptr_t outPtr, uint32_t outSize) {Instance* inst = get(handle);+ auto* out = reinterpret_cast<uint8_t*>(outPtr);if (!inst || !inst->frame || !out) return 0;const int w = plm_get_width(inst->plm);const int h = plm_get_height(inst->plm);diff --git a/tools/eht/generators/native_functions.py b/tools/eht/generators/native_functions.pyindex af2586337..f19f199d3 100644--- a/tools/eht/generators/native_functions.py+++ b/tools/eht/generators/native_functions.py@@ -262,10 +262,21 @@ class NativeFunctionsGenerator:self.skipped: List[Skipped] = []self.emitted: List[str] = []+ @staticmethod+ def _js_name(name: str) -> str:+ """The global the wrapper is registered under.++ `es_` marks the boundary surface, so a C name that already carries it keeps+ it — the video module's entry points are spelled `es_video_open`, and+ `es_es_video_open` would read as a mistake on both sides of the boundary.+ """+ return name if name.startswith('es_') else f'es_{name}'+def _wrapper(self, fn: Function) -> List[str]:js_params = [p for p in fn.params if p.cpp_type not in _IMPLICIT_REFS]out = [f'#ifdef {g}' for g in fn.guards]- out.append(f'static JSValue es_{fn.name}(JSContext* ctx, JSValueConst, int argc, JSValueConst* argv) {{')+ out.append(f'static JSValue {self._js_name(fn.name)}'+ f'(JSContext* ctx, JSValueConst, int argc, JSValueConst* argv) {{')if js_params:out.append(f' if (argc < {len(js_params)}) return JS_UNDEFINED;')out.append(' (void)argc; (void)argv;')@@ -418,7 +429,8 @@ class NativeFunctionsGenerator:])for fn in functions:takes_implicit = any(p.cpp_type in _IMPLICIT_REFS for p in fn.params)- lines.append(f" bind('{fn.name}', 'es_{fn.name}', {str(takes_implicit).lower()});")+ lines.append(f" bind('{fn.name}', '{self._js_name(fn.name)}', "+ f"{str(takes_implicit).lower()});")lines.extend([' return api as NativeEngineApi;','}',@@ -462,8 +474,9 @@ class NativeFunctionsGenerator:for fn in sorted(functions, key=lambda f: f.name):js_arity = len([p for p in fn.params if p.cpp_type not in _IMPLICIT_REFS])lines.extend(f'#ifdef {g}' for g in fn.guards)- lines.append(f' JS_SetPropertyStr(ctx, global, "es_{fn.name}", '- f'JS_NewCFunction(ctx, es_{fn.name}, "es_{fn.name}", {js_arity}));')+ js = self._js_name(fn.name)+ lines.append(f' JS_SetPropertyStr(ctx, global, "{js}", '+ f'JS_NewCFunction(ctx, {js}, "{js}", {js_arity}));')lines.extend('#endif' for _ in fn.guards)lines.append('}')lines.append('')