Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(native): the engine's video decoder on the device, and an ownership bug it exposed

yhha79912c0432d
diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json
index 90c267214..c07300e0a 100644
--- a/.vscode/c_cpp_properties.json
+++ b/.vscode/c_cpp_properties.json
@@ -19,7 +19,8 @@
"${env:ESTELLA_DAWN_DIR}/include",
"${env:ESTELLA_DAWN_BUILD}/gen/include",
"${env:ESTELLA_QUICKJS_DIR}",
- "${workspaceFolder}/build/native_intellisense"
+ "${workspaceFolder}/build/native_intellisense",
+ "${workspaceFolder}/third_party/pl_mpeg"
],
"defines": [
"_DEBUG",
@@ -47,7 +48,8 @@
"${workspaceFolder}/third_party/nlohmann",
"${workspaceFolder}/third_party/box2d/include",
"${workspaceFolder}/build/quickjs_headers",
- "${workspaceFolder}/build/native_intellisense"
+ "${workspaceFolder}/build/native_intellisense",
+ "${workspaceFolder}/third_party/pl_mpeg"
],
"defines": [
"_DEBUG",
@@ -71,7 +73,8 @@
"${workspaceFolder}/third_party/stb",
"${workspaceFolder}/third_party/spine-runtimes-4.2/spine-c/spine-c/include",
"${workspaceFolder}/third_party/nlohmann",
- "${workspaceFolder}/third_party/box2d/include"
+ "${workspaceFolder}/third_party/box2d/include",
+ "${workspaceFolder}/third_party/pl_mpeg"
],
"defines": [
"ES_PLATFORM_WEB",
@@ -97,10 +100,18 @@
"${env:ESTELLA_QUICKJS_DIR}",
"${workspaceFolder}/build/quickjs_headers",
"${workspaceFolder}/build-native/gen",
- "${workspaceFolder}/build/native_intellisense"
+ "${workspaceFolder}/build/native_intellisense",
+ "${workspaceFolder}/third_party/box2d/include",
+ "${workspaceFolder}/third_party/pl_mpeg",
+ "${workspaceFolder}/third_party/basis_universal/transcoder",
+ "${workspaceFolder}/third_party/miniaudio"
],
"defines": [
- "ES_ENABLE_WEBGPU"
+ "ES_ENABLE_WEBGPU",
+ "ES_ENABLE_BITMAP_TEXT",
+ "ES_ENABLE_TILEMAP",
+ "ES_ENABLE_PARTICLES",
+ "ES_ENABLE_POSTPROCESS"
],
"cStandard": "c17",
"cppStandard": "c++20",
@@ -122,10 +133,18 @@
"${env:ESTELLA_QUICKJS_DIR}",
"${workspaceFolder}/build/quickjs_headers",
"${workspaceFolder}/build-native-ios/gen",
- "${workspaceFolder}/build/native_intellisense"
+ "${workspaceFolder}/build/native_intellisense",
+ "${workspaceFolder}/third_party/box2d/include",
+ "${workspaceFolder}/third_party/pl_mpeg",
+ "${workspaceFolder}/third_party/basis_universal/transcoder",
+ "${workspaceFolder}/third_party/miniaudio"
],
"defines": [
- "ES_ENABLE_WEBGPU"
+ "ES_ENABLE_WEBGPU",
+ "ES_ENABLE_BITMAP_TEXT",
+ "ES_ENABLE_TILEMAP",
+ "ES_ENABLE_PARTICLES",
+ "ES_ENABLE_POSTPROCESS"
],
"cStandard": "c17",
"cppStandard": "c++20",
diff --git a/desktop/src/project/targetSupport.ts b/desktop/src/project/targetSupport.ts
index 05e9ee874..650a0e69f 100644
--- a/desktop/src/project/targetSupport.ts
+++ b/desktop/src/project/targetSupport.ts
@@ -6,9 +6,10 @@
* The editor authors every subsystem the engine has; a target does not
* necessarily compile all of them. The native app builds a subset: the flags
* `native/CMakeLists.txt` sets decide what `cmake/ESEngineSources.cmake` compiles
- * in, and the optional emscripten side modules (Box2D, Spine, video) have no
- * native counterpart yet. iOS and Android build from that same CMakeLists, so
- * their gaps are one list.
+ * in, and of the optional emscripten side modules Box2D and the video decoder are
+ * compiled into the host binary (there is no dynamic-linking story on a device) while
+ * the Spine runtime is not there yet. iOS and Android build from that same
+ * CMakeLists, so their gaps are one list.
*
* Without this, an export writes a package that is quietly missing half a
* scene. This is the one declaration of that gap, so the export can name it
@@ -76,9 +77,7 @@ export interface SubsystemGap {
/** What the native app cannot render. ONE list: iOS and Android compile the same
* `native/CMakeLists.txt`, so a flag flipped there closes the gap on both. */
const NATIVE_GAPS: readonly SubsystemGap[] = [
- { subsystem: 'physics', why: 'Box2D ships as an emscripten side module; the native host has no counterpart yet' },
{ subsystem: 'spine', why: 'the Spine runtime ships as an emscripten side module; the native host has no counterpart yet' },
- { subsystem: 'video', why: 'the video decoder ships as an emscripten side module; the native host has no counterpart yet' },
];
/**
diff --git a/desktop/tests/target-support.test.ts b/desktop/tests/target-support.test.ts
index 9b538f189..e2beede11 100644
--- a/desktop/tests/target-support.test.ts
+++ b/desktop/tests/target-support.test.ts
@@ -53,12 +53,12 @@ describe('collectSubsystems', () => {
});
describe('subsystemGapWarnings', () => {
- const usage = new Map<Subsystem, string[]>([['physics', ['scenes/b.esscene', 'scenes/a.esscene']]]);
+ const usage = new Map<Subsystem, string[]>([['spine', ['scenes/b.esscene', 'scenes/a.esscene']]]);
it('names the subsystem, the reason and the files', () => {
const [warning, ...rest] = subsystemGapWarnings('android', usage);
expect(rest).toEqual([]);
- expect(warning).toContain('Physics (Box2D) will not render');
+ expect(warning).toContain('Spine animation will not render');
expect(warning).toContain('scenes/a.esscene, scenes/b.esscene');
});
@@ -137,17 +137,17 @@ describe('exportGame warns about content the target cannot render', () => {
expect(res.errors).toEqual([]);
expect(res.ok).toBe(true);
const gaps = res.warnings.filter((w) => w.startsWith('android:'));
- expect(gaps).toHaveLength(2);
- expect(gaps.join('\n')).toContain('Physics (Box2D) will not render');
+ expect(gaps).toHaveLength(1);
expect(gaps.join('\n')).toContain('Spine animation will not render');
expect(gaps.join('\n')).toContain('scenes/main.esscene');
- // What the native core DOES compile is not warned about: text (the host
- // rasterizes glyphs for the same atlas), tilemaps, particles and
- // post-processing. The gaps are what the build really cannot do, not
- // everything that happens to be optional.
+ // What the native build DOES have is not warned about: text (the host
+ // rasterizes glyphs for the same atlas), tilemaps, particles,
+ // post-processing, and physics (Box2D compiles into the host binary). The
+ // gaps are what the build really cannot do, not everything optional.
expect(gaps.join('\n')).not.toContain('Text will not render');
expect(gaps.join('\n')).not.toContain('Tilemaps will not render');
expect(gaps.join('\n')).not.toContain('Particles will not render');
+ expect(gaps.join('\n')).not.toContain('Physics (Box2D) will not render');
}, 60_000);
it('stays quiet for a target that renders all of it', async () => {
diff --git a/sdk/src/video/PlatformVideoBackend.ts b/sdk/src/video/PlatformVideoBackend.ts
index fe918b56e..ed6c81542 100644
--- a/sdk/src/video/PlatformVideoBackend.ts
+++ b/sdk/src/video/PlatformVideoBackend.ts
@@ -26,7 +26,9 @@ export interface VideoStreamHandle {
setPlaybackRate(rate: number): void;
/** Upload the current frame if a new one arrived; called once per frame. */
- pump(module: ESEngineModule): void;
+ /** @param module the wasm engine module, or null on a core with no heap —
+ * the texture upload takes bytes directly there. */
+ pump(module: ESEngineModule | null): void;
onReady?: () => void;
onEnded?: () => void;
diff --git a/sdk/src/video/VideoAPI.ts b/sdk/src/video/VideoAPI.ts
index c538fb362..0b000c2a7 100644
--- a/sdk/src/video/VideoAPI.ts
+++ b/sdk/src/video/VideoAPI.ts
@@ -57,7 +57,7 @@ export class VideoAPI {
return resolved !== siblingRef ? resolved : undefined;
}
- update(module: ESEngineModule): void {
+ update(module: ESEngineModule | null): void {
if (this.disposed_) return;
for (const handle of this.handles_) {
// One stream's decode failure (wasm abort, detached heap) must not
diff --git a/sdk/src/video/VideoPlugin.ts b/sdk/src/video/VideoPlugin.ts
index d10894ece..e14b84338 100644
--- a/sdk/src/video/VideoPlugin.ts
+++ b/sdk/src/video/VideoPlugin.ts
@@ -100,8 +100,9 @@ export class VideoPlugin implements Plugin {
(_time: TimeData, videoAPI: VideoAPI) => {
if (isEditor() && !isPlayMode()) return;
const world = app.world;
+ // Null on a native core: the frame upload goes through the
+ // ResourceManager's byte path instead of a wasm heap.
const module = world.getWasmModule();
- if (!module) return;
const entities = world.getEntitiesWithComponents([Video]);
live.clear();
diff --git a/sdk/src/video/WasmVideoBackend.ts b/sdk/src/video/WasmVideoBackend.ts
index 0d02864ee..3d024f364 100644
--- a/sdk/src/video/WasmVideoBackend.ts
+++ b/sdk/src/video/WasmVideoBackend.ts
@@ -132,8 +132,15 @@ class WasmVideoStreamHandle implements VideoStreamHandle {
const ptr = video._malloc(data.length);
if (!ptr) throw new Error('videodec out of memory');
video.HEAPU8.set(data, ptr);
- // _es_video_open owns `ptr` from here (freed on close or on failure).
- const decoder = video._es_video_open(ptr, data.length);
+ // The decoder copies the bytes into its own memory, so this buffer is
+ // ours to release either way — ownership does not cross the boundary
+ // (the two sides share an allocator on the web and not on a device).
+ let decoder = 0;
+ try {
+ decoder = video._es_video_open(ptr, data.length);
+ } finally {
+ video._free(ptr);
+ }
if (!decoder) {
throw new Error(
`not a decodable MPEG-1 stream (or no free decoder slot): ${url} — ` +
@@ -266,7 +273,7 @@ class WasmVideoStreamHandle implements VideoStreamHandle {
// === texture pump =======================================================
- pump(module: ESEngineModule): void {
+ pump(module: ESEngineModule | null): void {
const video = this.mod_;
if (this.disposed_ || !video) return;
diff --git a/sdk/src/video/WebVideoBackend.ts b/sdk/src/video/WebVideoBackend.ts
index 7142b4919..4cb73dc4b 100644
--- a/sdk/src/video/WebVideoBackend.ts
+++ b/sdk/src/video/WebVideoBackend.ts
@@ -109,7 +109,10 @@ class WebVideoStreamHandle implements VideoStreamHandle {
// === texture pump =======================================================
- pump(module: ESEngineModule): void {
+ pump(module: ESEngineModule | null): void {
+ // This backend IS the browser's <video> element, so it only ever runs on a
+ // realm that has the wasm module (it samples through its GL context).
+ if (!module) return;
if (this.disposed_ || this.width_ <= 0 || this.height_ <= 0) return;
if (this.video_.readyState < 2) return; // HAVE_CURRENT_DATA — a frame to sample
diff --git a/src/esengine/bindings/VideoBindings.hpp b/src/esengine/bindings/VideoBindings.hpp
new file mode 100644
index 000000000..4ac32badc
--- /dev/null
+++ b/src/esengine/bindings/VideoBindings.hpp
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file VideoBindings.hpp
+ * @brief The MPEG-1 decoder's entry points, declared once for both platforms.
+ * @details The engine owns its video decode (pl_mpeg) rather than depending on a
+ * platform's media stack, which is what makes a cooked `.esv` play the same
+ * way everywhere. On the web it links as a wasm side module; a native app
+ * compiles the same TU into the host binary and reaches it through the
+ * QuickJS wrappers EHT generates from these declarations.
+ *
+ * Bulk data crosses as an offset into the caller's heap (wasm linear memory
+ * on the web, the host arena on a device): the source bytes in, the decoded
+ * RGBA frame out. Nothing owns anything across the boundary — see
+ * `es_video_open`.
+ *
+ * @author ESEngine Team
+ * @date 2026
+ *
+ * @copyright Copyright (c) 2026 ESEngine Team
+ * Licensed under the Apache License, Version 2.0.
+ */
+#pragma once
+
+#include <cstdint>
+
+extern "C" {
+
+/** Open a stream from `size` bytes at `dataPtr`; the bytes are copied, so the caller
+ * keeps its buffer. Returns a handle >= 1, or 0. */
+int es_video_open(uintptr_t dataPtr, uint32_t size);
+/** Release the instance and everything it decoded. Safe on a stale handle. */
+void es_video_close(int handle);
+
+uint32_t es_video_width(int handle);
+uint32_t es_video_height(int handle);
+double es_video_duration(int handle);
+double es_video_framerate(int handle);
+/** Presentation time of the last decoded frame, in seconds. */
+double es_video_time(int handle);
+void es_video_set_loop(int handle, int loop);
+int es_video_has_ended(int handle);
+
+/** Decode forward by `dt` seconds; returns 1 when a new frame became available. */
+int es_video_advance(int handle, double dt);
+/** Write the pending frame as RGBA into `outPtr` (>= width*height*4 bytes). */
+int es_video_frame_rgba(int handle, uintptr_t outPtr, uint32_t outSize);
+/** Exact-seek to `time` seconds, leaving the target frame pending. */
+int es_video_seek(int handle, double time);
+
+} // extern "C"
diff --git a/src/esengine/bindings/VideoModuleEntry.cpp b/src/esengine/bindings/VideoModuleEntry.cpp
index 90af98fbf..02c1cfeae 100644
--- a/src/esengine/bindings/VideoModuleEntry.cpp
+++ b/src/esengine/bindings/VideoModuleEntry.cpp
@@ -51,22 +51,27 @@ void onVideoFrame(plm_t*, plm_frame_t* frame, void* user) {
extern "C" {
/**
- * Open an MPEG-PS / MPEG-1 stream. Takes ownership of `data` (allocated in this
- * module's heap via _malloc); it is freed on close — and on failure — so the
- * caller never frees it. Returns a handle >= 1, or 0 on failure.
+ * Open an MPEG-PS / MPEG-1 stream from `dataPtr` (`size` bytes in the caller's heap).
+ * The bytes are COPIED into memory this module owns, so the caller keeps and frees
+ * its own buffer: ownership must not cross the boundary, because the two sides do not
+ * necessarily share an allocator. They do in the wasm build — one emscripten heap —
+ * and they do not in a native app, where the caller writes into the host's arena
+ * (host/heap.hpp) and this module's `free` is libc's. Returns a handle >= 1, or 0.
*/
-int es_video_open(uint8_t* data, uint32_t size) {
+int es_video_open(uintptr_t dataPtr, uint32_t size) {
+ const auto* source = reinterpret_cast<const uint8_t*>(dataPtr);
+ if (!source || size == 0) return 0;
int slot = -1;
for (int i = 0; i < kMaxInstances; i++) {
if (!g_instances[i].plm) { slot = i; break; }
}
- if (slot < 0) {
- free(data);
- return 0;
- }
- plm_t* plm = plm_create_with_memory(data, size, /* free_when_done */ TRUE);
+ if (slot < 0) return 0;
+ auto* owned = static_cast<uint8_t*>(malloc(size));
+ if (!owned) return 0;
+ memcpy(owned, source, size);
+ plm_t* plm = plm_create_with_memory(owned, size, /* free_when_done */ TRUE);
if (!plm) {
- free(data);
+ free(owned);
return 0;
}
// Validate via the parsed sequence header, not plm_probe(): probe scans
@@ -138,11 +143,13 @@ int es_video_advance(int handle, double dt) {
/**
* Convert the latest decoded frame into `out` as bottom-first RGBA rows — the
* orientation the engine's flipY-off texture upload samples upright. `outSize`
- * must be at least width*height*4. Returns 1 and clears the new-frame flag on
- * success.
+ * must be at least width*height*4. `outPtr` is an offset the caller wrote — wasm
+ * linear memory on the web, the host arena on a device — so the pixels land where
+ * the caller reads them. Returns 1 and clears the new-frame flag on success.
*/
-int es_video_frame_rgba(int handle, uint8_t* out, uint32_t outSize) {
+int es_video_frame_rgba(int handle, uintptr_t outPtr, uint32_t outSize) {
Instance* inst = get(handle);
+ auto* out = reinterpret_cast<uint8_t*>(outPtr);
if (!inst || !inst->frame || !out) return 0;
const int w = plm_get_width(inst->plm);
const int h = plm_get_height(inst->plm);
diff --git a/tools/eht/generators/native_functions.py b/tools/eht/generators/native_functions.py
index af2586337..f19f199d3 100644
--- a/tools/eht/generators/native_functions.py
+++ b/tools/eht/generators/native_functions.py
@@ -262,10 +262,21 @@ class NativeFunctionsGenerator:
self.skipped: List[Skipped] = []
self.emitted: List[str] = []
+ @staticmethod
+ def _js_name(name: str) -> str:
+ """The global the wrapper is registered under.
+
+ `es_` marks the boundary surface, so a C name that already carries it keeps
+ it — the video module's entry points are spelled `es_video_open`, and
+ `es_es_video_open` would read as a mistake on both sides of the boundary.
+ """
+ return name if name.startswith('es_') else f'es_{name}'
+
def _wrapper(self, fn: Function) -> List[str]:
js_params = [p for p in fn.params if p.cpp_type not in _IMPLICIT_REFS]
out = [f'#ifdef {g}' for g in fn.guards]
- out.append(f'static JSValue es_{fn.name}(JSContext* ctx, JSValueConst, int argc, JSValueConst* argv) {{')
+ out.append(f'static JSValue {self._js_name(fn.name)}'
+ f'(JSContext* ctx, JSValueConst, int argc, JSValueConst* argv) {{')
if js_params:
out.append(f' if (argc < {len(js_params)}) return JS_UNDEFINED;')
out.append(' (void)argc; (void)argv;')
@@ -418,7 +429,8 @@ class NativeFunctionsGenerator:
])
for fn in functions:
takes_implicit = any(p.cpp_type in _IMPLICIT_REFS for p in fn.params)
- lines.append(f" bind('{fn.name}', 'es_{fn.name}', {str(takes_implicit).lower()});")
+ lines.append(f" bind('{fn.name}', '{self._js_name(fn.name)}', "
+ f"{str(takes_implicit).lower()});")
lines.extend([
' return api as NativeEngineApi;',
'}',
@@ -462,8 +474,9 @@ class NativeFunctionsGenerator:
for fn in sorted(functions, key=lambda f: f.name):
js_arity = len([p for p in fn.params if p.cpp_type not in _IMPLICIT_REFS])
lines.extend(f'#ifdef {g}' for g in fn.guards)
- lines.append(f' JS_SetPropertyStr(ctx, global, "es_{fn.name}", '
- f'JS_NewCFunction(ctx, es_{fn.name}, "es_{fn.name}", {js_arity}));')
+ js = self._js_name(fn.name)
+ lines.append(f' JS_SetPropertyStr(ctx, global, "{js}", '
+ f'JS_NewCFunction(ctx, {js}, "{js}", {js_arity}));')
lines.extend('#endif' for _ in fn.guards)
lines.append('}')
lines.append('')