Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

perf(video): zero-copy texImage2D(video) upload on WebGL2 (M3)

yhhf805948242db
diff --git a/sdk/src/video/WebVideoBackend.ts b/sdk/src/video/WebVideoBackend.ts
index 9d05831ba..7142b4919 100644
--- a/sdk/src/video/WebVideoBackend.ts
+++ b/sdk/src/video/WebVideoBackend.ts
@@ -1,11 +1,14 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
-// Browser / Electron video backend: an HTMLVideoElement streams and decodes,
-// each frame is drawn to a canvas and uploaded into a pathless engine texture.
+// Browser / Electron video backend. An HTMLVideoElement streams and decodes;
+// on WebGL2 each frame is texImage2D'd straight onto the engine context
+// (zero-copy), else a canvas readback feeds updateTextureSubregion (WebGPU).
import type { ESEngineModule } from '../wasm';
import type { PlatformVideoBackend, VideoStreamHandle, VideoStreamOptions } from './PlatformVideoBackend';
import { createTextureFromPixels, updateTextureSubregion } from '../runtimeAssets';
import { requireResourceManager } from '../resourceManager';
+import { findWebGL2Context } from '../asset/loaders/TextureLoader';
+import { linearColorSpace } from '../env';
import { log } from '../logger';
let nextId_ = 1;
@@ -19,6 +22,13 @@ class WebVideoStreamHandle implements VideoStreamHandle {
onError?: (error: unknown) => void;
private readonly video_: HTMLVideoElement;
+ // Zero-copy GL path (WebGL2).
+ private gl_: WebGL2RenderingContext | null = null;
+ private glTexture_: WebGLTexture | null = null;
+ private glTexId_ = 0;
+ private glPool_: Record<number, WebGLTexture> | null = null;
+ private pathChosen_ = false;
+ // CPU fallback path (WebGPU / no GL context).
private canvas_: HTMLCanvasElement | null = null;
private ctx_: CanvasRenderingContext2D | null = null;
private texture_ = 0;
@@ -108,29 +118,73 @@ class WebVideoStreamHandle implements VideoStreamHandle {
const advanced = this.video_.currentTime !== this.lastUploadTime_;
if (this.ready_ && !this.newFrame_ && !advanced) return;
+ // First ready frame picks the path: WebGL2 → zero-copy, else CPU readback.
+ if (!this.pathChosen_) {
+ this.gl_ = findWebGL2Context(module.GL);
+ this.pathChosen_ = true;
+ }
+
+ try {
+ if (this.gl_) this.uploadGL_(module, this.gl_);
+ else this.uploadCPU_(module);
+ } catch (err) {
+ // Tainted source (cross-origin without CORS) or a lost context.
+ this.onError?.(err);
+ this.disposed_ = true;
+ return;
+ }
+
+ this.newFrame_ = false;
+ this.lastUploadTime_ = this.video_.currentTime;
+ if (!this.ready_) {
+ this.ready_ = true;
+ this.onReady?.();
+ }
+ }
+
+ // Zero-copy: texImage2D the frame straight onto the engine's GL context, then
+ // register it as an external texture. UNPACK_FLIP_Y gives the bottom-first
+ // orientation a Sprite samples upright (matching image uploads).
+ private uploadGL_(module: ESEngineModule, gl: WebGL2RenderingContext): void {
+ if (this.glTexture_) {
+ gl.bindTexture(gl.TEXTURE_2D, this.glTexture_);
+ gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 1);
+ gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, gl.RGBA, gl.UNSIGNED_BYTE, this.video_);
+ gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 0);
+ return;
+ }
+ const tex = gl.createTexture();
+ if (!tex) throw new Error('gl.createTexture failed');
+ this.glTexture_ = tex;
+ gl.bindTexture(gl.TEXTURE_2D, tex);
+ gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 1);
+ const internal = linearColorSpace() ? gl.SRGB8_ALPHA8 : gl.RGBA;
+ gl.texImage2D(gl.TEXTURE_2D, 0, internal, gl.RGBA, gl.UNSIGNED_BYTE, this.video_);
+ gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 0);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+ const pool = module.GL.textures;
+ const id = module.GL.getNewId(pool);
+ pool[id] = tex;
+ this.glTexId_ = id;
+ this.glPool_ = pool;
+ this.texture_ = requireResourceManager().registerExternalTexture(id, this.width_, this.height_);
+ }
+
+ private uploadCPU_(module: ESEngineModule): void {
this.ensureCanvas_();
const ctx = this.ctx_;
if (!ctx) return;
-
- // Draw flipped so getImageData yields bottom-first rows — the orientation
- // updateTextureSubregion (flipY-off) needs for a Sprite to sample upright.
+ // Flip so getImageData yields bottom-first rows (updateTextureSubregion is flipY-off).
ctx.save();
ctx.translate(0, this.height_);
ctx.scale(1, -1);
ctx.drawImage(this.video_ as CanvasImageSource, 0, 0, this.width_, this.height_);
ctx.restore();
-
- let frame: ImageData;
- try {
- frame = ctx.getImageData(0, 0, this.width_, this.height_);
- } catch (err) {
- // Tainted canvas (cross-origin without CORS) — give up on readback.
- this.onError?.(err);
- this.disposed_ = true;
- return;
- }
+ const frame = ctx.getImageData(0, 0, this.width_, this.height_);
const pixels = new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength);
-
if (!this.texture_) {
this.texture_ = createTextureFromPixels(
module,
@@ -141,13 +195,6 @@ class WebVideoStreamHandle implements VideoStreamHandle {
} else {
updateTextureSubregion(module, this.texture_, 0, 0, this.width_, this.height_, pixels);
}
-
- this.newFrame_ = false;
- this.lastUploadTime_ = this.video_.currentTime;
- if (!this.ready_) {
- this.ready_ = true;
- this.onReady?.();
- }
}
// === transport ==========================================================
@@ -187,6 +234,13 @@ class WebVideoStreamHandle implements VideoStreamHandle {
requireResourceManager().releaseTexture(this.texture_);
this.texture_ = 0;
}
+ // The engine's external texture is non-owning (owns_=false), so delete the
+ // JS-side GL texture ourselves after dropping the engine ref.
+ if (this.gl_ && this.glTexture_) {
+ this.gl_.deleteTexture(this.glTexture_);
+ if (this.glPool_ && this.glTexId_) delete this.glPool_[this.glTexId_];
+ this.glTexture_ = null;
+ }
this.canvas_ = null;
this.ctx_ = null;
}