A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
perf(video): zero-copy texImage2D(video) upload on WebGL2 (M3)
yhhf805948242db
diff --git a/sdk/src/video/WebVideoBackend.ts b/sdk/src/video/WebVideoBackend.tsindex 9d05831ba..7142b4919 100644--- a/sdk/src/video/WebVideoBackend.ts+++ b/sdk/src/video/WebVideoBackend.ts@@ -1,11 +1,14 @@// SPDX-License-Identifier: Apache-2.0// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team-// Browser / Electron video backend: an HTMLVideoElement streams and decodes,-// each frame is drawn to a canvas and uploaded into a pathless engine texture.+// Browser / Electron video backend. An HTMLVideoElement streams and decodes;+// on WebGL2 each frame is texImage2D'd straight onto the engine context+// (zero-copy), else a canvas readback feeds updateTextureSubregion (WebGPU).import type { ESEngineModule } from '../wasm';import type { PlatformVideoBackend, VideoStreamHandle, VideoStreamOptions } from './PlatformVideoBackend';import { createTextureFromPixels, updateTextureSubregion } from '../runtimeAssets';import { requireResourceManager } from '../resourceManager';+import { findWebGL2Context } from '../asset/loaders/TextureLoader';+import { linearColorSpace } from '../env';import { log } from '../logger';let nextId_ = 1;@@ -19,6 +22,13 @@ class WebVideoStreamHandle implements VideoStreamHandle {onError?: (error: unknown) => void;private readonly video_: HTMLVideoElement;+ // Zero-copy GL path (WebGL2).+ private gl_: WebGL2RenderingContext | null = null;+ private glTexture_: WebGLTexture | null = null;+ private glTexId_ = 0;+ private glPool_: Record<number, WebGLTexture> | null = null;+ private pathChosen_ = false;+ // CPU fallback path (WebGPU / no GL context).private canvas_: HTMLCanvasElement | null = null;private ctx_: CanvasRenderingContext2D | null = null;private texture_ = 0;@@ -108,29 +118,73 @@ class WebVideoStreamHandle implements VideoStreamHandle {const advanced = this.video_.currentTime !== this.lastUploadTime_;if (this.ready_ && !this.newFrame_ && !advanced) return;+ // First ready frame picks the path: WebGL2 → zero-copy, else CPU readback.+ if (!this.pathChosen_) {+ this.gl_ = findWebGL2Context(module.GL);+ this.pathChosen_ = true;+ }++ try {+ if (this.gl_) this.uploadGL_(module, this.gl_);+ else this.uploadCPU_(module);+ } catch (err) {+ // Tainted source (cross-origin without CORS) or a lost context.+ this.onError?.(err);+ this.disposed_ = true;+ return;+ }++ this.newFrame_ = false;+ this.lastUploadTime_ = this.video_.currentTime;+ if (!this.ready_) {+ this.ready_ = true;+ this.onReady?.();+ }+ }++ // Zero-copy: texImage2D the frame straight onto the engine's GL context, then+ // register it as an external texture. UNPACK_FLIP_Y gives the bottom-first+ // orientation a Sprite samples upright (matching image uploads).+ private uploadGL_(module: ESEngineModule, gl: WebGL2RenderingContext): void {+ if (this.glTexture_) {+ gl.bindTexture(gl.TEXTURE_2D, this.glTexture_);+ gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 1);+ gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, gl.RGBA, gl.UNSIGNED_BYTE, this.video_);+ gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 0);+ return;+ }+ const tex = gl.createTexture();+ if (!tex) throw new Error('gl.createTexture failed');+ this.glTexture_ = tex;+ gl.bindTexture(gl.TEXTURE_2D, tex);+ gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 1);+ const internal = linearColorSpace() ? gl.SRGB8_ALPHA8 : gl.RGBA;+ gl.texImage2D(gl.TEXTURE_2D, 0, internal, gl.RGBA, gl.UNSIGNED_BYTE, this.video_);+ gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, 0);+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);+ const pool = module.GL.textures;+ const id = module.GL.getNewId(pool);+ pool[id] = tex;+ this.glTexId_ = id;+ this.glPool_ = pool;+ this.texture_ = requireResourceManager().registerExternalTexture(id, this.width_, this.height_);+ }++ private uploadCPU_(module: ESEngineModule): void {this.ensureCanvas_();const ctx = this.ctx_;if (!ctx) return;-- // Draw flipped so getImageData yields bottom-first rows — the orientation- // updateTextureSubregion (flipY-off) needs for a Sprite to sample upright.+ // Flip so getImageData yields bottom-first rows (updateTextureSubregion is flipY-off).ctx.save();ctx.translate(0, this.height_);ctx.scale(1, -1);ctx.drawImage(this.video_ as CanvasImageSource, 0, 0, this.width_, this.height_);ctx.restore();-- let frame: ImageData;- try {- frame = ctx.getImageData(0, 0, this.width_, this.height_);- } catch (err) {- // Tainted canvas (cross-origin without CORS) — give up on readback.- this.onError?.(err);- this.disposed_ = true;- return;- }+ const frame = ctx.getImageData(0, 0, this.width_, this.height_);const pixels = new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength);-if (!this.texture_) {this.texture_ = createTextureFromPixels(module,@@ -141,13 +195,6 @@ class WebVideoStreamHandle implements VideoStreamHandle {} else {updateTextureSubregion(module, this.texture_, 0, 0, this.width_, this.height_, pixels);}-- this.newFrame_ = false;- this.lastUploadTime_ = this.video_.currentTime;- if (!this.ready_) {- this.ready_ = true;- this.onReady?.();- }}// === transport ==========================================================@@ -187,6 +234,13 @@ class WebVideoStreamHandle implements VideoStreamHandle {requireResourceManager().releaseTexture(this.texture_);this.texture_ = 0;}+ // The engine's external texture is non-owning (owns_=false), so delete the+ // JS-side GL texture ourselves after dropping the engine ref.+ if (this.gl_ && this.glTexture_) {+ this.gl_.deleteTexture(this.glTexture_);+ if (this.glPool_ && this.glTexId_) delete this.glPool_[this.glTexId_];+ this.glTexture_ = null;+ }this.canvas_ = null;this.ctx_ = null;}