Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(video): WeChat backend (M2) + type WeChat backends against official wx API

yhh5cb671e33ee5
diff --git a/desktop/electron/main.ts b/desktop/electron/main.ts
index 58504054c..f0bff0ab5 100644
--- a/desktop/electron/main.ts
+++ b/desktop/electron/main.ts
@@ -715,10 +715,8 @@ ipcMain.handle('project:saveDialog', async (_e, defaultRel?: string) => {
return rel;
});
-// Serve a project file, honoring an HTTP Range request (206) when present.
-// <video> streams and seeks via Range; every other client omits the header and
-// gets the whole file (200). Ranged reads pull only the requested slice off
-// disk, so seeking a large clip never loads it whole into memory.
+// Serve a project file, honoring an HTTP Range request (206) when present so
+// <video> streams/seeks; ranged reads pull only the requested slice off disk.
async function serveProjectFile(abs: string, request: Request): Promise<Response> {
const headers: Record<string, string> = {
'content-type': httpContentType(abs),
diff --git a/sdk/src/audio/WeChatAudioBackend.ts b/sdk/src/audio/WeChatAudioBackend.ts
index 3472011af..c1c74b301 100644
--- a/sdk/src/audio/WeChatAudioBackend.ts
+++ b/sdk/src/audio/WeChatAudioBackend.ts
@@ -1,38 +1,18 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/// <reference types="minigame-api-typings" />
import type { AudioHandle, AudioBufferHandle, PlayConfig, PlatformAudioBackend, AudioBackendInitOptions } from './PlatformAudioBackend';
import type { AudioMixer } from './AudioMixer';
import { log } from '../logger';
-interface WxInnerAudioContext {
- src: string;
- startTime: number;
- autoplay: boolean;
- loop: boolean;
- obeyMuteSwitch: boolean;
- volume: number;
- playbackRate: number;
- duration: number;
- currentTime: number;
- paused: boolean;
- play(): void;
- pause(): void;
- stop(): void;
- seek(position: number): void;
- destroy(): void;
- onEnded(callback: () => void): void;
- offEnded(callback: () => void): void;
- onError(callback: (res: { errMsg: string }) => void): void;
-}
-
class WeChatAudioHandle implements AudioHandle {
readonly id: number;
onEnd?: () => void;
- private ctx_: WxInnerAudioContext;
- private contexts_: Map<number, WxInnerAudioContext>;
+ private ctx_: WechatMinigame.InnerAudioContext;
+ private contexts_: Map<number, WechatMinigame.InnerAudioContext>;
- constructor(id: number, ctx: WxInnerAudioContext, contexts: Map<number, WxInnerAudioContext>) {
+ constructor(id: number, ctx: WechatMinigame.InnerAudioContext, contexts: Map<number, WechatMinigame.InnerAudioContext>) {
this.id = id;
this.ctx_ = ctx;
this.contexts_ = contexts;
@@ -89,7 +69,7 @@ class WeChatAudioHandle implements AudioHandle {
export class WeChatAudioBackend implements PlatformAudioBackend {
readonly name = 'WeChat';
- private contexts_ = new Map<number, WxInnerAudioContext>();
+ private contexts_ = new Map<number, WechatMinigame.InnerAudioContext>();
private urlCache_ = new Map<number, string>();
private nextId_ = 0;
@@ -129,7 +109,7 @@ export class WeChatAudioBackend implements PlatformAudioBackend {
throw new Error(`Buffer ${buffer.id} not found`);
}
- const ctx: WxInnerAudioContext = (wx as any).createInnerAudioContext();
+ const ctx: WechatMinigame.InnerAudioContext = wx.createInnerAudioContext();
ctx.loop = config.loop ?? false;
ctx.volume = config.volume ?? 1.0;
ctx.playbackRate = config.playbackRate ?? 1.0;
diff --git a/sdk/src/platform/wechat/index.ts b/sdk/src/platform/wechat/index.ts
index 7d4897890..320079a75 100644
--- a/sdk/src/platform/wechat/index.ts
+++ b/sdk/src/platform/wechat/index.ts
@@ -20,6 +20,8 @@ import type {
import { WeChatSocket } from '../../net/WeChatSocket';
import { WeChatAudioBackend } from '../../audio/WeChatAudioBackend';
import type { PlatformAudioBackend } from '../../audio/PlatformAudioBackend';
+import { WeChatVideoBackend } from '../../video/WeChatVideoBackend';
+import type { PlatformVideoBackend } from '../../video/PlatformVideoBackend';
import { wxFetch, polyfillFetch } from './fetch';
import { wxInstantiateWasm, polyfillWebAssembly } from './wasm';
import { wxReadFileSync, wxReadTextFileSync, wxFileExistsSync } from './fs';
@@ -173,6 +175,10 @@ class WeChatPlatformAdapter implements PlatformAdapter {
return new WeChatAudioBackend();
}
+ createVideoBackend(): PlatformVideoBackend {
+ return new WeChatVideoBackend();
+ }
+
createSocket(options: PlatformSocketOptions): PlatformSocket {
return new WeChatSocket(options);
}
diff --git a/sdk/src/video/NullVideoBackend.ts b/sdk/src/video/NullVideoBackend.ts
index 8ec7b5e4c..e7227b603 100644
--- a/sdk/src/video/NullVideoBackend.ts
+++ b/sdk/src/video/NullVideoBackend.ts
@@ -1,14 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
-/**
- * @file video/NullVideoBackend.ts
- * @brief Silent no-op video backend — the visual mirror of NullAudioBackend.
- * Used on headless Node (authoritative server), as the WeChat fallback
- * until WeChatVideoBackend lands, and anywhere a platform advertises no
- * video capability. Every call succeeds and does nothing; the frame
- * texture stays 0 so a driven Sprite simply shows no video. DOM-free,
- * so importing it never pulls browser globals into a headless bundle.
- */
+// Silent no-op video backend — headless Node and the fallback when a platform
+// has no video capability. textureHandle stays 0; DOM-free.
import type { PlatformVideoBackend, VideoStreamHandle, VideoStreamOptions } from './PlatformVideoBackend';
let nextId_ = 1;
diff --git a/sdk/src/video/PlatformVideoBackend.ts b/sdk/src/video/PlatformVideoBackend.ts
index 4a345e19b..6c82fde36 100644
--- a/sdk/src/video/PlatformVideoBackend.ts
+++ b/sdk/src/video/PlatformVideoBackend.ts
@@ -1,49 +1,23 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
-/**
- * @file video/PlatformVideoBackend.ts
- * @brief Platform-neutral video contract — the visual-side mirror of
- * {@link ../audio/PlatformAudioBackend}. A backend turns a source
- * reference into a {@link VideoStreamHandle}: a live decode that
- * exposes an engine texture handle, updated in place each frame from
- * the current decoded frame. Web decodes with an HTMLVideoElement,
- * WeChat with wx.createVideoDecoder, headless with a no-op — the
- * rest of the engine only ever sees this interface.
- */
+// Platform-neutral video contract. A backend turns a source ref into a live
+// decode exposing an engine texture handle, refreshed each frame.
import type { ESEngineModule } from '../wasm';
-/**
- * A live video decode. Unlike audio (where a decoded buffer and a playing
- * voice are separate handles), a video stream IS both the resource and the
- * transport, because a frame is only meaningful while decoding.
- *
- * `textureHandle` is the engine texture the current frame lands in — a
- * pathless, non-evictable texture the handle owns for its lifetime (created
- * once dimensions are known). It reads `0` until the first frame is ready;
- * callers sample it through the normal Sprite/Mesh2D texture slot.
- */
export interface VideoStreamHandle {
readonly id: number;
- /** Engine texture handle for the current frame; `0` until `isReady`. */
+ /** Frame texture handle; 0 until isReady. Pathless (never evicted). */
readonly textureHandle: number;
- /** Native frame size in pixels; `0` until `isReady`. */
readonly width: number;
readonly height: number;
- /**
- * VRAM bytes of the frame texture, for a future residency budget. Backends
- * that own no sampleable texture (native overlay, headless) omit it —
- * 0/undefined means untracked, mirroring `AudioBufferHandle.bytes`.
- */
readonly bytes?: number;
readonly isReady: boolean;
readonly isPlaying: boolean;
- /** Playback position / total length in seconds (0 when unknown). */
readonly currentTime: number;
readonly duration: number;
play(): void;
pause(): void;
- /** Stop and release the decode + frame texture. Idempotent. */
stop(): void;
seek(timeSeconds: number): void;
setVolume(volume: number): void;
@@ -51,17 +25,10 @@ export interface VideoStreamHandle {
setLoop(loop: boolean): void;
setPlaybackRate(rate: number): void;
- /**
- * Upload the current decoded frame into `textureHandle` when a new one is
- * available (creating the texture on the first ready frame). Called once
- * per frame by the video system with the live wasm module. Cheap to call
- * when no new frame has arrived.
- */
+ /** Upload the current frame if a new one arrived; called once per frame. */
pump(module: ESEngineModule): void;
- /** Fires once the first frame + dimensions are ready (texture now valid). */
onReady?: () => void;
- /** Fires when a non-looping stream reaches the end. */
onEnded?: () => void;
onError?: (error: unknown) => void;
}
@@ -70,17 +37,10 @@ export interface VideoStreamOptions {
autoplay?: boolean;
loop?: boolean;
muted?: boolean;
- /** 0..1, applied to the stream's own audio track. */
volume?: number;
playbackRate?: number;
}
-/**
- * The platform video factory. One `createStream` per playing video; the
- * returned handle owns its decode and frame texture. Selected through the
- * platform adapter (`getPlatform().createVideoBackend?.()`), never imported
- * directly — that keeps DOM/wx code out of the wrong bundles.
- */
export interface PlatformVideoBackend {
readonly name: string;
createStream(url: string, options: VideoStreamOptions): VideoStreamHandle;
diff --git a/sdk/src/video/VideoAPI.ts b/sdk/src/video/VideoAPI.ts
index 9053bfe3f..d73e91e57 100644
--- a/sdk/src/video/VideoAPI.ts
+++ b/sdk/src/video/VideoAPI.ts
@@ -1,18 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
-/**
- * @file video/VideoAPI.ts
- * @brief Per-app video service — the imperative face of the video system and
- * the visual mirror of AudioAPI. Code-driven playback (cutscenes,
- * splash) calls `video.play(src)`; the declarative `Video` component's
- * system uses the same API under the hood. Each `App` owns one
- * instance, created by `VideoPlugin.build()` and exposed as the
- * {@link VideoPlayer} resource (`Res(VideoPlayer)` / `getResource`).
- *
- * Video texture residency is simpler than audio's: a stream owns one pathless,
- * non-evictable frame texture for its lifetime and frees it on stop — there is
- * no re-fetchable decoded buffer to warm-cache, so there is no eviction budget.
- */
+// Per-app video service — the imperative face of the video system, exposed as
+// the VideoPlayer resource.
import type { ESEngineModule } from '../wasm';
import type { PlatformVideoBackend, VideoStreamHandle, VideoStreamOptions } from './PlatformVideoBackend';
import { defineResource } from '../resource';
@@ -25,7 +14,6 @@ export class VideoAPI {
private readonly handles_ = new Set<VideoStreamHandle>();
private refResolver_: ((ref: string) => string) | null = null;
private disposed_ = false;
- /** Legacy prefix for plain project-relative refs; `refResolver` wins. */
baseUrl = '';
constructor(backend: PlatformVideoBackend) {
@@ -34,50 +22,29 @@ export class VideoAPI {
get backendName(): string { return this.backend_.name; }
- /**
- * Route source refs through the realm's asset resolver — the same channel
- * every other asset resolves through (uuid manifest, cooked logical→staged
- * map, project base). Mirrors {@link AudioAPI.setRefResolver}.
- */
setRefResolver(resolver: ((ref: string) => string) | null): void {
this.refResolver_ = resolver;
}
private resolveUrl_(ref: string): string {
if (this.refResolver_) return this.refResolver_(ref);
- // Absolute or scheme-bearing (estella://, blob:, data:, http(s)://,
- // leading /) — already a URL the element can open.
if (!this.baseUrl || ref.includes('://') || ref.startsWith('/') || ref.startsWith('blob:') || ref.startsWith('data:')) {
return ref;
}
return `${this.baseUrl}/${ref}`;
}
- /**
- * Start a video from a source ref, returning its live handle. The frame
- * texture is created lazily on the first decoded frame (see the handle's
- * `textureHandle`); poll `handle.isReady`/`onReady` before sampling it.
- */
play(source: string, options: VideoPlayOptions = {}): VideoStreamHandle {
const handle = this.backend_.createStream(this.resolveUrl_(source), options);
this.handles_.add(handle);
- const userEnded = handle.onEnded;
- handle.onEnded = () => {
- userEnded?.();
- // A finished non-looping stream stays in the set (its last frame is
- // still valid to show); explicit stop() releases it.
- };
return handle;
}
- /** Advance every live stream: upload any newly-decoded frame into its
- * texture. Called once per frame by the video system with the live module. */
update(module: ESEngineModule): void {
if (this.disposed_) return;
for (const handle of this.handles_) handle.pump(module);
}
- /** Stop one stream and release its frame texture. */
stop(handle: VideoStreamHandle): void {
if (this.handles_.delete(handle)) handle.stop();
}
@@ -95,8 +62,4 @@ export class VideoAPI {
}
}
-/**
- * The per-app video service resource. Consume via `Res(VideoPlayer)` in a
- * system or `app.getResource(VideoPlayer)` outside ECS code.
- */
export const VideoPlayer = defineResource<VideoAPI>(null!, 'VideoPlayer');
diff --git a/sdk/src/video/VideoComponents.ts b/sdk/src/video/VideoComponents.ts
index c7361c8fa..7f6fb953b 100644
--- a/sdk/src/video/VideoComponents.ts
+++ b/sdk/src/video/VideoComponents.ts
@@ -1,32 +1,17 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
-/**
- * @file video/VideoComponents.ts
- * @brief The declarative `Video` component — the editor/designer face of the
- * video system, mirroring how `AudioSource` fronts the audio API.
- *
- * A `Video` renders through its entity's `Sprite`: the video system decodes the
- * source into a live texture and writes that handle (and, when `fitSize`, the
- * native pixel size) into the sibling Sprite each frame. So a video is "a Sprite
- * whose texture is alive" and inherits all of Sprite's rendering — layer, tint,
- * material, lighting, flip, blend — for free. Add a `Sprite` alongside `Video`
- * (the editor's add-component does this automatically); without one the system
- * warns once and draws nothing.
- */
+// The declarative Video component. It renders through its entity's Sprite (the
+// system writes the live frame texture into it), so a Video needs a Sprite.
import { defineComponent } from '../component';
export interface VideoData {
- /** Project-relative path / asset ref of the video (.mp4/.webm). */
source: string;
- /** Begin playing as soon as the source is ready. */
autoplay: boolean;
loop: boolean;
muted: boolean;
- /** Volume of the video's own audio track, 0..1. */
volume: number;
playbackRate: number;
- /** On first ready frame, drive the sibling Sprite's `size` to the video's
- * native pixel dimensions. Off = keep the Sprite's authored size (stretch). */
+ /** On first frame, size the Sprite to the video's native pixels. */
fitSize: boolean;
enabled: boolean;
}
diff --git a/sdk/src/video/VideoPlugin.ts b/sdk/src/video/VideoPlugin.ts
index 4029c8a79..e01ff07b5 100644
--- a/sdk/src/video/VideoPlugin.ts
+++ b/sdk/src/video/VideoPlugin.ts
@@ -1,14 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
-/**
- * @file video/VideoPlugin.ts
- * @brief Wires the video system into an App — the visual mirror of AudioPlugin.
- * Builds the platform video backend, exposes it as the `VideoPlayer`
- * resource, and runs one Update-schedule system that: starts a stream
- * per playing `Video` entity, uploads decoded frames into their pathless
- * textures, and drives each entity's sibling `Sprite` from the current
- * frame. Runs only in play mode (editor edit-mode leaves videos paused).
- */
+// Wires the video system into an App: builds the backend, exposes VideoPlayer,
+// and runs one play-mode system that streams each Video and drives its Sprite.
import type { App, Plugin } from '../app';
import type { Entity } from '../types';
import { defineSystem, Schedule } from '../system';
@@ -28,15 +21,12 @@ export class VideoPlugin implements Plugin {
private handles_: Map<number, VideoStreamHandle> | null = null;
build(app: App): void {
- // `createVideoBackend` is an optional platform capability (like
- // createSocket/loadSubpackage): web provides a real one, headless/WeChat
- // fall back to the silent Null backend until their own lands.
+ // createVideoBackend is optional; fall back to the silent Null backend.
const backend = getPlatform().createVideoBackend?.() ?? new NullVideoBackend();
const video = new VideoAPI(backend);
this.video_ = video;
app.insertResource(VideoPlayer, video);
- // entity id → its component-driven stream.
const handles = new Map<number, VideoStreamHandle>();
this.handles_ = handles;
const warnedNoSprite = new Set<number>();
@@ -64,8 +54,7 @@ export class VideoPlugin implements Plugin {
const entities = world.getEntitiesWithComponents([Video]);
live.clear();
- // Ensure a stream per playing entity (before the pump below,
- // so a freshly-started stream uploads its first frame this tick).
+ // Start streams before the pump so a new one uploads this tick.
for (const entity of entities) {
const v = world.get(entity, Video) as VideoData;
if (!v.enabled || !v.source) continue;
@@ -82,10 +71,8 @@ export class VideoPlugin implements Plugin {
}
}
- // Upload any newly-decoded frames (component + imperative streams).
videoAPI.update(module);
- // Project the current frame onto each entity's sibling Sprite.
for (const entity of entities) {
const id = entity as number;
const handle = handles.get(id);
@@ -100,12 +87,8 @@ export class VideoPlugin implements Plugin {
continue;
}
- // Builtin components hand back a snapshot copy — mutating it
- // is discarded unless written back with `world.insert` (the
- // same pattern SpriteAnimator uses). The video's per-frame
- // pixels ride the SAME texture handle (updated in place), so
- // the Sprite only needs a write-back when the handle or size
- // actually changes — typically once, on the first ready frame.
+ // Builtin get() is a snapshot copy — needs world.insert to
+ // persist. The handle is stable, so write back only on change.
let dirty = false;
if (sprite.texture !== handle.textureHandle) {
sprite.texture = handle.textureHandle;
@@ -120,7 +103,7 @@ export class VideoPlugin implements Plugin {
if (dirty) world.insert(entity, Sprite, sprite);
}
- // Reap streams whose entity vanished, was disabled, or lost its source.
+ // Reap streams whose entity is gone, disabled, or source-cleared.
for (const [id, handle] of handles) {
if (!live.has(id)) {
videoAPI.stop(handle);
diff --git a/sdk/src/video/WeChatVideoBackend.ts b/sdk/src/video/WeChatVideoBackend.ts
new file mode 100644
index 000000000..3ed3755c8
--- /dev/null
+++ b/sdk/src/video/WeChatVideoBackend.ts
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+// WeChat video backend: wx.createVideoDecoder → getFrameData RGBA → the same
+// pathless texture as web, with wx.createMediaAudioPlayer for audio.
+/// <reference types="minigame-api-typings" />
+import type { ESEngineModule } from '../wasm';
+import type { PlatformVideoBackend, VideoStreamHandle, VideoStreamOptions } from './PlatformVideoBackend';
+import { createTextureFromPixels, updateTextureSubregion } from '../runtimeAssets';
+import { requireResourceManager } from '../resourceManager';
+import { log } from '../logger';
+
+let nextId_ = 1;
+const clamp01 = (v: number): number => (v < 0 ? 0 : v > 1 ? 1 : v);
+
+class WeChatVideoStreamHandle implements VideoStreamHandle {
+ readonly id = nextId_++;
+ onReady?: () => void;
+ onEnded?: () => void;
+ onError?: (error: unknown) => void;
+
+ private readonly decoder_: WechatMinigame.VideoDecoder;
+ private audioPlayer_: WechatMinigame.MediaAudioPlayer | null = null;
+ private texture_ = 0;
+ private width_ = 0;
+ private height_ = 0;
+ private ready_ = false;
+ private disposed_ = false;
+ private playing_: boolean;
+ private loop_: boolean;
+ private lastPts_ = -1;
+ private flip_: Uint8Array | null = null;
+ private rateWarned_ = false;
+
+ constructor(url: string, options: VideoStreamOptions) {
+ this.playing_ = options.autoplay ?? true;
+ this.loop_ = options.loop ?? false;
+ const decoder = wx.createVideoDecoder();
+ this.decoder_ = decoder;
+ decoder.start({ source: url, mode: 0 }).catch((err) => this.onError?.(err)); // mode 0 = decode by PTS
+ decoder.on('ended', () => {
+ if (this.loop_) {
+ this.lastPts_ = -1;
+ decoder.seek(0).catch(() => { /* not seekable */ });
+ } else {
+ this.onEnded?.();
+ }
+ });
+ if (!(options.muted ?? false)) this.attachAudio_(options.volume ?? 1);
+ }
+
+ private attachAudio_(volume: number): void {
+ // Needs base lib ≥2.14.0; degrade to silent on older runtimes.
+ if (typeof wx.createMediaAudioPlayer !== 'function') return;
+ try {
+ const player = wx.createMediaAudioPlayer();
+ this.audioPlayer_ = player;
+ player.volume = clamp01(volume);
+ player.addAudioSource(this.decoder_)
+ .then(() => player.start())
+ .catch((err) => log.warn('video', 'WeChat media audio source failed', err));
+ } catch (err) {
+ log.warn('video', 'WeChat media audio unavailable', err);
+ }
+ }
+
+ // === texture pump =======================================================
+
+ pump(module: ESEngineModule): void {
+ if (this.disposed_ || !this.playing_) return;
+ const frame = this.decoder_.getFrameData();
+ if (!frame.data || frame.data.byteLength === 0 || frame.width <= 0) return; // not decoded yet
+ if (frame.pkPts === this.lastPts_) return; // no new frame
+ this.lastPts_ = frame.pkPts;
+
+ const w = frame.width | 0, h = frame.height | 0;
+ const pixels = this.flipRows_(new Uint8Array(frame.data), w, h);
+
+ if (!this.texture_) {
+ this.width_ = w;
+ this.height_ = h;
+ this.texture_ = createTextureFromPixels(
+ module,
+ { width: w, height: h, pixels },
+ /* flipY */ false,
+ { filterMode: 'linear', wrapMode: 'clamp' },
+ );
+ this.ready_ = true;
+ this.onReady?.();
+ } else if (w === this.width_ && h === this.height_) {
+ updateTextureSubregion(module, this.texture_, 0, 0, w, h, pixels);
+ }
+ }
+
+ // Flip rows to bottom-first (reused buffer): updateTextureSubregion is
+ // flipY-off and a Sprite samples like an image upload. [device-verify: drop
+ // if WeChat frames are already bottom-first.]
+ private flipRows_(src: Uint8Array, w: number, h: number): Uint8Array {
+ const stride = w * 4;
+ const need = stride * h;
+ if (!this.flip_ || this.flip_.length !== need) this.flip_ = new Uint8Array(need);
+ const dst = this.flip_;
+ for (let y = 0; y < h; y++) {
+ dst.set(src.subarray(y * stride, y * stride + stride), (h - 1 - y) * stride);
+ }
+ return dst;
+ }
+
+ // === transport ==========================================================
+
+ get textureHandle(): number { return this.texture_; }
+ get width(): number { return this.width_; }
+ get height(): number { return this.height_; }
+ get bytes(): number { return this.width_ * this.height_ * 4; }
+ get isReady(): boolean { return this.ready_; }
+ get isPlaying(): boolean { return this.playing_; }
+ get currentTime(): number { return 0; } // wx decoder exposes no playhead
+ get duration(): number { return 0; }
+
+ play(): void { this.playing_ = true; }
+ pause(): void { this.playing_ = false; }
+ seek(timeSeconds: number): void {
+ this.lastPts_ = -1;
+ this.decoder_.seek(timeSeconds).catch(() => { /* not seekable */ });
+ }
+ setVolume(volume: number): void { if (this.audioPlayer_) this.audioPlayer_.volume = clamp01(volume); }
+ setMuted(muted: boolean): void { if (this.audioPlayer_) this.audioPlayer_.volume = muted ? 0 : 1; }
+ setLoop(loop: boolean): void { this.loop_ = loop; }
+ setPlaybackRate(_rate: number): void {
+ if (!this.rateWarned_) {
+ log.warn('video', 'WeChat video decoder has no playback-rate control');
+ this.rateWarned_ = true;
+ }
+ }
+
+ stop(): void {
+ if (this.disposed_) return;
+ this.disposed_ = true;
+ this.audioPlayer_?.stop().catch(() => { /* ignore */ });
+ this.audioPlayer_?.destroy().catch(() => { /* ignore */ });
+ this.decoder_.stop().catch(() => { /* ignore */ });
+ this.decoder_.remove().catch(() => { /* ignore */ });
+ if (this.texture_) {
+ requireResourceManager().releaseTexture(this.texture_);
+ this.texture_ = 0;
+ }
+ this.flip_ = null;
+ }
+}
+
+export class WeChatVideoBackend implements PlatformVideoBackend {
+ readonly name = 'WeChat';
+
+ createStream(url: string, options: VideoStreamOptions): VideoStreamHandle {
+ return new WeChatVideoStreamHandle(url, options);
+ }
+
+ dispose(): void { /* handles own their decoders; released on stop() */ }
+}
diff --git a/sdk/src/video/WebVideoBackend.ts b/sdk/src/video/WebVideoBackend.ts
index 5351b2032..9d05831ba 100644
--- a/sdk/src/video/WebVideoBackend.ts
+++ b/sdk/src/video/WebVideoBackend.ts
@@ -1,20 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
-/**
- * @file video/WebVideoBackend.ts
- * @brief Browser / Electron video backend. An HTMLVideoElement does the
- * demux + decode + audio + loop + seek natively and streams over the
- * asset URL (HTTP range). Each newly-decoded frame is drawn into a
- * reused canvas and pushed into a pathless engine texture via the same
- * `updateTextureSubregion` sub-upload the dynamic glyph atlas uses, so
- * a Sprite sampling that handle shows live video.
- *
- * The frame texture is created once (at the video's native size) and updated in
- * place; being pathless it is never touched by the residency LRU while the
- * handle holds it. A future optimization can swap the CPU readback for a
- * zero-copy `texImage2D(video)` onto a shared-context external texture behind
- * this same contract — no interface change.
- */
+// Browser / Electron video backend: an HTMLVideoElement streams and decodes,
+// each frame is drawn to a canvas and uploaded into a pathless engine texture.
import type { ESEngineModule } from '../wasm';
import type { PlatformVideoBackend, VideoStreamHandle, VideoStreamOptions } from './PlatformVideoBackend';
import { createTextureFromPixels, updateTextureSubregion } from '../runtimeAssets';
@@ -48,10 +35,7 @@ class WebVideoStreamHandle implements VideoStreamHandle {
constructor(url: string, options: VideoStreamOptions) {
const video = document.createElement('video');
this.rvfcSupported_ = typeof video.requestVideoFrameCallback === 'function';
- // Same-origin `estella://` and blob URLs need no CORS, but a remote http
- // source must be CORS-clean or the canvas readback taints and throws.
- video.crossOrigin = 'anonymous';
- // Inline (never fullscreen-hijack on mobile Safari) and quiet unless asked.
+ video.crossOrigin = 'anonymous'; // else the canvas readback taints
video.playsInline = true;
video.setAttribute('playsinline', '');
video.loop = options.loop ?? false;
@@ -90,18 +74,12 @@ class WebVideoStreamHandle implements VideoStreamHandle {
};
private tryPlay_(): void {
- // play() returns a promise that rejects when autoplay policy blocks an
- // unmuted start; surface it as a one-time warning, playback resumes on
- // the next explicit play() after a user gesture.
const p = this.video_.play() as unknown as Promise<void> | undefined;
if (p && typeof p.catch === 'function') {
p.catch((err) => log.warn('video', 'autoplay blocked (mute the video or start on input)', err));
}
}
- /** requestVideoFrameCallback fires exactly when a new frame is presentable
- * — the modern, jank-free signal. Falls back to per-tick currentTime deltas
- * in `pump` when the API is absent. */
private scheduleFrame_(): void {
if (this.disposed_ || !this.rvfcSupported_) return;
this.rvfcId_ = this.video_.requestVideoFrameCallback(() => {
@@ -123,14 +101,10 @@ class WebVideoStreamHandle implements VideoStreamHandle {
pump(module: ESEngineModule): void {
if (this.disposed_ || this.width_ <= 0 || this.height_ <= 0) return;
- // HAVE_CURRENT_DATA — there is a frame to sample.
- if (this.video_.readyState < 2) return;
-
- // Upload when a new frame is signalled (requestVideoFrameCallback) OR the
- // playhead moved since the last upload. The currentTime check is the
- // fallback for runtimes without rVFC, and also covers hosts where rVFC is
- // starved because the window never composites (a hidden/offscreen window,
- // e.g. the headless verify harness).
+ if (this.video_.readyState < 2) return; // HAVE_CURRENT_DATA — a frame to sample
+
+ // Upload on a new-frame signal OR a moved playhead — the latter also
+ // covers hosts where rVFC is starved (hidden/offscreen window).
const advanced = this.video_.currentTime !== this.lastUploadTime_;
if (this.ready_ && !this.newFrame_ && !advanced) return;
@@ -138,10 +112,8 @@ class WebVideoStreamHandle implements VideoStreamHandle {
const ctx = this.ctx_;
if (!ctx) return;
- // Draw flipped vertically so getImageData yields bottom-first rows: the
- // engine's `updateTextureSubregion` uploads bytes as-is (flipY off),
- // and Sprite sampling expects the same orientation image loads produce
- // (which upload top-first pixels flipY-on). Flipping here reconciles them.
+ // Draw flipped so getImageData yields bottom-first rows — the orientation
+ // updateTextureSubregion (flipY-off) needs for a Sprite to sample upright.
ctx.save();
ctx.translate(0, this.height_);
ctx.scale(1, -1);
diff --git a/sdk/src/video/index.ts b/sdk/src/video/index.ts
index c3e71fb7d..6bbcec236 100644
--- a/sdk/src/video/index.ts
+++ b/sdk/src/video/index.ts
@@ -1,9 +1,5 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
-/**
- * @file video/index.ts
- * @brief Public surface of the video system.
- */
export { VideoPlugin, videoPlugin } from './VideoPlugin';
export { VideoAPI, VideoPlayer, type VideoHandle, type VideoPlayOptions } from './VideoAPI';
export { Video, type VideoData } from './VideoComponents';