A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
// Wires the video system into an App: builds the backend, exposes VideoPlayer,
// and runs one play-mode system that streams each Video onto its renderable.
import type { App, Plugin } from '../app/app';
import type { Entity } from '../types';
import type { World } from '../ecs/world';
import { defineSystem, Schedule } from '../ecs/system';
import { Res, Time, type TimeData } from '../ecs/resource';
import { VideoPlayer, VideoAPI } from './VideoAPI';
import { Video, type VideoData } from './VideoComponents';
import { Sprite, MeshRenderer, type SpriteData, type MeshRendererData } from '../ecs/component';
import type { UIVisualData } from '../ecs/component.generated';
import { UIVisual } from '../ui/core/ui-visual';
import type { VideoStreamHandle } from './PlatformVideoBackend';
import { NullVideoBackend } from './NullVideoBackend';
import { Audio } from '../audio/Audio';
import { getPlatform } from '../platform/base';
import { isEditor, isPlayMode } from '../ecs/env';
import { log } from '../util/logger';
const UI_VISUAL_IMAGE = 2; // UIVisualType.Image — samples the texture
// Drive the entity's renderable sibling (Sprite / UIVisual / MeshRenderer) from the
// current frame. Builtin get() is a snapshot copy, so mutations need world.insert;
// the handle is stable, so we write back only on change. Returns false if the
// entity has no renderable to show the video on.
function driveRenderable(world: World, entity: Entity, handle: VideoStreamHandle, video: VideoData): boolean {
const tex = handle.textureHandle;
const sprite = world.tryGet(entity, Sprite) as SpriteData | null;
if (sprite) {
let dirty = false;
if (sprite.texture !== tex) { sprite.texture = tex; dirty = true; }
if (video.fitSize && handle.width > 0 &&
(sprite.size.x !== handle.width || sprite.size.y !== handle.height)) {
sprite.size = { x: handle.width, y: handle.height };
dirty = true;
}
if (dirty) world.insert(entity, Sprite, sprite);
return true;
}
const uiv = world.tryGet(entity, UIVisual) as UIVisualData | null;
if (uiv) {
let dirty = false;
if (uiv.texture !== tex) { uiv.texture = tex; dirty = true; }
if (uiv.visualType !== UI_VISUAL_IMAGE) { uiv.visualType = UI_VISUAL_IMAGE; dirty = true; }
if (dirty) world.insert(entity, UIVisual, uiv);
return true;
}
const mesh = world.tryGet(entity, MeshRenderer) as MeshRendererData | null;
if (mesh) {
if (mesh.texture !== tex) { mesh.texture = tex; world.insert(entity, MeshRenderer, mesh); }
return true;
}
return false;
}
interface VideoPluginState {
video: VideoAPI;
handles: Map<number, VideoStreamHandle>;
offDespawn: () => void;
released: boolean;
}
type StreamSettings = Pick<VideoData, 'source' | 'volume' | 'muted' | 'loop' | 'playbackRate'>;
export class VideoPlugin implements Plugin {
name = 'video';
private readonly states_ = new WeakMap<App, VideoPluginState>();
private latestState_: VideoPluginState | null = null;
build(app: App): void {
this.cleanup(app);
// createVideoBackend is optional; fall back to the silent Null backend.
// The context's getters are lazy so the wasm backend resolves the
// side-module host / audio service per stream (they may attach to the
// app after plugins build).
const backend = getPlatform().createVideoBackend?.({
sideModules: () => app.sideModules,
audio: () => {
try { return app.getResource(Audio); } catch { return null; }
},
}) ?? new NullVideoBackend();
const video = new VideoAPI(backend, app.onExecutionChange.bind(app));
app.insertResource(VideoPlayer, video);
const handles = new Map<number, VideoStreamHandle>();
const settings = new WeakMap<VideoStreamHandle, StreamSettings>();
const warnedNoSprite = new Set<number>();
const live = new Set<number>();
const offDespawn = app.world.onDespawn((entity: Entity) => {
const handle = handles.get(entity as number);
if (handle) {
video.stop(handle);
handles.delete(entity as number);
}
warnedNoSprite.delete(entity as number);
});
const state: VideoPluginState = { video, handles, offDespawn, released: false };
this.states_.set(app, state);
this.latestState_ = state;
app.addSystemToSchedule(
Schedule.Update,
defineSystem(
[Res(Time), Res(VideoPlayer)],
(_time: TimeData, videoAPI: VideoAPI) => {
if (state.released || (isEditor() && !isPlayMode())) return;
const world = app.world;
// Null on a native core: the frame upload goes through the
// ResourceManager's byte path instead of a wasm heap.
const module = world.getWasmModule();
const entities = world.getEntitiesWithComponents([Video]);
live.clear();
// Start streams before the pump so a new one uploads this tick.
for (const entity of entities) {
const v = world.get(entity, Video) as VideoData;
if (!v.enabled || !v.source) continue;
const id = entity as number;
live.add(id);
const previous = handles.get(id);
const applied = previous ? settings.get(previous) : undefined;
if (!previous || !applied || applied.source !== v.source) {
if (previous) videoAPI.stop(previous);
const handle = videoAPI.play(v.source, {
autoplay: v.autoplay,
loop: v.loop,
muted: v.muted,
volume: v.volume,
playbackRate: v.playbackRate,
});
handles.set(id, handle);
settings.set(handle, { source: v.source, volume: v.volume, muted: v.muted,
loop: v.loop, playbackRate: v.playbackRate });
} else {
if (applied.volume !== v.volume) { previous.setVolume(v.volume); applied.volume = v.volume; }
if (applied.muted !== v.muted) { previous.setMuted(v.muted); applied.muted = v.muted; }
if (applied.loop !== v.loop) { previous.setLoop(v.loop); applied.loop = v.loop; }
if (applied.playbackRate !== v.playbackRate) {
previous.setPlaybackRate(v.playbackRate); applied.playbackRate = v.playbackRate;
}
}
}
videoAPI.update(module);
for (const entity of entities) {
const id = entity as number;
const handle = handles.get(id);
if (!handle || !handle.textureHandle) continue;
const v = world.get(entity, Video) as VideoData;
if (!driveRenderable(world, entity, handle, v) && !warnedNoSprite.has(id)) {
log.warn('video', `entity ${id} has a Video but no renderable (Sprite/UIVisual/MeshRenderer) to show it on`);
warnedNoSprite.add(id);
}
}
// Reap streams whose entity is gone, disabled, or source-cleared.
for (const [id, handle] of handles) {
if (!live.has(id)) {
videoAPI.stop(handle);
handles.delete(id);
warnedNoSprite.delete(id);
}
}
},
{ name: 'VideoUpdateSystem' }
)
);
}
stopAll(): void {
const state = this.latestState_;
if (!state || state.released) return;
for (const handle of state.handles.values()) state.video.stop(handle);
state.handles.clear();
}
cleanup(app?: App): void {
const state = app ? this.states_.get(app) : this.latestState_;
if (!state || state.released) return;
state.released = true;
if (app) this.states_.delete(app);
if (state === this.latestState_) this.latestState_ = null;
state.offDespawn();
state.handles.clear();
state.video.dispose();
}
}
export const videoPlugin = new VideoPlugin();