Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(video): Video component + playback runtime (M0)

yhh8ec7f8228deb
diff --git a/desktop/package.json b/desktop/package.json
index 284cb727b..c3d9865f2 100644
--- a/desktop/package.json
+++ b/desktop/package.json
@@ -17,6 +17,7 @@
"test": "vitest run",
"test:watch": "vitest",
"verify:render": "vite build && electron scripts/headless-verify.mjs",
+ "verify:render:video": "vite build && ESTELLA_VERIFY_PLAY=1 ESTELLA_VERIFY_SETTLE_MS=2500 ESTELLA_VERIFY_SCENE=/scenes/video-playback.esscene ESTELLA_VERIFY_MANIFEST=/scenes/video-playback.textures.json ESTELLA_VERIFY_W=256 ESTELLA_VERIFY_H=256 ESTELLA_VERIFY_EXPECT=\"[{\\\"x\\\":0.5,\\\"y\\\":0.25,\\\"rgb\\\":[255,0,0],\\\"tol\\\":90},{\\\"x\\\":0.5,\\\"y\\\":0.75,\\\"rgb\\\":[0,0,255],\\\"tol\\\":90}]\" electron scripts/headless-verify.mjs",
"editor:mcp": "node scripts/editor-mcp.mjs",
"editor:mcp:build": "vite build && node scripts/editor-mcp.mjs",
"editor:mcp:e2e": "vite build && node scripts/editor-mcp-e2e.mjs",
diff --git a/desktop/public/scenes/video-playback.esscene b/desktop/public/scenes/video-playback.esscene
new file mode 100644
index 000000000..7a8d9da1e
--- /dev/null
+++ b/desktop/public/scenes/video-playback.esscene
@@ -0,0 +1,47 @@
+{
+ "version": "1.0",
+ "name": "VideoPlayback",
+ "entities": [
+ {
+ "id": 0,
+ "name": "Camera",
+ "parent": null,
+ "children": [],
+ "components": [
+ { "type": "Transform", "data": { "position": { "x": 0, "y": 0, "z": 10 } } },
+ { "type": "Camera", "data": { "projectionType": 1, "orthoSize": 300, "isActive": true, "priority": 0 } }
+ ],
+ "visible": true
+ },
+ {
+ "id": 1,
+ "name": "Screen",
+ "parent": null,
+ "children": [],
+ "components": [
+ { "type": "Transform", "data": { "position": { "x": 0, "y": 0, "z": 0 } } },
+ {
+ "type": "Sprite",
+ "data": {
+ "size": { "x": 600, "y": 600 },
+ "color": { "r": 1, "g": 1, "b": 1, "a": 1 }
+ }
+ },
+ {
+ "type": "Video",
+ "data": {
+ "source": "/scenes/video-playback.mp4",
+ "autoplay": true,
+ "loop": true,
+ "muted": true,
+ "volume": 1,
+ "playbackRate": 1,
+ "fitSize": false,
+ "enabled": true
+ }
+ }
+ ],
+ "visible": true
+ }
+ ]
+}
diff --git a/desktop/public/scenes/video-playback.mp4 b/desktop/public/scenes/video-playback.mp4
new file mode 100644
index 000000000..24b301832
Binary files /dev/null and b/desktop/public/scenes/video-playback.mp4 differ
diff --git a/desktop/public/scenes/video-playback.textures.json b/desktop/public/scenes/video-playback.textures.json
new file mode 100644
index 000000000..0967ef424
--- /dev/null
+++ b/desktop/public/scenes/video-playback.textures.json
@@ -0,0 +1 @@
+{}
diff --git a/desktop/scripts/headless-verify.mjs b/desktop/scripts/headless-verify.mjs
index 3d44ea279..c2f7d9f71 100644
--- a/desktop/scripts/headless-verify.mjs
+++ b/desktop/scripts/headless-verify.mjs
@@ -72,6 +72,7 @@ const MIME = {
'.css': 'text/css', '.json': 'application/json', '.esscene': 'application/json',
'.wasm': 'application/wasm', '.png': 'image/png', '.jpg': 'image/jpeg',
'.webp': 'image/webp', '.woff': 'font/woff', '.woff2': 'font/woff2', '.svg': 'image/svg+xml',
+ '.mp4': 'video/mp4', '.webm': 'video/webm',
};
function serveDist() {
@@ -167,6 +168,20 @@ app.whenReady().then(async () => {
await api.step(1, 1 / 60);
}
})()`);
+ } else if (process.env.ESTELLA_VERIFY_SETTLE_MS) {
+ // Real-time settle: some content decodes on the wall clock, not engine dt
+ // — video (HTMLVideoElement) decodes frames in real time and its system
+ // uploads the newest frame each tick. Interleave engine steps with real
+ // delays so frames actually arrive before the capture. Requires play mode.
+ const ms = Number(process.env.ESTELLA_VERIFY_SETTLE_MS) || 1500;
+ await exec(`(async () => {
+ const api = window.__estellaHeadless.api;
+ const end = performance.now() + ${ms};
+ while (performance.now() < end) {
+ await api.step(1, 1 / 60);
+ await new Promise((r) => setTimeout(r, 16));
+ }
+ })()`);
} else {
await exec(`window.__estellaHeadless.api.step(${STEPS}, 1 / 60)`);
}
diff --git a/examples/video-playback/README.md b/examples/video-playback/README.md
new file mode 100644
index 000000000..b4fa12981
--- /dev/null
+++ b/examples/video-playback/README.md
@@ -0,0 +1,49 @@
+# Video Playback
+
+Play a video file on a Sprite using the **`Video`** component.
+
+![tag:Rendering](https://img.shields.io/badge/tag-Rendering-blue)
+
+## What it shows
+
+The `Screen` entity carries a `Sprite` **and** a `Video`. The engine's video
+system streams `assets/video/clip.mp4`, decodes each frame into a live GPU
+texture, and drives the Sprite's texture from it every frame — so a video is
+simply "a Sprite whose texture is alive." The Sprite's normal properties
+(layer, tint, material, flip, lighting) all apply to the video unchanged.
+
+The scene is fully declarative — there is no gameplay code (`src/main.ts` is
+empty). Add a `Video` next to a `Sprite`, point `source` at a clip, and it
+plays.
+
+## The `Video` component
+
+| field | meaning |
+|-------|---------|
+| `source` | project-relative path / ref of the clip (`.mp4`, `.webm`) |
+| `autoplay` | start as soon as the clip is ready |
+| `loop` | restart at the end |
+| `muted` | mute the clip's own audio track |
+| `volume` | audio-track volume, `0..1` |
+| `playbackRate` | speed multiplier |
+| `fitSize` | on first frame, resize the Sprite to the clip's native pixels |
+
+## Code-driven playback
+
+For cutscenes/splashes, use the imperative service instead of the component:
+
+```ts
+import { VideoPlayer } from 'esengine';
+
+const video = app.getResource(VideoPlayer);
+const handle = video.play('assets/video/clip.mp4', { loop: true, muted: true });
+handle.onEnded = () => console.log('finished');
+// once handle.isReady, sample handle.textureHandle from any Sprite/material.
+```
+
+## Status
+
+This is the **M0** milestone: the runtime playback path (web / desktop). It is
+verified on both WebGL2 and WebGPU (`desktop/scripts/headless-verify.mjs` →
+`verify:render:video`). The editor asset-slot picker, `.mp4` import, and the
+WeChat backend land in the following milestones.
diff --git a/examples/video-playback/assets/scenes/main.esscene b/examples/video-playback/assets/scenes/main.esscene
new file mode 100644
index 000000000..aa0c76b82
--- /dev/null
+++ b/examples/video-playback/assets/scenes/main.esscene
@@ -0,0 +1,47 @@
+{
+ "version": "1.0",
+ "name": "Main",
+ "entities": [
+ {
+ "id": 0,
+ "name": "Camera",
+ "parent": null,
+ "children": [],
+ "components": [
+ { "type": "Transform", "data": { "position": { "x": 0, "y": 0, "z": 10 } } },
+ { "type": "Camera", "data": { "projectionType": 1, "orthoSize": 240, "isActive": true, "priority": 0 } }
+ ],
+ "visible": true
+ },
+ {
+ "id": 1,
+ "name": "Screen",
+ "parent": null,
+ "children": [],
+ "components": [
+ { "type": "Transform", "data": { "position": { "x": 0, "y": 0, "z": 0 } } },
+ {
+ "type": "Sprite",
+ "data": {
+ "size": { "x": 360, "y": 360 },
+ "color": { "r": 1, "g": 1, "b": 1, "a": 1 }
+ }
+ },
+ {
+ "type": "Video",
+ "data": {
+ "source": "assets/video/clip.mp4",
+ "autoplay": true,
+ "loop": true,
+ "muted": true,
+ "volume": 1,
+ "playbackRate": 1,
+ "fitSize": false,
+ "enabled": true
+ }
+ }
+ ],
+ "visible": true
+ }
+ ]
+}
diff --git a/examples/video-playback/assets/scenes/main.esscene.meta b/examples/video-playback/assets/scenes/main.esscene.meta
new file mode 100644
index 000000000..e1103e47d
--- /dev/null
+++ b/examples/video-playback/assets/scenes/main.esscene.meta
@@ -0,0 +1,8 @@
+{
+ "uuid": "af4f2bf6-4c1e-4ee8-be72-e08f807fb4c7",
+ "version": "2.0",
+ "type": "scene",
+ "importer": {
+ "autoMigrate": true
+ }
+}
diff --git a/examples/video-playback/assets/video/clip.mp4 b/examples/video-playback/assets/video/clip.mp4
new file mode 100644
index 000000000..a4e72b662
Binary files /dev/null and b/examples/video-playback/assets/video/clip.mp4 differ
diff --git a/examples/video-playback/assets/video/clip.mp4.meta b/examples/video-playback/assets/video/clip.mp4.meta
new file mode 100644
index 000000000..7bf348d07
--- /dev/null
+++ b/examples/video-playback/assets/video/clip.mp4.meta
@@ -0,0 +1,10 @@
+{
+ "uuid": "4fadbb7a-a246-4f02-8883-f69c3d8f210f",
+ "version": "2.0",
+ "type": "video",
+ "importer": {
+ "loop": true,
+ "autoplay": true,
+ "muted": true
+ }
+}
diff --git a/examples/video-playback/project.esproject b/examples/video-playback/project.esproject
new file mode 100644
index 000000000..9b139a7cf
--- /dev/null
+++ b/examples/video-playback/project.esproject
@@ -0,0 +1,13 @@
+{
+ "formatVersion": "1",
+ "name": "Video Playback",
+ "description": "Play a video on a Sprite via the Video component (M0).",
+ "tag": "Rendering",
+ "version": "0.1.0",
+ "defaultScene": "assets/scenes/main.esscene",
+ "designResolution": {
+ "width": 640,
+ "height": 480
+ },
+ "spineVersion": "none"
+}
diff --git a/examples/video-playback/src/main.ts b/examples/video-playback/src/main.ts
new file mode 100644
index 000000000..0d0e8b94d
--- /dev/null
+++ b/examples/video-playback/src/main.ts
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: Apache-2.0
+// Video Playback — the Video component is fully declarative: the "Screen"
+// entity in main.esscene carries a Sprite + a Video, and the engine's video
+// system decodes the clip into a live texture and drives the Sprite each frame.
+// So this example needs no gameplay code at all.
+//
+// For code-driven playback (cutscenes, splash screens) you can instead reach
+// the imperative service and play onto any Sprite's texture yourself:
+//
+// import { VideoPlayer, Sprite } from 'esengine';
+// const video = app.getResource(VideoPlayer);
+// const handle = video.play('assets/video/clip.mp4', { loop: true, muted: true });
+// handle.onEnded = () => console.log('done');
+// // each frame, once handle.isReady: sprite.texture = handle.textureHandle
+//
+// Nothing to register here — the scene drives everything.
+export {};
diff --git a/examples/video-playback/tsconfig.json b/examples/video-playback/tsconfig.json
new file mode 100644
index 000000000..0a8528b29
--- /dev/null
+++ b/examples/video-playback/tsconfig.json
@@ -0,0 +1,22 @@
+{
+ "compilerOptions": {
+ "target": "ES2020",
+ "module": "ESNext",
+ "moduleResolution": "bundler",
+ "strict": true,
+ "esModuleInterop": true,
+ "skipLibCheck": true,
+ "forceConsistentCasingInFileNames": true,
+ "declaration": false,
+ "outDir": "./dist",
+ "rootDir": ".",
+ "baseUrl": ".",
+ "paths": {
+ "esengine": ["./.esengine/sdk/index.d.ts"],
+ "esengine/wasm": ["./.esengine/sdk/wasm.d.ts"],
+ "@esengine/editor": ["./.esengine/editor/index.d.ts"]
+ }
+ },
+ "include": ["src/**/*"],
+ "exclude": ["node_modules"]
+}
diff --git a/sdk/etc/index.api.md b/sdk/etc/index.api.md
index 1930c09ac..eead2c412 100644
--- a/sdk/etc/index.api.md
+++ b/sdk/etc/index.api.md
@@ -1,7 +1,7 @@
# API surface — esengine/index
<!-- Generated by tools/api-surface.mjs — do not edit. Run --update to accept changes. -->
-Symbols: 1154 stable · 17 beta · 1 deprecated
+Symbols: 1165 stable · 17 beta · 1 deprecated
## ABI_LAYOUT_HASH — const
```
@@ -4189,6 +4189,7 @@ createAudioBackend: () => PlatformAudioBackend
createCanvas: (width: number, height: number) => HTMLCanvasElement | OffscreenCanvas
createImage: () => HTMLImageElement
createSocket: ((options: PlatformSocketOptions) => PlatformSocket) | undefined
+createVideoBackend: (() => PlatformVideoBackend) | undefined
devicePixelRatio: () => number
fetch: (url: string, options?: PlatformRequestOptions) => Promise<PlatformResponse>
fileExists: (path: string) => Promise<boolean>
@@ -4272,6 +4273,13 @@ url: string
'web' | 'wechat'
```
+## PlatformVideoBackend — interface
+```
+createStream: (url: string, options: VideoStreamOptions) => VideoStreamHandle
+dispose: () => void
+name: string
+```
+
## PlayConfig — interface
```
bus: string | undefined
@@ -6963,6 +6971,96 @@ name: string
type: DataType
```
+## Video — const
+```
+ComponentDef<VideoData>
+```
+
+## VideoAPI — class
+```
+backendName: string
+baseUrl: string
+dispose: () => void
+play: (source: string, options?: VideoPlayOptions) => VideoStreamHandle
+setRefResolver: (resolver: ((ref: string) => string) | null) => void
+stop: (handle: VideoStreamHandle) => void
+stopAll: () => void
+update: (module: ESEngineModule) => void
+static new (backend: PlatformVideoBackend): VideoAPI
+static prototype: VideoAPI
+```
+
+## VideoData — interface
+```
+autoplay: boolean
+enabled: boolean
+fitSize: boolean
+loop: boolean
+muted: boolean
+playbackRate: number
+source: string
+volume: number
+```
+
+## VideoHandle — type
+```
+VideoStreamHandle
+```
+
+## VideoPlayOptions — type
+```
+VideoStreamOptions
+```
+
+## VideoPlayer — const
+```
+ResourceDef<VideoAPI>
+```
+
+## VideoPlugin — class
+```
+build: (app: App) => void
+cleanup: () => void
+name: string
+stopAll: () => void
+static new (): VideoPlugin
+static prototype: VideoPlugin
+```
+
+## VideoStreamHandle — interface
+```
+bytes: number | undefined
+currentTime: number
+duration: number
+height: number
+id: number
+isPlaying: boolean
+isReady: boolean
+onEnded: (() => void) | undefined
+onError: ((error: unknown) => void) | undefined
+onReady: (() => void) | undefined
+pause: () => void
+play: () => void
+pump: (module: ESEngineModule) => void
+seek: (timeSeconds: number) => void
+setLoop: (loop: boolean) => void
+setMuted: (muted: boolean) => void
+setPlaybackRate: (rate: number) => void
+setVolume: (volume: number) => void
+stop: () => void
+textureHandle: number
+width: number
+```
+
+## VideoStreamOptions — interface
+```
+autoplay: boolean | undefined
+loop: boolean | undefined
+muted: boolean | undefined
+playbackRate: number | undefined
+volume: number | undefined
+```
+
## Viewport — interface
```
h: number
@@ -9058,6 +9156,11 @@ UIRenderOrderPlugin
(x?: number, y?: number, z?: number, w?: number) => Vec4
```
+## videoPlugin — const
+```
+VideoPlugin
+```
+
## warn — function
```
(category: string, message: string, data?: unknown): void
diff --git a/sdk/etc/index.node.api.md b/sdk/etc/index.node.api.md
index ffb3b9142..4a3e5cbdd 100644
--- a/sdk/etc/index.node.api.md
+++ b/sdk/etc/index.node.api.md
@@ -1,7 +1,7 @@
# API surface — esengine/index.node
<!-- Generated by tools/api-surface.mjs — do not edit. Run --update to accept changes. -->
-Symbols: 1153 stable · 19 beta · 1 deprecated
+Symbols: 1164 stable · 19 beta · 1 deprecated
## ANCHOR_AXES — const
```
@@ -4184,6 +4184,7 @@ createAudioBackend: () => PlatformAudioBackend
createCanvas: (width: number, height: number) => HTMLCanvasElement | OffscreenCanvas
createImage: () => HTMLImageElement
createSocket: ((options: PlatformSocketOptions) => PlatformSocket) | undefined
+createVideoBackend: (() => PlatformVideoBackend) | undefined
devicePixelRatio: () => number
fetch: (url: string, options?: PlatformRequestOptions) => Promise<PlatformResponse>
fileExists: (path: string) => Promise<boolean>
@@ -4267,6 +4268,13 @@ url: string
'web' | 'wechat'
```
+## PlatformVideoBackend — interface
+```
+createStream: (url: string, options: VideoStreamOptions) => VideoStreamHandle
+dispose: () => void
+name: string
+```
+
## PlayConfig — interface
```
bus: string | undefined
@@ -6958,6 +6966,96 @@ name: string
type: DataType
```
+## Video — const
+```
+ComponentDef<VideoData>
+```
+
+## VideoAPI — class
+```
+backendName: string
+baseUrl: string
+dispose: () => void
+play: (source: string, options?: VideoPlayOptions) => VideoStreamHandle
+setRefResolver: (resolver: ((ref: string) => string) | null) => void
+stop: (handle: VideoStreamHandle) => void
+stopAll: () => void
+update: (module: ESEngineModule) => void
+static new (backend: PlatformVideoBackend): VideoAPI
+static prototype: VideoAPI
+```
+
+## VideoData — interface
+```
+autoplay: boolean
+enabled: boolean
+fitSize: boolean
+loop: boolean
+muted: boolean
+playbackRate: number
+source: string
+volume: number
+```
+
+## VideoHandle — type
+```
+VideoStreamHandle
+```
+
+## VideoPlayOptions — type
+```
+VideoStreamOptions
+```
+
+## VideoPlayer — const
+```
+ResourceDef<VideoAPI>
+```
+
+## VideoPlugin — class
+```
+build: (app: App) => void
+cleanup: () => void
+name: string
+stopAll: () => void
+static new (): VideoPlugin
+static prototype: VideoPlugin
+```
+
+## VideoStreamHandle — interface
+```
+bytes: number | undefined
+currentTime: number
+duration: number
+height: number
+id: number
+isPlaying: boolean
+isReady: boolean
+onEnded: (() => void) | undefined
+onError: ((error: unknown) => void) | undefined
+onReady: (() => void) | undefined
+pause: () => void
+play: () => void
+pump: (module: ESEngineModule) => void
+seek: (timeSeconds: number) => void
+setLoop: (loop: boolean) => void
+setMuted: (muted: boolean) => void
+setPlaybackRate: (rate: number) => void
+setVolume: (volume: number) => void
+stop: () => void
+textureHandle: number
+width: number
+```
+
+## VideoStreamOptions — interface
+```
+autoplay: boolean | undefined
+loop: boolean | undefined
+muted: boolean | undefined
+playbackRate: number | undefined
+volume: number | undefined
+```
+
## Viewport — interface
```
h: number
@@ -9063,6 +9161,11 @@ UIRenderOrderPlugin
(x?: number, y?: number, z?: number, w?: number) => Vec4
```
+## videoPlugin — const
+```
+VideoPlugin
+```
+
## warn — function
```
(category: string, message: string, data?: unknown): void
diff --git a/sdk/src/core-content.ts b/sdk/src/core-content.ts
index 3ba2b9b18..5a7df1fbb 100644
--- a/sdk/src/core-content.ts
+++ b/sdk/src/core-content.ts
@@ -321,6 +321,24 @@ export {
type PooledAudioNode,
} from './audio';
+// =============================================================================
+// Video
+// =============================================================================
+
+export {
+ VideoPlayer,
+ VideoAPI,
+ VideoPlugin,
+ videoPlugin,
+ Video,
+ type VideoHandle,
+ type VideoPlayOptions,
+ type VideoData,
+ type PlatformVideoBackend,
+ type VideoStreamHandle,
+ type VideoStreamOptions,
+} from './video';
+
// =============================================================================
// Particle
// =============================================================================
diff --git a/sdk/src/platform/types.ts b/sdk/src/platform/types.ts
index 993068fb8..b199b8363 100644
--- a/sdk/src/platform/types.ts
+++ b/sdk/src/platform/types.ts
@@ -146,6 +146,12 @@ export interface PlatformAdapter {
createAudioBackend(): import('../audio/PlatformAudioBackend').PlatformAudioBackend;
+ /** Create the platform video backend (HTMLVideoElement on web,
+ * wx.createVideoDecoder on WeChat). Optional — like createSocket, platforms
+ * without video decode omit it and the video system falls back to a silent
+ * Null backend (headless server, WeChat until its backend lands). */
+ createVideoBackend?(): import('../video/PlatformVideoBackend').PlatformVideoBackend;
+
/** Download an on-demand asset subpackage by name and resolve when its files
* are available. WeChat → wx.loadSubpackage; platforms with no subpackage
* concept (web) omit it and lazy groups load directly from their URLs. */
diff --git a/sdk/src/platform/web.ts b/sdk/src/platform/web.ts
index 5adba4fa1..4bc9ae639 100644
--- a/sdk/src/platform/web.ts
+++ b/sdk/src/platform/web.ts
@@ -18,6 +18,8 @@ import type {
} from './types';
import { WebAudioBackend } from '../audio/WebAudioBackend';
import type { PlatformAudioBackend } from '../audio/PlatformAudioBackend';
+import { WebVideoBackend } from '../video/WebVideoBackend';
+import type { PlatformVideoBackend } from '../video/PlatformVideoBackend';
import { GameSocket } from '../net/GameSocket';
const WHEEL_LINE_HEIGHT = 16;
@@ -276,6 +278,10 @@ class WebPlatformAdapter implements PlatformAdapter {
return new WebAudioBackend();
}
+ createVideoBackend(): PlatformVideoBackend {
+ return new WebVideoBackend();
+ }
+
createSocket(options: PlatformSocketOptions): PlatformSocket {
return new GameSocket(options);
}
diff --git a/sdk/src/playRealmRuntime.ts b/sdk/src/playRealmRuntime.ts
index ff93baf00..b6f9f23e9 100644
--- a/sdk/src/playRealmRuntime.ts
+++ b/sdk/src/playRealmRuntime.ts
@@ -18,6 +18,7 @@
import type { App } from './app';
import type { ESEngineModule } from './wasm';
import { Audio } from './audio/Audio';
+import { VideoPlayer } from './video/VideoAPI';
import { initRuntime } from './runtimeLoader';
import type { RuntimeAssetSource } from './runtimeAssets';
import { HttpBackend } from './asset/Backend';
@@ -161,6 +162,12 @@ export async function initPlayRealmRuntime(config: PlayRealmRuntimeConfig): Prom
(ref) => resolvePlayAssetRef(ref, assetManifest, assetBaseUrl, config.assetPathMap),
);
}
+ // Video source refs resolve through the same channel (see the Audio note above).
+ if (app.hasResource(VideoPlayer)) {
+ app.getResource(VideoPlayer).setRefResolver(
+ (ref) => resolvePlayAssetRef(ref, assetManifest, assetBaseUrl, config.assetPathMap),
+ );
+ }
const entryName = config.entrySceneName ?? '__play';
await initRuntime({
app,
diff --git a/sdk/src/video/NullVideoBackend.ts b/sdk/src/video/NullVideoBackend.ts
new file mode 100644
index 000000000..8ec7b5e4c
--- /dev/null
+++ b/sdk/src/video/NullVideoBackend.ts
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file video/NullVideoBackend.ts
+ * @brief Silent no-op video backend — the visual mirror of NullAudioBackend.
+ * Used on headless Node (authoritative server), as the WeChat fallback
+ * until WeChatVideoBackend lands, and anywhere a platform advertises no
+ * video capability. Every call succeeds and does nothing; the frame
+ * texture stays 0 so a driven Sprite simply shows no video. DOM-free,
+ * so importing it never pulls browser globals into a headless bundle.
+ */
+import type { PlatformVideoBackend, VideoStreamHandle, VideoStreamOptions } from './PlatformVideoBackend';
+
+let nextId_ = 1;
+
+class NullVideoStreamHandle implements VideoStreamHandle {
+ readonly id = nextId_++;
+ readonly textureHandle = 0;
+ readonly width = 0;
+ readonly height = 0;
+ readonly isReady = false;
+ private playing_: boolean;
+ onReady?: () => void;
+ onEnded?: () => void;
+ onError?: (error: unknown) => void;
+
+ constructor(options: VideoStreamOptions) {
+ this.playing_ = options.autoplay ?? true;
+ }
+
+ get isPlaying(): boolean { return this.playing_; }
+ get currentTime(): number { return 0; }
+ get duration(): number { return 0; }
+
+ play(): void { this.playing_ = true; }
+ pause(): void { this.playing_ = false; }
+ stop(): void { this.playing_ = false; this.onEnded = undefined; }
+ seek(): void { /* no-op */ }
+ setVolume(): void { /* no-op */ }
+ setMuted(): void { /* no-op */ }
+ setLoop(): void { /* no-op */ }
+ setPlaybackRate(): void { /* no-op */ }
+ pump(): void { /* no frames */ }
+}
+
+export class NullVideoBackend implements PlatformVideoBackend {
+ readonly name = 'null';
+
+ createStream(_url: string, options: VideoStreamOptions): VideoStreamHandle {
+ return new NullVideoStreamHandle(options);
+ }
+
+ dispose(): void { /* nothing to release */ }
+}
diff --git a/sdk/src/video/PlatformVideoBackend.ts b/sdk/src/video/PlatformVideoBackend.ts
new file mode 100644
index 000000000..4a345e19b
--- /dev/null
+++ b/sdk/src/video/PlatformVideoBackend.ts
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file video/PlatformVideoBackend.ts
+ * @brief Platform-neutral video contract — the visual-side mirror of
+ * {@link ../audio/PlatformAudioBackend}. A backend turns a source
+ * reference into a {@link VideoStreamHandle}: a live decode that
+ * exposes an engine texture handle, updated in place each frame from
+ * the current decoded frame. Web decodes with an HTMLVideoElement,
+ * WeChat with wx.createVideoDecoder, headless with a no-op — the
+ * rest of the engine only ever sees this interface.
+ */
+import type { ESEngineModule } from '../wasm';
+
+/**
+ * A live video decode. Unlike audio (where a decoded buffer and a playing
+ * voice are separate handles), a video stream IS both the resource and the
+ * transport, because a frame is only meaningful while decoding.
+ *
+ * `textureHandle` is the engine texture the current frame lands in — a
+ * pathless, non-evictable texture the handle owns for its lifetime (created
+ * once dimensions are known). It reads `0` until the first frame is ready;
+ * callers sample it through the normal Sprite/Mesh2D texture slot.
+ */
+export interface VideoStreamHandle {
+ readonly id: number;
+ /** Engine texture handle for the current frame; `0` until `isReady`. */
+ readonly textureHandle: number;
+ /** Native frame size in pixels; `0` until `isReady`. */
+ readonly width: number;
+ readonly height: number;
+ /**
+ * VRAM bytes of the frame texture, for a future residency budget. Backends
+ * that own no sampleable texture (native overlay, headless) omit it —
+ * 0/undefined means untracked, mirroring `AudioBufferHandle.bytes`.
+ */
+ readonly bytes?: number;
+ readonly isReady: boolean;
+ readonly isPlaying: boolean;
+ /** Playback position / total length in seconds (0 when unknown). */
+ readonly currentTime: number;
+ readonly duration: number;
+
+ play(): void;
+ pause(): void;
+ /** Stop and release the decode + frame texture. Idempotent. */
+ stop(): void;
+ seek(timeSeconds: number): void;
+ setVolume(volume: number): void;
+ setMuted(muted: boolean): void;
+ setLoop(loop: boolean): void;
+ setPlaybackRate(rate: number): void;
+
+ /**
+ * Upload the current decoded frame into `textureHandle` when a new one is
+ * available (creating the texture on the first ready frame). Called once
+ * per frame by the video system with the live wasm module. Cheap to call
+ * when no new frame has arrived.
+ */
+ pump(module: ESEngineModule): void;
+
+ /** Fires once the first frame + dimensions are ready (texture now valid). */
+ onReady?: () => void;
+ /** Fires when a non-looping stream reaches the end. */
+ onEnded?: () => void;
+ onError?: (error: unknown) => void;
+}
+
+export interface VideoStreamOptions {
+ autoplay?: boolean;
+ loop?: boolean;
+ muted?: boolean;
+ /** 0..1, applied to the stream's own audio track. */
+ volume?: number;
+ playbackRate?: number;
+}
+
+/**
+ * The platform video factory. One `createStream` per playing video; the
+ * returned handle owns its decode and frame texture. Selected through the
+ * platform adapter (`getPlatform().createVideoBackend?.()`), never imported
+ * directly — that keeps DOM/wx code out of the wrong bundles.
+ */
+export interface PlatformVideoBackend {
+ readonly name: string;
+ createStream(url: string, options: VideoStreamOptions): VideoStreamHandle;
+ dispose(): void;
+}
diff --git a/sdk/src/video/VideoAPI.ts b/sdk/src/video/VideoAPI.ts
new file mode 100644
index 000000000..9053bfe3f
--- /dev/null
+++ b/sdk/src/video/VideoAPI.ts
@@ -0,0 +1,102 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file video/VideoAPI.ts
+ * @brief Per-app video service — the imperative face of the video system and
+ * the visual mirror of AudioAPI. Code-driven playback (cutscenes,
+ * splash) calls `video.play(src)`; the declarative `Video` component's
+ * system uses the same API under the hood. Each `App` owns one
+ * instance, created by `VideoPlugin.build()` and exposed as the
+ * {@link VideoPlayer} resource (`Res(VideoPlayer)` / `getResource`).
+ *
+ * Video texture residency is simpler than audio's: a stream owns one pathless,
+ * non-evictable frame texture for its lifetime and frees it on stop — there is
+ * no re-fetchable decoded buffer to warm-cache, so there is no eviction budget.
+ */
+import type { ESEngineModule } from '../wasm';
+import type { PlatformVideoBackend, VideoStreamHandle, VideoStreamOptions } from './PlatformVideoBackend';
+import { defineResource } from '../resource';
+
+export type VideoHandle = VideoStreamHandle;
+export type VideoPlayOptions = VideoStreamOptions;
+
+export class VideoAPI {
+ private readonly backend_: PlatformVideoBackend;
+ private readonly handles_ = new Set<VideoStreamHandle>();
+ private refResolver_: ((ref: string) => string) | null = null;
+ private disposed_ = false;
+ /** Legacy prefix for plain project-relative refs; `refResolver` wins. */
+ baseUrl = '';
+
+ constructor(backend: PlatformVideoBackend) {
+ this.backend_ = backend;
+ }
+
+ get backendName(): string { return this.backend_.name; }
+
+ /**
+ * Route source refs through the realm's asset resolver — the same channel
+ * every other asset resolves through (uuid manifest, cooked logical→staged
+ * map, project base). Mirrors {@link AudioAPI.setRefResolver}.
+ */
+ setRefResolver(resolver: ((ref: string) => string) | null): void {
+ this.refResolver_ = resolver;
+ }
+
+ private resolveUrl_(ref: string): string {
+ if (this.refResolver_) return this.refResolver_(ref);
+ // Absolute or scheme-bearing (estella://, blob:, data:, http(s)://,
+ // leading /) — already a URL the element can open.
+ if (!this.baseUrl || ref.includes('://') || ref.startsWith('/') || ref.startsWith('blob:') || ref.startsWith('data:')) {
+ return ref;
+ }
+ return `${this.baseUrl}/${ref}`;
+ }
+
+ /**
+ * Start a video from a source ref, returning its live handle. The frame
+ * texture is created lazily on the first decoded frame (see the handle's
+ * `textureHandle`); poll `handle.isReady`/`onReady` before sampling it.
+ */
+ play(source: string, options: VideoPlayOptions = {}): VideoStreamHandle {
+ const handle = this.backend_.createStream(this.resolveUrl_(source), options);
+ this.handles_.add(handle);
+ const userEnded = handle.onEnded;
+ handle.onEnded = () => {
+ userEnded?.();
+ // A finished non-looping stream stays in the set (its last frame is
+ // still valid to show); explicit stop() releases it.
+ };
+ return handle;
+ }
+
+ /** Advance every live stream: upload any newly-decoded frame into its
+ * texture. Called once per frame by the video system with the live module. */
+ update(module: ESEngineModule): void {
+ if (this.disposed_) return;
+ for (const handle of this.handles_) handle.pump(module);
+ }
+
+ /** Stop one stream and release its frame texture. */
+ stop(handle: VideoStreamHandle): void {
+ if (this.handles_.delete(handle)) handle.stop();
+ }
+
+ stopAll(): void {
+ for (const handle of this.handles_) handle.stop();
+ this.handles_.clear();
+ }
+
+ dispose(): void {
+ if (this.disposed_) return;
+ this.disposed_ = true;
+ this.stopAll();
+ this.backend_.dispose();
+ }
+}
+
+/**
+ * The per-app video service resource. Consume via `Res(VideoPlayer)` in a
+ * system or `app.getResource(VideoPlayer)` outside ECS code.
+ */
+export const VideoPlayer = defineResource<VideoAPI>(null!, 'VideoPlayer');
diff --git a/sdk/src/video/VideoComponents.ts b/sdk/src/video/VideoComponents.ts
new file mode 100644
index 000000000..828f7e623
--- /dev/null
+++ b/sdk/src/video/VideoComponents.ts
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file video/VideoComponents.ts
+ * @brief The declarative `Video` component — the editor/designer face of the
+ * video system, mirroring how `AudioSource` fronts the audio API.
+ *
+ * A `Video` renders through its entity's `Sprite`: the video system decodes the
+ * source into a live texture and writes that handle (and, when `fitSize`, the
+ * native pixel size) into the sibling Sprite each frame. So a video is "a Sprite
+ * whose texture is alive" and inherits all of Sprite's rendering — layer, tint,
+ * material, lighting, flip, blend — for free. Add a `Sprite` alongside `Video`
+ * (the editor's add-component does this automatically); without one the system
+ * warns once and draws nothing.
+ */
+import { defineComponent } from '../component';
+
+export interface VideoData {
+ /** Project-relative path / asset ref of the video (.mp4/.webm). */
+ source: string;
+ /** Begin playing as soon as the source is ready. */
+ autoplay: boolean;
+ loop: boolean;
+ muted: boolean;
+ /** Volume of the video's own audio track, 0..1. */
+ volume: number;
+ playbackRate: number;
+ /** On first ready frame, drive the sibling Sprite's `size` to the video's
+ * native pixel dimensions. Off = keep the Sprite's authored size (stretch). */
+ fitSize: boolean;
+ enabled: boolean;
+}
+
+export const Video = defineComponent<VideoData>('Video', {
+ source: '',
+ autoplay: true,
+ loop: true,
+ muted: true,
+ volume: 1.0,
+ playbackRate: 1.0,
+ fitSize: true,
+ enabled: true,
+});
diff --git a/sdk/src/video/VideoPlugin.ts b/sdk/src/video/VideoPlugin.ts
new file mode 100644
index 000000000..4029c8a79
--- /dev/null
+++ b/sdk/src/video/VideoPlugin.ts
@@ -0,0 +1,151 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file video/VideoPlugin.ts
+ * @brief Wires the video system into an App — the visual mirror of AudioPlugin.
+ * Builds the platform video backend, exposes it as the `VideoPlayer`
+ * resource, and runs one Update-schedule system that: starts a stream
+ * per playing `Video` entity, uploads decoded frames into their pathless
+ * textures, and drives each entity's sibling `Sprite` from the current
+ * frame. Runs only in play mode (editor edit-mode leaves videos paused).
+ */
+import type { App, Plugin } from '../app';
+import type { Entity } from '../types';
+import { defineSystem, Schedule } from '../system';
+import { Res, Time, type TimeData } from '../resource';
+import { VideoPlayer, VideoAPI } from './VideoAPI';
+import { Video, type VideoData } from './VideoComponents';
+import { Sprite, type SpriteData } from '../component';
+import type { VideoStreamHandle } from './PlatformVideoBackend';
+import { NullVideoBackend } from './NullVideoBackend';
+import { getPlatform } from '../platform/base';
+import { isEditor, isPlayMode } from '../env';
+import { log } from '../logger';
+
+export class VideoPlugin implements Plugin {
+ name = 'video';
+ private video_: VideoAPI | null = null;
+ private handles_: Map<number, VideoStreamHandle> | null = null;
+
+ build(app: App): void {
+ // `createVideoBackend` is an optional platform capability (like
+ // createSocket/loadSubpackage): web provides a real one, headless/WeChat
+ // fall back to the silent Null backend until their own lands.
+ const backend = getPlatform().createVideoBackend?.() ?? new NullVideoBackend();
+ const video = new VideoAPI(backend);
+ this.video_ = video;
+ app.insertResource(VideoPlayer, video);
+
+ // entity id → its component-driven stream.
+ const handles = new Map<number, VideoStreamHandle>();
+ this.handles_ = handles;
+ const warnedNoSprite = new Set<number>();
+ const live = new Set<number>();
+
+ app.world.onDespawn((entity: Entity) => {
+ const handle = handles.get(entity as number);
+ if (handle) {
+ video.stop(handle);
+ handles.delete(entity as number);
+ }
+ warnedNoSprite.delete(entity as number);
+ });
+
+ app.addSystemToSchedule(
+ Schedule.Update,
+ defineSystem(
+ [Res(Time), Res(VideoPlayer)],
+ (_time: TimeData, videoAPI: VideoAPI) => {
+ if (isEditor() && !isPlayMode()) return;
+ const world = app.world;
+ const module = world.getWasmModule();
+ if (!module) return;
+
+ const entities = world.getEntitiesWithComponents([Video]);
+ live.clear();
+
+ // Ensure a stream per playing entity (before the pump below,
+ // so a freshly-started stream uploads its first frame this tick).
+ for (const entity of entities) {
+ const v = world.get(entity, Video) as VideoData;
+ if (!v.enabled || !v.source) continue;
+ const id = entity as number;
+ live.add(id);
+ if (!handles.has(id)) {
+ handles.set(id, videoAPI.play(v.source, {
+ autoplay: v.autoplay,
+ loop: v.loop,
+ muted: v.muted,
+ volume: v.volume,
+ playbackRate: v.playbackRate,
+ }));
+ }
+ }
+
+ // Upload any newly-decoded frames (component + imperative streams).
+ videoAPI.update(module);
+
+ // Project the current frame onto each entity's sibling Sprite.
+ for (const entity of entities) {
+ const id = entity as number;
+ const handle = handles.get(id);
+ if (!handle || !handle.textureHandle) continue;
+
+ const sprite = world.tryGet(entity, Sprite) as SpriteData | null;
+ if (!sprite) {
+ if (!warnedNoSprite.has(id)) {
+ log.warn('video', `entity ${id} has a Video but no Sprite to render into — add a Sprite component`);
+ warnedNoSprite.add(id);
+ }
+ continue;
+ }
+
+ // Builtin components hand back a snapshot copy — mutating it
+ // is discarded unless written back with `world.insert` (the
+ // same pattern SpriteAnimator uses). The video's per-frame
+ // pixels ride the SAME texture handle (updated in place), so
+ // the Sprite only needs a write-back when the handle or size
+ // actually changes — typically once, on the first ready frame.
+ let dirty = false;
+ if (sprite.texture !== handle.textureHandle) {
+ sprite.texture = handle.textureHandle;
+ dirty = true;
+ }
+ const v = world.get(entity, Video) as VideoData;
+ if (v.fitSize && handle.width > 0 &&
+ (sprite.size.x !== handle.width || sprite.size.y !== handle.height)) {
+ sprite.size = { x: handle.width, y: handle.height };
+ dirty = true;
+ }
+ if (dirty) world.insert(entity, Sprite, sprite);
+ }
+
+ // Reap streams whose entity vanished, was disabled, or lost its source.
+ for (const [id, handle] of handles) {
+ if (!live.has(id)) {
+ videoAPI.stop(handle);
+ handles.delete(id);
+ warnedNoSprite.delete(id);
+ }
+ }
+ },
+ { name: 'VideoUpdateSystem' }
+ )
+ );
+ }
+
+ stopAll(): void {
+ if (this.handles_) {
+ for (const handle of this.handles_.values()) handle.stop();
+ this.handles_.clear();
+ }
+ }
+
+ cleanup(): void {
+ this.stopAll();
+ this.video_?.dispose();
+ this.video_ = null;
+ }
+}
+
+export const videoPlugin = new VideoPlugin();
diff --git a/sdk/src/video/WebVideoBackend.ts b/sdk/src/video/WebVideoBackend.ts
new file mode 100644
index 000000000..5351b2032
--- /dev/null
+++ b/sdk/src/video/WebVideoBackend.ts
@@ -0,0 +1,231 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file video/WebVideoBackend.ts
+ * @brief Browser / Electron video backend. An HTMLVideoElement does the
+ * demux + decode + audio + loop + seek natively and streams over the
+ * asset URL (HTTP range). Each newly-decoded frame is drawn into a
+ * reused canvas and pushed into a pathless engine texture via the same
+ * `updateTextureSubregion` sub-upload the dynamic glyph atlas uses, so
+ * a Sprite sampling that handle shows live video.
+ *
+ * The frame texture is created once (at the video's native size) and updated in
+ * place; being pathless it is never touched by the residency LRU while the
+ * handle holds it. A future optimization can swap the CPU readback for a
+ * zero-copy `texImage2D(video)` onto a shared-context external texture behind
+ * this same contract — no interface change.
+ */
+import type { ESEngineModule } from '../wasm';
+import type { PlatformVideoBackend, VideoStreamHandle, VideoStreamOptions } from './PlatformVideoBackend';
+import { createTextureFromPixels, updateTextureSubregion } from '../runtimeAssets';
+import { requireResourceManager } from '../resourceManager';
+import { log } from '../logger';
+
+let nextId_ = 1;
+
+const clamp01 = (v: number): number => (v < 0 ? 0 : v > 1 ? 1 : v);
+
+class WebVideoStreamHandle implements VideoStreamHandle {
+ readonly id = nextId_++;
+ onReady?: () => void;
+ onEnded?: () => void;
+ onError?: (error: unknown) => void;
+
+ private readonly video_: HTMLVideoElement;
+ private canvas_: HTMLCanvasElement | null = null;
+ private ctx_: CanvasRenderingContext2D | null = null;
+ private texture_ = 0;
+ private width_ = 0;
+ private height_ = 0;
+ private ready_ = false;
+ private disposed_ = false;
+ private newFrame_ = true;
+ private lastUploadTime_ = -1;
+ private rvfcId_ = 0;
+ private readonly rvfcSupported_: boolean;
+ private wantsPlay_: boolean;
+
+ constructor(url: string, options: VideoStreamOptions) {
+ const video = document.createElement('video');
+ this.rvfcSupported_ = typeof video.requestVideoFrameCallback === 'function';
+ // Same-origin `estella://` and blob URLs need no CORS, but a remote http
+ // source must be CORS-clean or the canvas readback taints and throws.
+ video.crossOrigin = 'anonymous';
+ // Inline (never fullscreen-hijack on mobile Safari) and quiet unless asked.
+ video.playsInline = true;
+ video.setAttribute('playsinline', '');
+ video.loop = options.loop ?? false;
+ video.muted = options.muted ?? false;
+ video.volume = clamp01(options.volume ?? 1);
+ video.playbackRate = options.playbackRate ?? 1;
+ video.preload = 'auto';
+ this.wantsPlay_ = options.autoplay ?? true;
+
+ video.addEventListener('loadedmetadata', this.onLoadedMetadata_);
+ video.addEventListener('ended', this.onEndedEvent_);
+ video.addEventListener('error', this.onErrorEvent_);
+ video.src = url;
+ video.load();
+ this.video_ = video;
+
+ if (this.wantsPlay_) this.tryPlay_();
+ }
+
+ // === lifecycle ==========================================================
+
+ private onLoadedMetadata_ = (): void => {
+ this.width_ = this.video_.videoWidth | 0;
+ this.height_ = this.video_.videoHeight | 0;
+ this.scheduleFrame_();
+ };
+
+ private onEndedEvent_ = (): void => {
+ if (!this.video_.loop) this.onEnded?.();
+ };
+
+ private onErrorEvent_ = (): void => {
+ const err = this.video_.error;
+ log.warn('video', `decode error (code ${err?.code ?? '?'}): ${err?.message ?? 'unknown'}`);
+ this.onError?.(err ?? new Error('video decode error'));
+ };
+
+ private tryPlay_(): void {
+ // play() returns a promise that rejects when autoplay policy blocks an
+ // unmuted start; surface it as a one-time warning, playback resumes on
+ // the next explicit play() after a user gesture.
+ const p = this.video_.play() as unknown as Promise<void> | undefined;
+ if (p && typeof p.catch === 'function') {
+ p.catch((err) => log.warn('video', 'autoplay blocked (mute the video or start on input)', err));
+ }
+ }
+
+ /** requestVideoFrameCallback fires exactly when a new frame is presentable
+ * — the modern, jank-free signal. Falls back to per-tick currentTime deltas
+ * in `pump` when the API is absent. */
+ private scheduleFrame_(): void {
+ if (this.disposed_ || !this.rvfcSupported_) return;
+ this.rvfcId_ = this.video_.requestVideoFrameCallback(() => {
+ this.newFrame_ = true;
+ this.scheduleFrame_();
+ });
+ }
+
+ private ensureCanvas_(): void {
+ if (this.canvas_ || this.width_ <= 0 || this.height_ <= 0) return;
+ const canvas = document.createElement('canvas');
+ canvas.width = this.width_;
+ canvas.height = this.height_;
+ this.canvas_ = canvas;
+ this.ctx_ = canvas.getContext('2d', { willReadFrequently: true });
+ }
+
+ // === texture pump =======================================================
+
+ pump(module: ESEngineModule): void {
+ if (this.disposed_ || this.width_ <= 0 || this.height_ <= 0) return;
+ // HAVE_CURRENT_DATA — there is a frame to sample.
+ if (this.video_.readyState < 2) return;
+
+ // Upload when a new frame is signalled (requestVideoFrameCallback) OR the
+ // playhead moved since the last upload. The currentTime check is the
+ // fallback for runtimes without rVFC, and also covers hosts where rVFC is
+ // starved because the window never composites (a hidden/offscreen window,
+ // e.g. the headless verify harness).
+ const advanced = this.video_.currentTime !== this.lastUploadTime_;
+ if (this.ready_ && !this.newFrame_ && !advanced) return;
+
+ this.ensureCanvas_();
+ const ctx = this.ctx_;
+ if (!ctx) return;
+
+ // Draw flipped vertically so getImageData yields bottom-first rows: the
+ // engine's `updateTextureSubregion` uploads bytes as-is (flipY off),
+ // and Sprite sampling expects the same orientation image loads produce
+ // (which upload top-first pixels flipY-on). Flipping here reconciles them.
+ ctx.save();
+ ctx.translate(0, this.height_);
+ ctx.scale(1, -1);
+ ctx.drawImage(this.video_ as CanvasImageSource, 0, 0, this.width_, this.height_);
+ ctx.restore();
+
+ let frame: ImageData;
+ try {
+ frame = ctx.getImageData(0, 0, this.width_, this.height_);
+ } catch (err) {
+ // Tainted canvas (cross-origin without CORS) — give up on readback.
+ this.onError?.(err);
+ this.disposed_ = true;
+ return;
+ }
+ const pixels = new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength);
+
+ if (!this.texture_) {
+ this.texture_ = createTextureFromPixels(
+ module,
+ { width: this.width_, height: this.height_, pixels },
+ /* flipY */ false,
+ { filterMode: 'linear', wrapMode: 'clamp' },
+ );
+ } else {
+ updateTextureSubregion(module, this.texture_, 0, 0, this.width_, this.height_, pixels);
+ }
+
+ this.newFrame_ = false;
+ this.lastUploadTime_ = this.video_.currentTime;
+ if (!this.ready_) {
+ this.ready_ = true;
+ this.onReady?.();
+ }
+ }
+
+ // === transport ==========================================================
+
+ get textureHandle(): number { return this.texture_; }
+ get width(): number { return this.width_; }
+ get height(): number { return this.height_; }
+ get bytes(): number { return this.width_ * this.height_ * 4; }
+ get isReady(): boolean { return this.ready_; }
+ get isPlaying(): boolean { return !this.video_.paused && !this.video_.ended; }
+ get currentTime(): number { return this.video_.currentTime; }
+ get duration(): number { return Number.isFinite(this.video_.duration) ? this.video_.duration : 0; }
+
+ play(): void { this.wantsPlay_ = true; this.tryPlay_(); }
+ pause(): void { this.wantsPlay_ = false; this.video_.pause(); }
+ seek(timeSeconds: number): void {
+ try { this.video_.currentTime = timeSeconds; this.newFrame_ = true; } catch { /* not seekable yet */ }
+ }
+ setVolume(volume: number): void { this.video_.volume = clamp01(volume); }
+ setMuted(muted: boolean): void { this.video_.muted = muted; }
+ setLoop(loop: boolean): void { this.video_.loop = loop; }
+ setPlaybackRate(rate: number): void { this.video_.playbackRate = rate > 0 ? rate : 0; }
+
+ stop(): void {
+ if (this.disposed_) return;
+ this.disposed_ = true;
+ if (this.rvfcId_ && this.rvfcSupported_) {
+ this.video_.cancelVideoFrameCallback(this.rvfcId_);
+ }
+ this.video_.removeEventListener('loadedmetadata', this.onLoadedMetadata_);
+ this.video_.removeEventListener('ended', this.onEndedEvent_);
+ this.video_.removeEventListener('error', this.onErrorEvent_);
+ try { this.video_.pause(); } catch { /* ignore */ }
+ this.video_.removeAttribute('src');
+ try { this.video_.load(); } catch { /* ignore */ }
+ if (this.texture_) {
+ requireResourceManager().releaseTexture(this.texture_);
+ this.texture_ = 0;
+ }
+ this.canvas_ = null;
+ this.ctx_ = null;
+ }
+}
+
+export class WebVideoBackend implements PlatformVideoBackend {
+ readonly name = 'web';
+
+ createStream(url: string, options: VideoStreamOptions): VideoStreamHandle {
+ return new WebVideoStreamHandle(url, options);
+ }
+
+ dispose(): void { /* handles own their own resources; released on stop() */ }
+}
diff --git a/sdk/src/video/index.ts b/sdk/src/video/index.ts
new file mode 100644
index 000000000..c3e71fb7d
--- /dev/null
+++ b/sdk/src/video/index.ts
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file video/index.ts
+ * @brief Public surface of the video system.
+ */
+export { VideoPlugin, videoPlugin } from './VideoPlugin';
+export { VideoAPI, VideoPlayer, type VideoHandle, type VideoPlayOptions } from './VideoAPI';
+export { Video, type VideoData } from './VideoComponents';
+export type {
+ PlatformVideoBackend,
+ VideoStreamHandle,
+ VideoStreamOptions,
+} from './PlatformVideoBackend';
+export { WebVideoBackend } from './WebVideoBackend';
+export { NullVideoBackend } from './NullVideoBackend';
diff --git a/sdk/src/webAppFactory.ts b/sdk/src/webAppFactory.ts
index b89780f36..fabd08a64 100644
--- a/sdk/src/webAppFactory.ts
+++ b/sdk/src/webAppFactory.ts
@@ -9,6 +9,7 @@ import { seedEngineComponents } from './component';
import { uiPlugins } from './uiPlugins';
import { animationPlugin } from './animation';
import { audioPlugin } from './audio';
+import { videoPlugin } from './video';
import { particlePlugin } from './particle';
import { trailPlugin } from './trail';
import { mesh2dPlugin } from './mesh2d';
@@ -40,6 +41,7 @@ export { safeAreaPlugin, SafeAreaPlugin } from './ui/layout/safe-area';
export { PhysicsPlugin, PhysicsEvents, Physics, loadPhysicsModule } from './physics';
export { AnimationPlugin, animationPlugin } from './animation';
export { AudioPlugin, audioPlugin } from './audio';
+export { VideoPlugin, videoPlugin, VideoAPI, VideoPlayer, Video } from './video';
export { ParticlePlugin, particlePlugin } from './particle';
export { TrailPlugin, trailPlugin } from './trail';
export { Mesh2DPlugin, mesh2dPlugin } from './mesh2d';
@@ -66,7 +68,7 @@ export interface CreateWebAppOptions extends WebAppOptions {
wasmBaseUrl?: string;
}
-const basePlugins = [timerPlugin, lifecyclePlugin(), animationPlugin, audioPlugin, particlePlugin, trailPlugin, mesh2dPlugin, tilemapPlugin, postProcessPlugin, timelinePlugin, perceptionPlugin, fsmPlugin, btPlugin, navPlugin, replicationPlugin];
+const basePlugins = [timerPlugin, lifecyclePlugin(), animationPlugin, audioPlugin, videoPlugin, particlePlugin, trailPlugin, mesh2dPlugin, tilemapPlugin, postProcessPlugin, timelinePlugin, perceptionPlugin, fsmPlugin, btPlugin, navPlugin, replicationPlugin];
export function createWebApp(module: ESEngineModule, options?: CreateWebAppOptions): App {
const sideModules: SideModuleHost | undefined = options?.sideModules