Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(video): play on any renderable — Sprite, UIVisual, or Mesh2D

yhhfcc37203c7fe
diff --git a/desktop/package.json b/desktop/package.json
index c3d9865f2..d934ddcda 100644
--- a/desktop/package.json
+++ b/desktop/package.json
@@ -18,6 +18,7 @@
"test:watch": "vitest",
"verify:render": "vite build && electron scripts/headless-verify.mjs",
"verify:render:video": "vite build && ESTELLA_VERIFY_PLAY=1 ESTELLA_VERIFY_SETTLE_MS=2500 ESTELLA_VERIFY_SCENE=/scenes/video-playback.esscene ESTELLA_VERIFY_MANIFEST=/scenes/video-playback.textures.json ESTELLA_VERIFY_W=256 ESTELLA_VERIFY_H=256 ESTELLA_VERIFY_EXPECT=\"[{\\\"x\\\":0.5,\\\"y\\\":0.25,\\\"rgb\\\":[255,0,0],\\\"tol\\\":90},{\\\"x\\\":0.5,\\\"y\\\":0.75,\\\"rgb\\\":[0,0,255],\\\"tol\\\":90}]\" electron scripts/headless-verify.mjs",
+ "verify:render:video:ui": "vite build && ESTELLA_VERIFY_PLAY=1 ESTELLA_VERIFY_SETTLE_MS=2500 ESTELLA_VERIFY_SCENE=/scenes/video-ui.esscene ESTELLA_VERIFY_MANIFEST=/scenes/video-ui.textures.json ESTELLA_VERIFY_W=800 ESTELLA_VERIFY_H=600 ESTELLA_VERIFY_EXPECT=\"[{\\\"x\\\":0.5,\\\"y\\\":0.25,\\\"rgb\\\":[255,0,0],\\\"tol\\\":90},{\\\"x\\\":0.5,\\\"y\\\":0.75,\\\"rgb\\\":[0,0,255],\\\"tol\\\":90}]\" electron scripts/headless-verify.mjs",
"editor:mcp": "node scripts/editor-mcp.mjs",
"editor:mcp:build": "vite build && node scripts/editor-mcp.mjs",
"editor:mcp:e2e": "vite build && node scripts/editor-mcp-e2e.mjs",
diff --git a/desktop/public/scenes/video-ui.esscene b/desktop/public/scenes/video-ui.esscene
new file mode 100644
index 000000000..419ecf137
--- /dev/null
+++ b/desktop/public/scenes/video-ui.esscene
@@ -0,0 +1,63 @@
+{
+ "version": "1.0",
+ "name": "VideoUI",
+ "entities": [
+ {
+ "id": 0,
+ "name": "Camera",
+ "parent": null,
+ "children": [],
+ "components": [
+ { "type": "Transform", "data": { "position": { "x": 0, "y": 0, "z": 10 } } },
+ { "type": "Camera", "data": { "projectionType": 1, "orthoSize": 300, "isActive": true, "priority": 0 } }
+ ],
+ "visible": true
+ },
+ {
+ "id": 1,
+ "name": "Canvas",
+ "parent": null,
+ "children": [2],
+ "components": [
+ { "type": "Transform", "data": {} },
+ { "type": "Canvas", "data": { "designResolution": { "x": 800, "y": 600 }, "scaleMode": 2 } },
+ { "type": "UINode", "data": {} }
+ ],
+ "visible": true
+ },
+ {
+ "id": 2,
+ "name": "Screen",
+ "parent": 1,
+ "children": [],
+ "components": [
+ { "type": "Transform", "data": {} },
+ {
+ "type": "UINode",
+ "data": {
+ "position": 1,
+ "insetLeft": { "value": 100, "unit": 0 },
+ "insetTop": { "value": 0, "unit": 0 },
+ "width": { "value": 600, "unit": 0 },
+ "height": { "value": 600, "unit": 0 }
+ }
+ },
+ { "type": "UIVisual", "data": { "visualType": 2, "color": { "r": 1, "g": 1, "b": 1, "a": 1 }, "enabled": true } },
+ {
+ "type": "Video",
+ "data": {
+ "source": "/scenes/video-playback.mp4",
+ "autoplay": true,
+ "loop": true,
+ "muted": true,
+ "volume": 1,
+ "playbackRate": 1,
+ "fitSize": false,
+ "enabled": true
+ }
+ }
+ ],
+ "visible": true
+ }
+ ]
+}
diff --git a/desktop/public/scenes/video-ui.textures.json b/desktop/public/scenes/video-ui.textures.json
new file mode 100644
index 000000000..0967ef424
--- /dev/null
+++ b/desktop/public/scenes/video-ui.textures.json
@@ -0,0 +1 @@
+{}
diff --git a/desktop/src/engine/SceneCommands.ts b/desktop/src/engine/SceneCommands.ts
index 2ad82030d..f15010a12 100644
--- a/desktop/src/engine/SceneCommands.ts
+++ b/desktop/src/engine/SceneCommands.ts
@@ -907,18 +907,21 @@ export class SceneCommandsImpl {
/** Add a component (with its registered/schema defaults) to an entity. Undoable. */
// Apply an add to the model and return its undo op, or null if it's a no-op
// (entity gone / component already present). Shared by the single + batch paths.
- /** Components auto-added alongside another (a lightweight dependency). A Video
- * renders through a Sprite, so adding a Video ensures a Sprite is present. */
- private requiredComponents(compName: string): readonly string[] {
- if (compName === 'Video') return ['Sprite'];
- return [];
+ /** Components auto-added alongside another. A Video renders through a
+ * renderable, so adding one ensures a Sprite unless the entity already has a
+ * renderable (a UI Video keeps its UIVisual, not a redundant Sprite). */
+ private requiredComponents(sourceId: EntityId, compName: string): readonly string[] {
+ if (compName !== 'Video') return [];
+ const comps = this.model.entityBySource(sourceId)?.components ?? [];
+ const hasRenderable = comps.some((c) => c.type === 'Sprite' || c.type === 'UIVisual' || c.type === 'Mesh2D');
+ return hasRenderable ? [] : ['Sprite'];
}
/** The add op for `compName` plus ops for any components it requires. Each op is
* null when that component already exists, so dependencies never duplicate. */
private addComponentOpsWithDeps(sourceId: EntityId, compName: string): UndoOp[] {
return [
- ...this.requiredComponents(compName).map((dep) => this.addComponentOp(sourceId, dep)),
+ ...this.requiredComponents(sourceId, compName).map((dep) => this.addComponentOp(sourceId, dep)),
this.addComponentOp(sourceId, compName),
].filter((o): o is UndoOp => !!o);
}
diff --git a/examples/video-playback/project.esproject b/examples/video-playback/project.esproject
index 9b139a7cf..79078b2f2 100644
--- a/examples/video-playback/project.esproject
+++ b/examples/video-playback/project.esproject
@@ -9,5 +9,17 @@
"width": 640,
"height": 480
},
- "spineVersion": "none"
+ "spineVersion": "none",
+ "packaging": {
+ "platform": "wechat",
+ "config": "shipping",
+ "sourceMaps": false,
+ "openFolder": true,
+ "compressTextures": false,
+ "atlasTextures": false,
+ "compressAudio": false,
+ "outDir": {
+ "wechat": "dist-wechat"
+ }
+ }
}
diff --git a/sdk/src/video/VideoPlugin.ts b/sdk/src/video/VideoPlugin.ts
index e01ff07b5..c1869b199 100644
--- a/sdk/src/video/VideoPlugin.ts
+++ b/sdk/src/video/VideoPlugin.ts
@@ -1,20 +1,63 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
// Wires the video system into an App: builds the backend, exposes VideoPlayer,
-// and runs one play-mode system that streams each Video and drives its Sprite.
+// and runs one play-mode system that streams each Video onto its renderable.
import type { App, Plugin } from '../app';
import type { Entity } from '../types';
+import type { World } from '../world';
import { defineSystem, Schedule } from '../system';
import { Res, Time, type TimeData } from '../resource';
import { VideoPlayer, VideoAPI } from './VideoAPI';
import { Video, type VideoData } from './VideoComponents';
-import { Sprite, type SpriteData } from '../component';
+import { Sprite, Mesh2D, type SpriteData, type Mesh2DData } from '../component';
+import type { UIVisualData } from '../component.generated';
+import { UIVisual } from '../ui/core/ui-visual';
import type { VideoStreamHandle } from './PlatformVideoBackend';
import { NullVideoBackend } from './NullVideoBackend';
import { getPlatform } from '../platform/base';
import { isEditor, isPlayMode } from '../env';
import { log } from '../logger';
+const UI_VISUAL_IMAGE = 2; // UIVisualType.Image — samples the texture
+
+// Drive the entity's renderable sibling (Sprite / UIVisual / Mesh2D) from the
+// current frame. Builtin get() is a snapshot copy, so mutations need world.insert;
+// the handle is stable, so we write back only on change. Returns false if the
+// entity has no renderable to show the video on.
+function driveRenderable(world: World, entity: Entity, handle: VideoStreamHandle, video: VideoData): boolean {
+ const tex = handle.textureHandle;
+
+ const sprite = world.tryGet(entity, Sprite) as SpriteData | null;
+ if (sprite) {
+ let dirty = false;
+ if (sprite.texture !== tex) { sprite.texture = tex; dirty = true; }
+ if (video.fitSize && handle.width > 0 &&
+ (sprite.size.x !== handle.width || sprite.size.y !== handle.height)) {
+ sprite.size = { x: handle.width, y: handle.height };
+ dirty = true;
+ }
+ if (dirty) world.insert(entity, Sprite, sprite);
+ return true;
+ }
+
+ const uiv = world.tryGet(entity, UIVisual) as UIVisualData | null;
+ if (uiv) {
+ let dirty = false;
+ if (uiv.texture !== tex) { uiv.texture = tex; dirty = true; }
+ if (uiv.visualType !== UI_VISUAL_IMAGE) { uiv.visualType = UI_VISUAL_IMAGE; dirty = true; }
+ if (dirty) world.insert(entity, UIVisual, uiv);
+ return true;
+ }
+
+ const mesh = world.tryGet(entity, Mesh2D) as Mesh2DData | null;
+ if (mesh) {
+ if (mesh.texture !== tex) { mesh.texture = tex; world.insert(entity, Mesh2D, mesh); }
+ return true;
+ }
+
+ return false;
+}
+
export class VideoPlugin implements Plugin {
name = 'video';
private video_: VideoAPI | null = null;
@@ -78,29 +121,11 @@ export class VideoPlugin implements Plugin {
const handle = handles.get(id);
if (!handle || !handle.textureHandle) continue;
- const sprite = world.tryGet(entity, Sprite) as SpriteData | null;
- if (!sprite) {
- if (!warnedNoSprite.has(id)) {
- log.warn('video', `entity ${id} has a Video but no Sprite to render into — add a Sprite component`);
- warnedNoSprite.add(id);
- }
- continue;
- }
-
- // Builtin get() is a snapshot copy — needs world.insert to
- // persist. The handle is stable, so write back only on change.
- let dirty = false;
- if (sprite.texture !== handle.textureHandle) {
- sprite.texture = handle.textureHandle;
- dirty = true;
- }
const v = world.get(entity, Video) as VideoData;
- if (v.fitSize && handle.width > 0 &&
- (sprite.size.x !== handle.width || sprite.size.y !== handle.height)) {
- sprite.size = { x: handle.width, y: handle.height };
- dirty = true;
+ if (!driveRenderable(world, entity, handle, v) && !warnedNoSprite.has(id)) {
+ log.warn('video', `entity ${id} has a Video but no renderable (Sprite/UIVisual/Mesh2D) to show it on`);
+ warnedNoSprite.add(id);
}
- if (dirty) world.insert(entity, Sprite, sprite);
}
// Reap streams whose entity is gone, disabled, or source-cleared.