A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(video): play on any renderable — Sprite, UIVisual, or Mesh2D
yhhfcc37203c7fe
diff --git a/desktop/package.json b/desktop/package.jsonindex c3d9865f2..d934ddcda 100644--- a/desktop/package.json+++ b/desktop/package.json@@ -18,6 +18,7 @@"test:watch": "vitest","verify:render": "vite build && electron scripts/headless-verify.mjs","verify:render:video": "vite build && ESTELLA_VERIFY_PLAY=1 ESTELLA_VERIFY_SETTLE_MS=2500 ESTELLA_VERIFY_SCENE=/scenes/video-playback.esscene ESTELLA_VERIFY_MANIFEST=/scenes/video-playback.textures.json ESTELLA_VERIFY_W=256 ESTELLA_VERIFY_H=256 ESTELLA_VERIFY_EXPECT=\"[{\\\"x\\\":0.5,\\\"y\\\":0.25,\\\"rgb\\\":[255,0,0],\\\"tol\\\":90},{\\\"x\\\":0.5,\\\"y\\\":0.75,\\\"rgb\\\":[0,0,255],\\\"tol\\\":90}]\" electron scripts/headless-verify.mjs",+ "verify:render:video:ui": "vite build && ESTELLA_VERIFY_PLAY=1 ESTELLA_VERIFY_SETTLE_MS=2500 ESTELLA_VERIFY_SCENE=/scenes/video-ui.esscene ESTELLA_VERIFY_MANIFEST=/scenes/video-ui.textures.json ESTELLA_VERIFY_W=800 ESTELLA_VERIFY_H=600 ESTELLA_VERIFY_EXPECT=\"[{\\\"x\\\":0.5,\\\"y\\\":0.25,\\\"rgb\\\":[255,0,0],\\\"tol\\\":90},{\\\"x\\\":0.5,\\\"y\\\":0.75,\\\"rgb\\\":[0,0,255],\\\"tol\\\":90}]\" electron scripts/headless-verify.mjs","editor:mcp": "node scripts/editor-mcp.mjs","editor:mcp:build": "vite build && node scripts/editor-mcp.mjs","editor:mcp:e2e": "vite build && node scripts/editor-mcp-e2e.mjs",diff --git a/desktop/public/scenes/video-ui.esscene b/desktop/public/scenes/video-ui.esscenenew file mode 100644index 000000000..419ecf137--- /dev/null+++ b/desktop/public/scenes/video-ui.esscene@@ -0,0 +1,63 @@+{+ "version": "1.0",+ "name": "VideoUI",+ "entities": [+ {+ "id": 0,+ "name": "Camera",+ "parent": null,+ "children": [],+ "components": [+ { "type": "Transform", "data": { "position": { "x": 0, "y": 0, "z": 10 } } },+ { "type": "Camera", "data": { "projectionType": 1, "orthoSize": 300, "isActive": true, "priority": 0 } }+ ],+ "visible": true+ },+ {+ "id": 1,+ "name": "Canvas",+ "parent": null,+ "children": [2],+ "components": [+ { "type": "Transform", "data": {} },+ { "type": "Canvas", "data": { "designResolution": { "x": 800, "y": 600 }, "scaleMode": 2 } },+ { "type": "UINode", "data": {} }+ ],+ "visible": true+ },+ {+ "id": 2,+ "name": "Screen",+ "parent": 1,+ "children": [],+ "components": [+ { "type": "Transform", "data": {} },+ {+ "type": "UINode",+ "data": {+ "position": 1,+ "insetLeft": { "value": 100, "unit": 0 },+ "insetTop": { "value": 0, "unit": 0 },+ "width": { "value": 600, "unit": 0 },+ "height": { "value": 600, "unit": 0 }+ }+ },+ { "type": "UIVisual", "data": { "visualType": 2, "color": { "r": 1, "g": 1, "b": 1, "a": 1 }, "enabled": true } },+ {+ "type": "Video",+ "data": {+ "source": "/scenes/video-playback.mp4",+ "autoplay": true,+ "loop": true,+ "muted": true,+ "volume": 1,+ "playbackRate": 1,+ "fitSize": false,+ "enabled": true+ }+ }+ ],+ "visible": true+ }+ ]+}diff --git a/desktop/public/scenes/video-ui.textures.json b/desktop/public/scenes/video-ui.textures.jsonnew file mode 100644index 000000000..0967ef424--- /dev/null+++ b/desktop/public/scenes/video-ui.textures.json@@ -0,0 +1 @@+{}diff --git a/desktop/src/engine/SceneCommands.ts b/desktop/src/engine/SceneCommands.tsindex 2ad82030d..f15010a12 100644--- a/desktop/src/engine/SceneCommands.ts+++ b/desktop/src/engine/SceneCommands.ts@@ -907,18 +907,21 @@ export class SceneCommandsImpl {/** Add a component (with its registered/schema defaults) to an entity. Undoable. */// Apply an add to the model and return its undo op, or null if it's a no-op// (entity gone / component already present). Shared by the single + batch paths.- /** Components auto-added alongside another (a lightweight dependency). A Video- * renders through a Sprite, so adding a Video ensures a Sprite is present. */- private requiredComponents(compName: string): readonly string[] {- if (compName === 'Video') return ['Sprite'];- return [];+ /** Components auto-added alongside another. A Video renders through a+ * renderable, so adding one ensures a Sprite unless the entity already has a+ * renderable (a UI Video keeps its UIVisual, not a redundant Sprite). */+ private requiredComponents(sourceId: EntityId, compName: string): readonly string[] {+ if (compName !== 'Video') return [];+ const comps = this.model.entityBySource(sourceId)?.components ?? [];+ const hasRenderable = comps.some((c) => c.type === 'Sprite' || c.type === 'UIVisual' || c.type === 'Mesh2D');+ return hasRenderable ? [] : ['Sprite'];}/** The add op for `compName` plus ops for any components it requires. Each op is* null when that component already exists, so dependencies never duplicate. */private addComponentOpsWithDeps(sourceId: EntityId, compName: string): UndoOp[] {return [- ...this.requiredComponents(compName).map((dep) => this.addComponentOp(sourceId, dep)),+ ...this.requiredComponents(sourceId, compName).map((dep) => this.addComponentOp(sourceId, dep)),this.addComponentOp(sourceId, compName),].filter((o): o is UndoOp => !!o);}diff --git a/examples/video-playback/project.esproject b/examples/video-playback/project.esprojectindex 9b139a7cf..79078b2f2 100644--- a/examples/video-playback/project.esproject+++ b/examples/video-playback/project.esproject@@ -9,5 +9,17 @@"width": 640,"height": 480},- "spineVersion": "none"+ "spineVersion": "none",+ "packaging": {+ "platform": "wechat",+ "config": "shipping",+ "sourceMaps": false,+ "openFolder": true,+ "compressTextures": false,+ "atlasTextures": false,+ "compressAudio": false,+ "outDir": {+ "wechat": "dist-wechat"+ }+ }}diff --git a/sdk/src/video/VideoPlugin.ts b/sdk/src/video/VideoPlugin.tsindex e01ff07b5..c1869b199 100644--- a/sdk/src/video/VideoPlugin.ts+++ b/sdk/src/video/VideoPlugin.ts@@ -1,20 +1,63 @@// SPDX-License-Identifier: Apache-2.0// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team// Wires the video system into an App: builds the backend, exposes VideoPlayer,-// and runs one play-mode system that streams each Video and drives its Sprite.+// and runs one play-mode system that streams each Video onto its renderable.import type { App, Plugin } from '../app';import type { Entity } from '../types';+import type { World } from '../world';import { defineSystem, Schedule } from '../system';import { Res, Time, type TimeData } from '../resource';import { VideoPlayer, VideoAPI } from './VideoAPI';import { Video, type VideoData } from './VideoComponents';-import { Sprite, type SpriteData } from '../component';+import { Sprite, Mesh2D, type SpriteData, type Mesh2DData } from '../component';+import type { UIVisualData } from '../component.generated';+import { UIVisual } from '../ui/core/ui-visual';import type { VideoStreamHandle } from './PlatformVideoBackend';import { NullVideoBackend } from './NullVideoBackend';import { getPlatform } from '../platform/base';import { isEditor, isPlayMode } from '../env';import { log } from '../logger';+const UI_VISUAL_IMAGE = 2; // UIVisualType.Image — samples the texture++// Drive the entity's renderable sibling (Sprite / UIVisual / Mesh2D) from the+// current frame. Builtin get() is a snapshot copy, so mutations need world.insert;+// the handle is stable, so we write back only on change. Returns false if the+// entity has no renderable to show the video on.+function driveRenderable(world: World, entity: Entity, handle: VideoStreamHandle, video: VideoData): boolean {+ const tex = handle.textureHandle;++ const sprite = world.tryGet(entity, Sprite) as SpriteData | null;+ if (sprite) {+ let dirty = false;+ if (sprite.texture !== tex) { sprite.texture = tex; dirty = true; }+ if (video.fitSize && handle.width > 0 &&+ (sprite.size.x !== handle.width || sprite.size.y !== handle.height)) {+ sprite.size = { x: handle.width, y: handle.height };+ dirty = true;+ }+ if (dirty) world.insert(entity, Sprite, sprite);+ return true;+ }++ const uiv = world.tryGet(entity, UIVisual) as UIVisualData | null;+ if (uiv) {+ let dirty = false;+ if (uiv.texture !== tex) { uiv.texture = tex; dirty = true; }+ if (uiv.visualType !== UI_VISUAL_IMAGE) { uiv.visualType = UI_VISUAL_IMAGE; dirty = true; }+ if (dirty) world.insert(entity, UIVisual, uiv);+ return true;+ }++ const mesh = world.tryGet(entity, Mesh2D) as Mesh2DData | null;+ if (mesh) {+ if (mesh.texture !== tex) { mesh.texture = tex; world.insert(entity, Mesh2D, mesh); }+ return true;+ }++ return false;+}+export class VideoPlugin implements Plugin {name = 'video';private video_: VideoAPI | null = null;@@ -78,29 +121,11 @@ export class VideoPlugin implements Plugin {const handle = handles.get(id);if (!handle || !handle.textureHandle) continue;- const sprite = world.tryGet(entity, Sprite) as SpriteData | null;- if (!sprite) {- if (!warnedNoSprite.has(id)) {- log.warn('video', `entity ${id} has a Video but no Sprite to render into — add a Sprite component`);- warnedNoSprite.add(id);- }- continue;- }-- // Builtin get() is a snapshot copy — needs world.insert to- // persist. The handle is stable, so write back only on change.- let dirty = false;- if (sprite.texture !== handle.textureHandle) {- sprite.texture = handle.textureHandle;- dirty = true;- }const v = world.get(entity, Video) as VideoData;- if (v.fitSize && handle.width > 0 &&- (sprite.size.x !== handle.width || sprite.size.y !== handle.height)) {- sprite.size = { x: handle.width, y: handle.height };- dirty = true;+ if (!driveRenderable(world, entity, handle, v) && !warnedNoSprite.has(id)) {+ log.warn('video', `entity ${id} has a Video but no renderable (Sprite/UIVisual/Mesh2D) to show it on`);+ warnedNoSprite.add(id);}- if (dirty) world.insert(entity, Sprite, sprite);}// Reap streams whose entity is gone, disabled, or source-cleared.