A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(video): editor integration for the Video component (M1)
yhh120f897e7c67
diff --git a/desktop/electron/assetMeta.ts b/desktop/electron/assetMeta.tsindex 44f0ed543..727add970 100644--- a/desktop/electron/assetMeta.ts+++ b/desktop/electron/assetMeta.ts@@ -25,6 +25,7 @@ export const EXT_TO_TYPE: Record<string, string> = {'.png': 'texture', '.jpg': 'texture', '.jpeg': 'texture', '.webp': 'texture', '.bmp': 'texture','.ktx2': 'texture','.wav': 'audio', '.mp3': 'audio', '.ogg': 'audio', '.aac': 'audio', '.flac': 'audio', '.m4a': 'audio', '.webm': 'audio',+ '.mp4': 'video', '.m4v': 'video', '.mov': 'video','.esprefab': 'prefab', '.esscene': 'scene', '.esshader': 'shader', '.esmaterial': 'material', '.esmat': 'material','.esanim': 'animclip', '.esanimclip': 'animclip', '.estimeline': 'animation','.estileset': 'tileset', '.esfsm': 'statemachine', '.esbt': 'behaviortree', '.eslocale': 'locale',diff --git a/desktop/electron/main.ts b/desktop/electron/main.tsindex c51af5718..58504054c 100644--- a/desktop/electron/main.ts+++ b/desktop/electron/main.ts@@ -2,7 +2,7 @@// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Teamimport { app, BrowserWindow, Menu, shell, ipcMain, dialog, protocol } from 'electron';import { fileURLToPath } from 'node:url';-import { readFile, writeFile } from 'node:fs/promises';+import { readFile, writeFile, open, stat } from 'node:fs/promises';import { existsSync } from 'node:fs';import path from 'node:path';import {@@ -715,23 +715,73 @@ ipcMain.handle('project:saveDialog', async (_e, defaultRel?: string) => {return rel;});+// Serve a project file, honoring an HTTP Range request (206) when present.+// <video> streams and seeks via Range; every other client omits the header and+// gets the whole file (200). Ranged reads pull only the requested slice off+// disk, so seeking a large clip never loads it whole into memory.+async function serveProjectFile(abs: string, request: Request): Promise<Response> {+ const headers: Record<string, string> = {+ 'content-type': httpContentType(abs),+ // The play realm (app:// origin) loads project assets cross-scheme via+ // <img crossorigin> + fetch; allow it. estella:// is only reachable inside+ // the Electron app, so there is no untrusted-web exposure.+ 'access-control-allow-origin': '*',+ 'accept-ranges': 'bytes',+ 'cache-control': 'no-store',+ };+ const rangeHeader = request.headers.get('range');+ const range = rangeHeader ? parseByteRange(rangeHeader, (await stat(abs)).size) : null;+ if (range === 'invalid') {+ return new Response('range not satisfiable', {+ status: 416, headers: { ...headers, 'content-range': `bytes */${(await stat(abs)).size}` },+ });+ }+ if (range) {+ const fh = await open(abs, 'r');+ try {+ const buf = Buffer.alloc(range.length);+ await fh.read(buf, 0, range.length, range.start);+ return new Response(new Uint8Array(buf), {+ status: 206,+ headers: { ...headers, 'content-range': `bytes ${range.start}-${range.end}/${range.size}`, 'content-length': String(range.length) },+ });+ } finally {+ await fh.close();+ }+ }+ const bytes = await readFile(abs);+ return new Response(new Uint8Array(bytes), { headers });+}++/** Parse a single-range `bytes=start-end` header against a known file size.+ * Returns the resolved slice, `'invalid'` for an unsatisfiable range, or null+ * when the header isn't a form we serve (caller sends the whole file). */+function parseByteRange(header: string, size: number):+ { start: number; end: number; length: number; size: number } | 'invalid' | null {+ const m = /^bytes=(\d*)-(\d*)$/.exec(header.trim());+ if (!m) return null;+ let start = m[1] === '' ? NaN : parseInt(m[1], 10);+ let end = m[2] === '' ? NaN : parseInt(m[2], 10);+ if (Number.isNaN(start)) {+ // suffix range `bytes=-N`: the last N bytes.+ const n = Number.isNaN(end) ? 0 : end;+ start = Math.max(0, size - n);+ end = size - 1;+ } else if (Number.isNaN(end)) {+ end = size - 1;+ }+ if (size === 0 || start > end || start >= size) return 'invalid';+ end = Math.min(end, size - 1);+ return { start, end, length: end - start + 1, size };+}+// estella://project/<relpath> → bytes from the open project root (sandboxed).async function handleEstella(request: Request): Promise<Response> {if (!projectRoot) return new Response('no project open', { status: 503 });try {const rel = decodeURIComponent(new URL(request.url).pathname).replace(/^\/+/, '');const abs = resolveInRoot(projectRoot, rel); // throws if it escapes the root- const bytes = await readFile(abs);- return new Response(new Uint8Array(bytes), {- headers: {- 'content-type': httpContentType(abs),- // The play realm (app:// origin) loads project assets cross-scheme via- // <img crossorigin> + fetch; allow it. estella:// is only reachable inside- // the Electron app, so there is no untrusted-web exposure.- 'access-control-allow-origin': '*',- 'cache-control': 'no-store',- },- });+ return await serveProjectFile(abs, request);} catch (err) {return new Response(String(err), { status: 404 });}@@ -752,10 +802,7 @@ async function handleApp(request: Request): Promise<Response> {if (rel.startsWith(PROJECT_PREFIX)) {if (!projectRoot) return new Response('no project open', { status: 503 });const abs = resolveInRoot(projectRoot, rel.slice(PROJECT_PREFIX.length));- const bytes = await readFile(abs);- return new Response(new Uint8Array(bytes), {- headers: { 'content-type': httpContentType(abs), 'access-control-allow-origin': '*', 'cache-control': 'no-store' },- });+ return await serveProjectFile(abs, request);}const abs = path.join(RENDERER_DIST, rel);if (abs !== RENDERER_DIST && !abs.startsWith(RENDERER_DIST + path.sep)) {diff --git a/desktop/electron/mimeTypes.ts b/desktop/electron/mimeTypes.tsindex 76e8e4c5e..251c89f84 100644--- a/desktop/electron/mimeTypes.ts+++ b/desktop/electron/mimeTypes.ts@@ -21,6 +21,7 @@ const HTTP_MIME: Record<string, string> = {png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif',webp: 'image/webp', svg: 'image/svg+xml', ktx2: 'image/ktx2', ico: 'image/x-icon',ogg: 'audio/ogg', mp3: 'audio/mpeg', wav: 'audio/wav', m4a: 'audio/mp4', aac: 'audio/aac',+ mp4: 'video/mp4', m4v: 'video/x-m4v', mov: 'video/quicktime',ttf: 'font/ttf', woff: 'font/woff', woff2: 'font/woff2',};diff --git a/desktop/src/engine/SceneCommands.ts b/desktop/src/engine/SceneCommands.tsindex de911871c..2ad82030d 100644--- a/desktop/src/engine/SceneCommands.ts+++ b/desktop/src/engine/SceneCommands.ts@@ -907,6 +907,22 @@ export class SceneCommandsImpl {/** Add a component (with its registered/schema defaults) to an entity. Undoable. */// Apply an add to the model and return its undo op, or null if it's a no-op// (entity gone / component already present). Shared by the single + batch paths.+ /** Components auto-added alongside another (a lightweight dependency). A Video+ * renders through a Sprite, so adding a Video ensures a Sprite is present. */+ private requiredComponents(compName: string): readonly string[] {+ if (compName === 'Video') return ['Sprite'];+ return [];+ }++ /** The add op for `compName` plus ops for any components it requires. Each op is+ * null when that component already exists, so dependencies never duplicate. */+ private addComponentOpsWithDeps(sourceId: EntityId, compName: string): UndoOp[] {+ return [+ ...this.requiredComponents(compName).map((dep) => this.addComponentOp(sourceId, dep)),+ this.addComponentOp(sourceId, compName),+ ].filter((o): o is UndoOp => !!o);+ }+private addComponentOp(sourceId: EntityId, compName: string): UndoOp | null {const entity = this.model.entityBySource(sourceId);if (!entity || entity.components.some((c) => c.type === compName)) return null;@@ -944,8 +960,8 @@ export class SceneCommandsImpl {}addComponent(sourceId: EntityId, compName: string): void {- const op = this.addComponentOp(sourceId, compName);- if (op) this.history.record(`Add ${prettyLabel(compName)}`, op.forward, op.reverse);+ const ops = this.addComponentOpsWithDeps(sourceId, compName);+ if (ops.length) this.history.batch(`Add ${prettyLabel(compName)}`, ops);}/**@@ -981,7 +997,7 @@ export class SceneCommandsImpl {/** Add a component to many entities (multi-select) as ONE undo step. */addComponentMany(sourceIds: readonly EntityId[], compName: string): void {- const ops = sourceIds.map((id) => this.addComponentOp(id, compName)).filter((o): o is UndoOp => !!o);+ const ops = sourceIds.flatMap((id) => this.addComponentOpsWithDeps(id, compName));this.history.batch(`Add ${prettyLabel(compName)}`, ops);}diff --git a/desktop/src/engine/schema.ts b/desktop/src/engine/schema.tsindex b85e03ea5..4418534bb 100644--- a/desktop/src/engine/schema.ts+++ b/desktop/src/engine/schema.ts@@ -91,6 +91,7 @@ const COMPONENT_CATEGORY: Record<string, string> = {ShapeRenderer: 'Rendering',BitmapText: 'Rendering',TilemapLayer: 'Rendering',+ Video: 'Rendering',Canvas: 'UI',SpineAnimation: 'Animation',ParticleEmitter: 'Effects',diff --git a/desktop/src/panels/Details.tsx b/desktop/src/panels/Details.tsxindex c5fe3786d..79e98a80f 100644--- a/desktop/src/panels/Details.tsx+++ b/desktop/src/panels/Details.tsx@@ -2329,6 +2329,17 @@ function GenericAssetInspector({ path }: { path: string }) {Import Settings sit above the fold (the reason to open an asset). */}{type === 'audio' ? (<AudioWavePreview path={path} />+ ) : type === 'video' ? (+ <div className="cb-prev" style={{ height: 160 }}>+ <video+ src={`estella://project/${path}`}+ controls+ muted+ loop+ playsInline+ style={{ maxWidth: '100%', maxHeight: 160, display: 'block', margin: '0 auto' }}+ />+ </div>) : (<div className="cb-prev" style={{ height: 108 }}><div className="pv">diff --git a/desktop/src/project/ProjectStore.ts b/desktop/src/project/ProjectStore.tsindex 2091ac755..f8af71f93 100644--- a/desktop/src/project/ProjectStore.ts+++ b/desktop/src/project/ProjectStore.ts@@ -105,6 +105,7 @@ function metaTypeToSlot(metaType: string | undefined): string | null {case 'material': return 'material';case 'font': case 'bitmapFont': return 'font';case 'audio': return 'audio';+ case 'video': return 'video';case 'animclip': return 'anim-clip';case 'animation': return 'timeline';case 'tilemap': return 'tilemap';@@ -668,6 +669,10 @@ class ProjectStoreImpl {return assets.loadFont(ref).then((r) => this.recordHandle('font', path, r.handle));case 'audio':return assets.loadAudio(ref);+ case 'video':+ // Video streams at runtime (the video system, play-mode only) — there is+ // no edit-mode handle to preload. The ref is valid; nothing to load here.+ return Promise.resolve();case 'anim-clip':return assets.loadAnimClip(ref); // raw ref: the loader aliases it for component lookupscase 'tilemap':diff --git a/desktop/src/project/assetImporter.ts b/desktop/src/project/assetImporter.tsindex 009645f47..edf24ed50 100644--- a/desktop/src/project/assetImporter.ts+++ b/desktop/src/project/assetImporter.ts@@ -67,6 +67,18 @@ const AUDIO: ImporterFieldSpec[] = [},];+const VIDEO: ImporterFieldSpec[] = [+ {+ key: 'loop', label: 'Loop', type: 'bool', default: true, category: 'Video',+ tooltip: 'Suggested loop state when this clip is assigned to a Video component.',+ },+ { key: 'autoplay', label: 'Autoplay', type: 'bool', default: true, category: 'Video' },+ {+ key: 'muted', label: 'Muted', type: 'bool', default: true, category: 'Video',+ tooltip: 'Muted clips can autoplay; unmuted autoplay is blocked until a user gesture.',+ },+];+const SCENELIKE: ImporterFieldSpec[] = [{key: 'autoMigrate', label: 'Auto Migrate', type: 'bool', default: true, category: 'Import',@@ -81,6 +93,7 @@ export const IMPORTER_SCHEMAS: Record<string, ImporterFieldSpec[]> = {sprite: TEXTURE,spine: SPINE,audio: AUDIO,+ video: VIDEO,scene: SCENELIKE,prefab: SCENELIKE,};diff --git a/desktop/src/project/assetTypes.ts b/desktop/src/project/assetTypes.tsindex b8d307264..c4a9d9a2b 100644--- a/desktop/src/project/assetTypes.ts+++ b/desktop/src/project/assetTypes.ts@@ -13,7 +13,7 @@*/import {Folder, Film, Image, FileImage, PersonStanding, Music,- Component, Blend, FileCode2, Clapperboard, Grid3x3, File, Workflow, Gamepad2, GitBranch, ListTree, Languages, Images,+ Component, Blend, FileCode2, Clapperboard, Grid3x3, File, Workflow, Gamepad2, GitBranch, ListTree, Languages, Images, Video,type LucideIcon,} from 'lucide-react';import type { AssetType } from '@/types';@@ -37,6 +37,7 @@ export const ASSET_TYPES: Record<AssetType, AssetTypeDef> = {sprite: { extensions: ['jpg', 'jpeg', 'gif'], badge: 'IMG', icon: Image, tint: '#7fa6c4' },spine: { extensions: ['atlas', 'skel'], badge: 'SPN', icon: PersonStanding, tint: '#9b8fc0' },audio: { extensions: ['ogg', 'mp3', 'wav', 'aac', 'flac', 'm4a', 'webm'], badge: 'AUD', icon: Music, tint: '#7faf9c' },+ video: { extensions: ['mp4', 'm4v', 'mov'], badge: 'VID', icon: Video, tint: '#c08fb5' },prefab: { extensions: ['esprefab'], badge: 'PFB', icon: Component, tint: '#c2a274' },// .esmaterial is the real extension (the SDK MaterialAssetLoader only loads it);// .esmat is tolerated as a legacy alias (cf. electron/importAssets.ts).diff --git a/desktop/src/types.ts b/desktop/src/types.tsindex 71adffa32..00d6bd5d1 100644--- a/desktop/src/types.ts+++ b/desktop/src/types.ts@@ -155,6 +155,7 @@ export type AssetType =| 'texture'| 'spine'| 'audio'+ | 'video'| 'prefab'| 'material'| 'materialgraph'diff --git a/desktop/tests/video-asset.test.ts b/desktop/tests/video-asset.test.tsnew file mode 100644index 000000000..4e65e6db6--- /dev/null+++ b/desktop/tests/video-asset.test.ts@@ -0,0 +1,35 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file Guards the video asset-type wiring (M1): the content-browser type table,+ * the `.meta` type vocabulary, and the import-settings schema all recognize+ * video containers, while `webm` stays with audio (the disambiguated+ * extension the two families share).+ */+import { describe, it, expect } from 'vitest';+import { assetTypeOf } from '../src/project/assetTypes';+import { metaTypeFor } from '../electron/assetMeta';+import { importerDefaults, hasImporterSettings } from '../src/project/assetImporter';++describe('video asset type', () => {+ it('classifies video containers as "video" in the content browser', () => {+ expect(assetTypeOf('intro.mp4')).toBe('video');+ expect(assetTypeOf('clip.m4v')).toBe('video');+ expect(assetTypeOf('capture.MOV')).toBe('video');+ });++ it('keeps the shared webm extension with audio (disambiguated by .meta)', () => {+ expect(assetTypeOf('sound.webm')).toBe('audio');+ });++ it('mints a "video" .meta type on import', () => {+ expect(metaTypeFor('intro.mp4')).toBe('video');+ expect(metaTypeFor('clip.mov')).toBe('video');+ expect(metaTypeFor('sound.webm')).toBe('audio');+ });++ it('exposes video import settings (playback defaults)', () => {+ expect(hasImporterSettings('video')).toBe(true);+ expect(importerDefaults('video')).toEqual({ loop: true, autoplay: true, muted: true });+ });+});diff --git a/sdk/etc/index.api.md b/sdk/etc/index.api.mdindex eead2c412..d9f390586 100644--- a/sdk/etc/index.api.md+++ b/sdk/etc/index.api.md@@ -458,7 +458,7 @@ static prototype: ArrayDataSource<any>## AssetFieldType — type```-'texture' | 'material' | 'font' | 'anim-clip' | 'audio' | 'tilemap' | 'tileset' | 'timeline' | 'statemachine' | 'behaviortree'+'texture' | 'material' | 'font' | 'anim-clip' | 'audio' | 'video' | 'tilemap' | 'tileset' | 'timeline' | 'statemachine' | 'behaviortree'```## AssetPlugin — class@@ -2222,7 +2222,7 @@ undefineMaterial: (materialId: number) => void## EditorAssetType — type```| 'texture' | 'material' | 'shader' | 'spine-atlas' | 'spine-skeleton'- | 'bitmap-font' | 'prefab' | 'json' | 'audio' | 'scene' | 'anim-clip'+ | 'bitmap-font' | 'prefab' | 'json' | 'audio' | 'video' | 'scene' | 'anim-clip'| 'tilemap' | 'timeline'| 'unknown'```diff --git a/sdk/etc/index.node.api.md b/sdk/etc/index.node.api.mdindex 4a3e5cbdd..3423ce598 100644--- a/sdk/etc/index.node.api.md+++ b/sdk/etc/index.node.api.md@@ -453,7 +453,7 @@ static prototype: ArrayDataSource<any>## AssetFieldType — type```-'texture' | 'material' | 'font' | 'anim-clip' | 'audio' | 'tilemap' | 'tileset' | 'timeline' | 'statemachine' | 'behaviortree'+'texture' | 'material' | 'font' | 'anim-clip' | 'audio' | 'video' | 'tilemap' | 'tileset' | 'timeline' | 'statemachine' | 'behaviortree'```## AssetPlugin — class@@ -2217,7 +2217,7 @@ undefineMaterial: (materialId: number) => void## EditorAssetType — type```| 'texture' | 'material' | 'shader' | 'spine-atlas' | 'spine-skeleton'- | 'bitmap-font' | 'prefab' | 'json' | 'audio' | 'scene' | 'anim-clip'+ | 'bitmap-font' | 'prefab' | 'json' | 'audio' | 'video' | 'scene' | 'anim-clip'| 'tilemap' | 'timeline'| 'unknown'```diff --git a/sdk/src/asset/AssetFieldRegistry.ts b/sdk/src/asset/AssetFieldRegistry.tsindex 7d7abd717..0c32863fd 100644--- a/sdk/src/asset/AssetFieldRegistry.ts+++ b/sdk/src/asset/AssetFieldRegistry.ts@@ -1,6 +1,6 @@// SPDX-License-Identifier: Apache-2.0// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team-export type AssetFieldType = 'texture' | 'material' | 'font' | 'anim-clip' | 'audio' | 'tilemap' | 'tileset' | 'timeline' | 'statemachine' | 'behaviortree';+export type AssetFieldType = 'texture' | 'material' | 'font' | 'anim-clip' | 'audio' | 'video' | 'tilemap' | 'tileset' | 'timeline' | 'statemachine' | 'behaviortree';export interface AssetFieldDescriptor {field: string;@@ -85,6 +85,10 @@ export function initBuiltinAssetFields(): void {{ field: 'clip', type: 'audio' },]);+ registerAssetFields('Video', [+ { field: 'source', type: 'video' },+ ]);+registerAssetFields('ParticleEmitter', [{ field: 'texture', type: 'texture' },{ field: 'material', type: 'material' },diff --git a/sdk/src/assetTypes.ts b/sdk/src/assetTypes.tsindex 6805be0ef..2f4d21e75 100644--- a/sdk/src/assetTypes.ts+++ b/sdk/src/assetTypes.ts@@ -8,7 +8,7 @@ export type AddressableAssetType =export type EditorAssetType =| 'texture' | 'material' | 'shader' | 'spine-atlas' | 'spine-skeleton'- | 'bitmap-font' | 'prefab' | 'json' | 'audio' | 'scene' | 'anim-clip'+ | 'bitmap-font' | 'prefab' | 'json' | 'audio' | 'video' | 'scene' | 'anim-clip'| 'tilemap' | 'timeline'| 'unknown';@@ -33,6 +33,10 @@ const ASSET_TYPE_REGISTRY: readonly AssetTypeEntry[] = [// extensions — KTX2 textures are fs-read (transcoded), not image-decoded.{ extensions: ['ktx2'], contentType: 'binary', editorType: 'texture', addressableType: 'texture', wechatPackInclude: true, hasTransitiveDeps: false },{ extensions: ['mp3', 'wav', 'ogg', 'aac', 'flac', 'm4a', 'webm'], contentType: 'audio', editorType: 'audio', addressableType: 'audio', wechatPackInclude: false, hasTransitiveDeps: false },+ // Video container, streamed + decoded by the video system (HTMLVideoElement /+ // wx.createVideoDecoder). `webm` stays with audio above to avoid the ambiguity;+ // a .webm video is disambiguated by its `.meta` type. Cooked/copied as binary.+ { extensions: ['mp4', 'm4v', 'mov'], contentType: 'binary', editorType: 'video', addressableType: 'binary', wechatPackInclude: false, hasTransitiveDeps: false },{ extensions: ['esmaterial'], contentType: 'json', editorType: 'material', addressableType: 'material', wechatPackInclude: true, hasTransitiveDeps: true },{ extensions: ['esshader'], contentType: 'text', editorType: 'shader', addressableType: null, wechatPackInclude: false, hasTransitiveDeps: false },{ extensions: ['atlas'], contentType: 'text', editorType: 'spine-atlas', addressableType: 'binary', wechatPackInclude: true, hasTransitiveDeps: true },@@ -71,6 +75,9 @@ const MIME_MAP: Record<string, string> = {m4a: 'audio/mp4',flac: 'audio/flac',webm: 'audio/webm',+ mp4: 'video/mp4',+ m4v: 'video/x-m4v',+ mov: 'video/quicktime',json: 'application/json',atlas: 'text/plain',skel: 'application/octet-stream',diff --git a/sdk/src/scene.ts b/sdk/src/scene.tsindex bbe58bc3e..f5734d9ec 100644--- a/sdk/src/scene.ts+++ b/sdk/src/scene.ts@@ -105,7 +105,7 @@ export class MissingAssetsError extends Error {// Asset Field Types// =============================================================================-export type AssetFieldType = 'texture' | 'material' | 'font' | 'anim-clip' | 'audio' | 'tilemap' | 'tileset' | 'timeline' | 'statemachine' | 'behaviortree';+export type AssetFieldType = 'texture' | 'material' | 'font' | 'anim-clip' | 'audio' | 'video' | 'tilemap' | 'tileset' | 'timeline' | 'statemachine' | 'behaviortree';// =============================================================================// Component Query Helpers (read from self-describing ComponentDef)diff --git a/sdk/src/video/VideoComponents.ts b/sdk/src/video/VideoComponents.tsindex 828f7e623..c7361c8fa 100644--- a/sdk/src/video/VideoComponents.ts+++ b/sdk/src/video/VideoComponents.ts@@ -40,4 +40,6 @@ export const Video = defineComponent<VideoData>('Video', {playbackRate: 1.0,fitSize: true,enabled: true,+}, {+ assetFields: [{ field: 'source', type: 'video' }],});