A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
feat(video): cook transcodes to .esv + m4a audio track slaved as the clock (V2)
esengine7dbbff069b81
diff --git a/NOTICE b/NOTICEindex 905527f50..db8f87be3 100644--- a/NOTICE+++ b/NOTICE@@ -35,6 +35,13 @@ license text is included in its directory under third_party/.pl_mpeg third_party/pl_mpeg MITSpine Runtimes third_party/spine-runtimes-* Spine Runtimes License+The desktop editor additionally bundles an FFmpeg binary (via the ffmpeg-static+npm package; FFmpeg is a trademark of Fabrice Bellard). It is licensed under the+GPL (see the LICENSE file inside the ffmpeg-static package) and is invoked by+the editor's asset cook strictly as a separate external process — it is not+linked into, nor does its license extend to, Estella itself. It does not ship+in exported games. Set ESTELLA_FFMPEG to use a system FFmpeg instead.+IMPORTANT — Spine Runtimes are NOT open source. Relicensing Estella to Apache-2.0does NOT change this: the Spine Runtimes remain governed by the Spine RuntimesLicense Agreement from Esoteric Software (see third_party/spine-runtimes-*/LICENSE).diff --git a/desktop/electron-builder.yml b/desktop/electron-builder.ymlindex 4d5327a18..18dc88a6e 100644--- a/desktop/electron-builder.yml+++ b/desktop/electron-builder.yml@@ -25,6 +25,9 @@ files:# directory copy, which cannot source from inside an asar.asarUnpack:- "**/node_modules/{esbuild,@esbuild}/**"+ # ffmpeg (video cook) spawns as a subprocess — executables cannot run from+ # inside the asar (videoCook resolves the app.asar.unpacked twin).+ - "**/node_modules/ffmpeg-static/**"- dist-electron/hosts/**# The MCP front: an MCP client runs this with plain node (node cannot read# inside an asar), which spawns/attaches the editor as the MCP host.diff --git a/desktop/electron/cookAssets.ts b/desktop/electron/cookAssets.tsindex b369a9924..099abafca 100644--- a/desktop/electron/cookAssets.ts+++ b/desktop/electron/cookAssets.ts@@ -197,13 +197,14 @@ export async function cookAssets(opts: {entryScenes: string[]; outDir: string;contentAddressed?: boolean; compressTextures?: boolean; atlasTextures?: boolean;- compressAudio?: boolean;+ compressAudio?: boolean; transcodeVideo?: boolean;},): Promise<CookResult> {const contentAddressed = opts.contentAddressed ?? false;const compressTextures = opts.compressTextures ?? false;const atlasTextures = opts.atlasTextures ?? false;const compressAudio = opts.compressAudio ?? false;+ const transcodeVideo = opts.transcodeVideo ?? false;const { index } = await scanAssetDatabase(root, { write: false, adopt: false });const byUuid = new Map(index.entries.map((e) => [e.uuid, e]));const byPath = new Map(index.entries.map((e) => [e.path, e]));@@ -267,6 +268,12 @@ export async function cookAssets(if (compressAudio) {audioEnc = await import('./audioCook');}+ // Video → MPEG-1 `.esv` + `.m4a` audio sibling, for targets whose only video+ // path is the wasm decoder (WeChat). ffmpeg loads lazily the same way.+ let videoEnc: typeof import('./videoCook') | null = null;+ if (transcodeVideo) {+ videoEnc = await import('./videoCook');+ }const manifestEntries: CookManifestEntry[] = [];const staged = new Set<string>(); // staged output paths, for content-addressed dedup@@ -391,6 +398,21 @@ export async function cookAssets(}}}+ // Video → `.esv` (MPEG-1 for the wasm decoder) + audio-track sibling.+ // `.esv` sources are already in the cooked format and pass through.+ let videoAudio: Uint8Array | null = null;+ if (videoEnc && entry.type === 'video' && ext.toLowerCase() !== '.esv') {+ const settings = videoEnc.videoImportSettings(entry.importer);+ const res = await videoEnc.transcodeVideoForWasm(path.join(root, entry.path), settings);+ warnings.push(...res.warnings.map((w) => `${entry.path}: ${w}`));+ if (res.esv) {+ data = res.esv;+ ext = '.esv';+ videoAudio = res.audio;+ } else {+ warnings.push(`${entry.path}: shipped untranscoded — this video will NOT play on the wasm decode path (WeChat)`);+ }+ }// Materials: relative refs → logical project paths (see rewriteMaterialRefs).if (ext.toLowerCase() === '.esmaterial') {try {@@ -450,6 +472,27 @@ export async function cookAssets(group,...(compressedFormats ? { compressedFormats } : {}),});+ // The video's audio track ships as an `<staged>.m4a` SIBLING of the+ // `.esv` — appended (not extension-swapped) so it can never collide with+ // a real project `.m4a`, and the runtime derives it from the resolved+ // video URL with no metadata channel. Its own manifest entry keeps+ // addressable/CDN flows aware of the file.+ if (videoAudio) {+ const audioOutRel = `${outRel}.m4a`;+ if (!staged.has(audioOutRel)) {+ await writeFile(path.join(absOut, audioOutRel), videoAudio);+ staged.add(audioOutRel);+ }+ manifestEntries.push({+ uuid: `${entry.uuid}-audio`,+ path: audioOutRel,+ sourcePath: `${entry.path}.m4a`,+ type: 'audio',+ contentHash: contentHashHex(videoAudio),+ size: videoAudio.byteLength,+ group,+ });+ }} catch (err) {warnings.push(`copy failed ${entry.path}: ${err instanceof Error ? err.message : String(err)}`);}diff --git a/desktop/electron/exportWeChat.ts b/desktop/electron/exportWeChat.tsindex 3f8e481ff..454287acc 100644--- a/desktop/electron/exportWeChat.ts+++ b/desktop/electron/exportWeChat.ts@@ -303,7 +303,7 @@ export async function exportWeChat(opts: {// manifest. KTX2 textures are fine here: the scan below sees the staged// .ktx2 files and ships the Basis transcoder side module with them.progress({ phase: 'Cooking assets' });- const cook = await cookAssets(opts.root, { entryScenes: scenes.map((s) => s.path), outDir: absOut, contentAddressed: opts.contentAddressed, compressTextures: opts.compressTextures, compressAudio: opts.compressAudio, atlasTextures: opts.atlasTextures });+ const cook = await cookAssets(opts.root, { entryScenes: scenes.map((s) => s.path), outDir: absOut, contentAddressed: opts.contentAddressed, compressTextures: opts.compressTextures, compressAudio: opts.compressAudio, atlasTextures: opts.atlasTextures, transcodeVideo: true });warnings.push(...cook.warnings);// 1a. Restage for WeChat's code-package suffix whitelist (it has no `ktx2`diff --git a/desktop/electron/videoCook.ts b/desktop/electron/videoCook.tsnew file mode 100644index 000000000..9da5fd615--- /dev/null+++ b/desktop/electron/videoCook.ts@@ -0,0 +1,124 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * @file videoCook.ts — cook-time video transcode for the wasm decode path.+ * WeChat's only video backend is the engine-owned MPEG-1 decoder+ * (videodec side module), so the cook turns each authored video into a+ * codec-tagged `.esv` (MPEG-1 Program Stream) plus an `.m4a` audio-track+ * sibling played through the audio pipeline. Transcoding shells out to+ * the bundled ffmpeg binary (ffmpeg-static) — a separate process, loaded+ * lazily and only when a cook actually ships video.+ */+import { spawn } from 'node:child_process';+import { mkdtemp, readFile, rm } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import path from 'node:path';++/** Resolve the ffmpeg binary: env override first, then the bundled static+ * build. Inside a packaged app the binary must run from the unpacked twin of+ * the asar (executables cannot spawn from an archive). Null if unavailable. */+export async function resolveFfmpeg(): Promise<string | null> {+ if (process.env.ESTELLA_FFMPEG) return process.env.ESTELLA_FFMPEG;+ try {+ const mod = await import('ffmpeg-static');+ const p = (mod.default ?? mod) as unknown as string | null;+ return p ? p.replace(`app.asar${path.sep}`, `app.asar.unpacked${path.sep}`) : null;+ } catch {+ return null;+ }+}++function run(bin: string, args: string[]): Promise<{ code: number; stderr: string }> {+ return new Promise((resolve, reject) => {+ const child = spawn(bin, args, { stdio: ['ignore', 'ignore', 'pipe'] });+ let stderr = '';+ child.stderr.on('data', (d: Buffer) => { stderr += d.toString(); });+ child.on('error', reject);+ child.on('close', (code) => resolve({ code: code ?? -1, stderr }));+ });+}++export interface VideoImportSettings {+ /** mpeg1video quantizer (2 = near-lossless, larger = smaller file). */+ quality: number;+ audioBitrateKbps: number;+}++/** Per-asset video import settings from the `.meta` importer block (tolerant). */+export function videoImportSettings(importer: Record<string, unknown> | undefined): VideoImportSettings {+ const q = importer?.quality;+ const kbps = importer?.audioBitrateKbps;+ return {+ quality: typeof q === 'number' && q >= 2 && q <= 31 ? Math.round(q) : 4,+ audioBitrateKbps: kbps === 96 || kbps === 128 || kbps === 192 ? (kbps as number) : 128,+ };+}++export interface VideoCookResult {+ /** MPEG-1 PS bytes, or null when the transcode failed. */+ esv: Uint8Array | null;+ /** AAC (m4a) audio track, or null when the source has none. */+ audio: Uint8Array | null;+ warnings: string[];+}++// MPEG-1 dimensions must be even (macroblock chroma), and its frame rate is a+// fixed table — sources with a rate outside it are resampled on retry.+const EVEN_SCALE = 'scale=trunc(iw/2)*2:trunc(ih/2)*2';++/**+ * Transcode `srcAbs` into the wasm decode path's artifacts. Video failures are+ * reported (esv null); a missing audio track is normal and silent.+ */+export async function transcodeVideoForWasm(+ srcAbs: string,+ settings: VideoImportSettings,+): Promise<VideoCookResult> {+ const warnings: string[] = [];+ const ffmpeg = await resolveFfmpeg();+ if (!ffmpeg) {+ return { esv: null, audio: null, warnings: ['ffmpeg unavailable — cannot transcode video for the wasm decoder'] };+ }+ const tmp = await mkdtemp(path.join(tmpdir(), 'estella-video-'));+ try {+ const esvPath = path.join(tmp, 'out.esv');+ const videoArgs = (extraFilter: string) => [+ '-y', '-i', srcAbs, '-an',+ '-c:v', 'mpeg1video', '-q:v', String(settings.quality),+ '-vf', extraFilter ? `${EVEN_SCALE},${extraFilter}` : EVEN_SCALE,+ '-f', 'mpeg', esvPath,+ ];+ let res = await run(ffmpeg, videoArgs(''));+ if (res.code !== 0 && /fps|frame ?rate/i.test(res.stderr)) {+ // Source frame rate is outside MPEG-1's table (or variable) — resample.+ res = await run(ffmpeg, videoArgs('fps=30'));+ }+ let esv: Uint8Array | null = null;+ if (res.code === 0) {+ esv = new Uint8Array(await readFile(esvPath));+ } else {+ warnings.push(`video transcode failed: ${lastLine(res.stderr)}`);+ }++ const m4aPath = path.join(tmp, 'out.m4a');+ const audioRes = await run(ffmpeg, [+ '-y', '-i', srcAbs, '-vn',+ '-c:a', 'aac', '-b:a', `${settings.audioBitrateKbps}k`,+ '-f', 'ipod', m4aPath,+ ]);+ let audio: Uint8Array | null = null;+ if (audioRes.code === 0) {+ audio = new Uint8Array(await readFile(m4aPath));+ } else if (!/does not contain any stream|Stream map .* matches no streams|no audio/i.test(audioRes.stderr)) {+ warnings.push(`audio track extract failed: ${lastLine(audioRes.stderr)}`);+ }+ return { esv, audio, warnings };+ } finally {+ await rm(tmp, { recursive: true, force: true });+ }+}++function lastLine(stderr: string): string {+ const lines = stderr.trim().split('\n');+ return lines[lines.length - 1] ?? 'unknown ffmpeg error';+}diff --git a/desktop/package.json b/desktop/package.jsonindex d934ddcda..2a3f59fa4 100644--- a/desktop/package.json+++ b/desktop/package.json@@ -68,6 +68,7 @@"dockview": "^6.6.1","esbuild": "^0.27.0","esengine": "workspace:*",+ "ffmpeg-static": "^5.3.0","lucide-react": "^1.21.0","pngjs": "^7.0.0","react": "^19.2.0",diff --git a/desktop/tests/video-cook.test.ts b/desktop/tests/video-cook.test.tsnew file mode 100644index 000000000..488953dce--- /dev/null+++ b/desktop/tests/video-cook.test.ts@@ -0,0 +1,159 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team+/**+ * Cook-time video: mp4 → `.esv` (MPEG-1 PS for the wasm decoder) + `.m4a`+ * audio-track sibling, through the real bundled ffmpeg. Sources are generated+ * at test time with the same binary (lavfi), so no binary fixtures ship.+ * When the videodec wasm is built locally, the cooked `.esv` is decoded+ * end-to-end and its frames asserted.+ */+import { describe, it, expect, beforeAll } from 'vitest';+import { execFile } from 'node:child_process';+import { promisify } from 'node:util';+import { existsSync, readFileSync } from 'node:fs';+import { mkdtemp } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import path from 'node:path';+import { resolveFfmpeg, transcodeVideoForWasm, videoImportSettings } from '../electron/videoCook';+import { cookAssets } from '../electron/cookAssets';++const execFileAsync = promisify(execFile);++const WASM_DIR = path.resolve(__dirname, '../../build/wasm/web');+const HAS_VIDEODEC = existsSync(path.join(WASM_DIR, 'videodec.wasm'));++const ffmpeg = await resolveFfmpeg();+let tmp = '';++beforeAll(async () => {+ tmp = await mkdtemp(path.join(tmpdir(), 'estella-videocook-test-'));+});++/** Generate a source clip: solid lime video, optional 440Hz tone, ODD width. */+async function makeSource(name: string, withAudio: boolean): Promise<string> {+ const out = path.join(tmp, name);+ const args = ['-y', '-f', 'lavfi', '-i', 'color=c=lime:s=63x32:r=25:d=1'];+ if (withAudio) args.push('-f', 'lavfi', '-i', 'sine=frequency=440:duration=1', '-c:a', 'aac', '-shortest');+ args.push('-c:v', 'libx264', '-pix_fmt', 'yuv420p', out);+ await execFileAsync(ffmpeg!, args);+ return out;+}++describe('videoImportSettings', () => {+ it('defaults and clamps', () => {+ expect(videoImportSettings(undefined)).toEqual({ quality: 4, audioBitrateKbps: 128 });+ expect(videoImportSettings({ quality: 8, audioBitrateKbps: 192 })).toEqual({ quality: 8, audioBitrateKbps: 192 });+ expect(videoImportSettings({ quality: 99, audioBitrateKbps: 7 })).toEqual({ quality: 4, audioBitrateKbps: 128 });+ });+});++describe.skipIf(!ffmpeg)('transcodeVideoForWasm', () => {+ it('produces an MPEG-PS .esv and an .m4a audio track', async () => {+ const src = await makeSource('with-audio.mp4', true);+ const res = await transcodeVideoForWasm(src, videoImportSettings(undefined));+ expect(res.warnings).toEqual([]);+ expect(res.esv).not.toBeNull();+ // MPEG-PS pack header: 00 00 01 BA.+ expect(Array.from(res.esv!.subarray(0, 4))).toEqual([0, 0, 1, 0xba]);+ expect(res.audio).not.toBeNull();+ // MP4/M4A container: 'ftyp' at offset 4.+ expect(Buffer.from(res.audio!.subarray(4, 8)).toString('ascii')).toBe('ftyp');+ });++ it('a silent source yields esv with no audio sibling and no warnings', async () => {+ const src = await makeSource('silent.mp4', false);+ const res = await transcodeVideoForWasm(src, videoImportSettings(undefined));+ expect(res.warnings).toEqual([]);+ expect(res.esv).not.toBeNull();+ expect(res.audio).toBeNull();+ });++ it('reports failure for an undecodable source', async () => {+ const bogus = path.join(tmp, 'bogus.mp4');+ const fs = await import('node:fs/promises');+ await fs.writeFile(bogus, Buffer.alloc(256, 0x42));+ const res = await transcodeVideoForWasm(bogus, videoImportSettings(undefined));+ expect(res.esv).toBeNull();+ expect(res.warnings.length).toBeGreaterThan(0);+ });++ it.skipIf(!HAS_VIDEODEC)('cooked .esv decodes in the real videodec module with evened dimensions', async () => {+ const src = await makeSource('roundtrip.mp4', false);+ const res = await transcodeVideoForWasm(src, videoImportSettings(undefined));+ expect(res.esv).not.toBeNull();++ const factory = (await import(path.join(WASM_DIR, 'videodec.js'))).default;+ const mod = await factory({ wasmBinary: readFileSync(path.join(WASM_DIR, 'videodec.wasm')) });+ const ptr = mod._malloc(res.esv!.length);+ mod.HEAPU8.set(res.esv!, ptr);+ const h = mod._es_video_open(ptr, res.esv!.length);+ expect(h).toBeGreaterThan(0);+ // The odd 63px source width is evened for MPEG-1 macroblocks.+ expect(mod._es_video_width(h)).toBe(62);+ expect(mod._es_video_height(h)).toBe(32);++ expect(mod._es_video_advance(h, 0.05)).toBe(1);+ const out = mod._malloc(62 * 32 * 4);+ expect(mod._es_video_frame_rgba(h, out, 62 * 32 * 4)).toBe(1);+ const px = out + ((16 * 62) + 31) * 4;+ const [r, g, b, a] = Array.from(mod.HEAPU8.subarray(px, px + 4)) as number[];+ expect(g).toBeGreaterThan(200);+ expect(r).toBeLessThan(60);+ expect(b).toBeLessThan(60);+ expect(a).toBe(255);+ mod._free(out);+ mod._es_video_close(h);+ });+});++describe.skipIf(!ffmpeg)('cookAssets video wiring', () => {+ const VIDEO_UUID = 'dddddddd-dddd-4ddd-8ddd-dddddddddddd';+ const SCENE_UUID = 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee';++ async function makeProject(): Promise<string> {+ const fs = await import('node:fs');+ const root = await mkdtemp(path.join(tmpdir(), 'estella-videowire-'));+ const write = (rel: string, body: string | Uint8Array, meta: object) => {+ const abs = path.join(root, rel);+ fs.mkdirSync(path.dirname(abs), { recursive: true });+ fs.writeFileSync(abs, body);+ fs.writeFileSync(`${abs}.meta`, JSON.stringify({ version: '2.0', importer: {}, ...meta }));+ };+ const clip = readFileSync(await makeSource('wire-src.mp4', true));+ write('assets/videos/clip.mp4', clip, { uuid: VIDEO_UUID, type: 'video' });+ write('assets/scenes/main.esscene', JSON.stringify({+ version: '1.0', name: 'v',+ entities: [{ id: 1, name: 'V', parent: null, children: [], components: [+ { type: 'Sprite', data: { texture: 0 } },+ { type: 'Video', data: { source: `@uuid:${VIDEO_UUID}` } },+ ] }],+ }), { uuid: SCENE_UUID, type: 'scene' });+ return root;+ }++ it('transcodeVideo stages .esv plus the .m4a sibling, both in the manifest', async () => {+ const root = await makeProject();+ const res = await cookAssets(root, {+ entryScenes: ['assets/scenes/main.esscene'], outDir: 'build', transcodeVideo: true,+ });+ expect(res.warnings).toEqual([]);+ const manifest = JSON.parse(readFileSync(path.join(root, 'build/assets.manifest.json'), 'utf8'));+ const video = manifest.entries.find((e: { uuid: string }) => e.uuid === VIDEO_UUID);+ expect(video.path).toBe('assets/videos/clip.esv');+ expect(video.sourcePath).toBe('assets/videos/clip.mp4');+ const audio = manifest.entries.find((e: { uuid: string }) => e.uuid === `${VIDEO_UUID}-audio`);+ expect(audio.path).toBe('assets/videos/clip.esv.m4a');+ expect(audio.type).toBe('audio');+ expect(existsSync(path.join(root, 'build/assets/videos/clip.esv'))).toBe(true);+ expect(existsSync(path.join(root, 'build/assets/videos/clip.esv.m4a'))).toBe(true);+ });++ it('without transcodeVideo the mp4 passes through untouched', async () => {+ const root = await makeProject();+ await cookAssets(root, { entryScenes: ['assets/scenes/main.esscene'], outDir: 'build' });+ const manifest = JSON.parse(readFileSync(path.join(root, 'build/assets.manifest.json'), 'utf8'));+ const video = manifest.entries.find((e: { uuid: string }) => e.uuid === VIDEO_UUID);+ expect(video.path).toBe('assets/videos/clip.mp4');+ expect(manifest.entries.find((e: { uuid: string }) => e.uuid === `${VIDEO_UUID}-audio`)).toBeUndefined();+ });+});diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 538a5d1f9..871bb8119 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -35,6 +35,9 @@ importers:esengine:specifier: workspace:*version: link:../sdk+ ffmpeg-static:+ specifier: ^5.3.0+ version: 5.3.0lucide-react:specifier: ^1.21.0version: 1.21.0([email protected])@@ -206,6 +209,10 @@ packages:resolution: {integrity: sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==}engines: {node: '>=18'}+ '@derhuerst/[email protected]':+ resolution: {integrity: sha512-F9rL9k9Xjf5blCz8HsJRO4diy111cayL2vkY2XE4r4t3n0yPXVYy3KD3nJ1qbrSn9743UWSXH4IwuCa/HWlGFw==}+ engines: {node: '>=6.0.0'}+'@electron-internal/[email protected]':resolution: {integrity: sha512-OjKpjB7gohtEjZiq6nDx1egqjZJhGPN1iFOIED+NFhB/MMkXw/XRcHjh1DGXKT5z2W9eW7Jy2UKU3gpjvusFTQ==}engines: {node: '>=22.12.0'}@@ -951,6 +958,9 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==}+ '@types/[email protected]':+ resolution: {integrity: sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==}+'@types/[email protected]':resolution: {integrity: sha512-fRa09kZTgu8o71KFcDjUFuc7F+dEbZYZmkI0mg5YBTRs0yMKjYHsq/c0urDKeDb+D5qVgXOdFcuu+DZPKOITwA==}@@ -1052,6 +1062,10 @@ packages:engines: {node: '>=0.4.0'}hasBin: true+ resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}+ engines: {node: '>= 6.0.0'}+resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==}engines: {node: '>= 14'}@@ -1204,6 +1218,9 @@ packages:resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==}engines: {node: '>= 0.4'}+ resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==}+resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==}engines: {node: '>=18'}@@ -1287,6 +1304,10 @@ packages:resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}+ resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==}+ engines: {'0': node >= 6.0}+resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==}@@ -1579,6 +1600,10 @@ packages:resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==}+ resolution: {integrity: sha512-H+K6sW6TiIX6VGend0KQwthe+kaceeH/luE8dIZyOP35ik7ahYojDuqlTV1bOrtEwl01sy2HFNGQfi5IDJvotg==}+ engines: {node: '>=16'}+resolution: {integrity: sha512-5giy2PkLYY1cP39p17Ech+2xlpTRL9HLspOfEgm0L6CwBXBTgsK5ou0JtzYuepxkaQ/tvhCFIJ5uXo0OrM2DxA==}@@ -1745,10 +1770,17 @@ packages:resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==}engines: {node: '>= 14'}+ resolution: {integrity: sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==}+resolution: {integrity: sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==}engines: {node: '>=10.19.0'}+ resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}+ engines: {node: '>= 6'}+resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==}engines: {node: '>= 14'}@@ -2209,6 +2241,9 @@ packages:resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==}+ resolution: {integrity: sha512-60zvsJReQPX5/QP0Kzfd/VrpjScIQ7SHBW6bFCYfEP+fp0Eppr1SHhIO5nd1PjZtvclzSzES9D/p5nFJurwfWg==}+resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==}@@ -2372,6 +2407,10 @@ packages:resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==}+ resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}+ engines: {node: '>= 6'}+resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}engines: {node: '>=8.10.0'}@@ -2716,6 +2755,9 @@ packages:resolution: {integrity: sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==}engines: {node: '>= 18'}+ resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==}+resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==}engines: {node: '>=14.17'}@@ -3081,6 +3123,13 @@ snapshots:'@csstools/[email protected]':optional: true+ '@derhuerst/[email protected]':+ dependencies:+ caseless: 0.12.0+ concat-stream: 2.0.0+ http-response-object: 3.0.2+ parse-cache-control: 1.0.1+'@electron-internal/[email protected]': {}'@electron/[email protected]':@@ -3646,6 +3695,8 @@ snapshots:'@types/[email protected]': {}+ '@types/[email protected]': {}+'@types/[email protected]':dependencies:undici-types: 7.18.2@@ -3761,6 +3812,12 @@ snapshots:+ dependencies:+ debug: 4.4.3+ transitivePeerDependencies:+ - supports-color+@@ -3961,6 +4018,8 @@ snapshots:call-bind-apply-helpers: 1.0.2get-intrinsic: 1.3.0+ [email protected]: {}+dependencies:assertion-error: 2.0.1@@ -4037,6 +4096,13 @@ snapshots:+ dependencies:+ buffer-from: 1.1.2+ inherits: 2.0.4+ readable-stream: 3.6.2+ typedarray: 0.0.6+@@ -4401,6 +4467,15 @@ snapshots:+ dependencies:+ '@derhuerst/http-basic': 8.2.4+ env-paths: 2.2.1+ https-proxy-agent: 5.0.1+ progress: 2.0.3+ transitivePeerDependencies:+ - supports-color+dependencies:minimatch: 5.1.9@@ -4618,11 +4693,22 @@ snapshots:transitivePeerDependencies:- supports-color+ dependencies:+ '@types/node': 10.17.60+dependencies:quick-lru: 5.1.1resolve-alpn: 1.2.1+ dependencies:+ agent-base: 6.0.2+ debug: 4.4.3+ transitivePeerDependencies:+ - supports-color+dependencies:agent-base: 7.1.4@@ -5036,6 +5122,8 @@ snapshots:+ [email protected]: {}+dependencies:entities: 6.0.1@@ -5200,6 +5288,12 @@ snapshots:string_decoder: 1.1.1util-deprecate: 1.0.2+ dependencies:+ inherits: 2.0.4+ string_decoder: 1.1.1+ util-deprecate: 1.0.2+dependencies:picomatch: 2.3.1@@ -5610,6 +5704,8 @@ snapshots:media-typer: 1.1.0mime-types: 3.0.2+ [email protected]: {}+diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex f9b6e0144..6ac9f3e3a 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -4,3 +4,4 @@ packages:onlyBuiltDependencies:- esbuild- electron+ - ffmpeg-staticdiff --git a/sdk/etc/index.api.md b/sdk/etc/index.api.mdindex 22f20fe72..d47bac42a 100644--- a/sdk/etc/index.api.md+++ b/sdk/etc/index.api.md@@ -562,6 +562,7 @@ invalidateBuffer: (url: string) => booleanmuteBus: (busName: string, muted: boolean) => voidplayBGM: (url: string, config?: { volume?: number; fadeIn?: number; crossFade?: number; }) => voidplaySFX: (url: string, config?: { volume?: number; pitch?: number; pan?: number; priority?: number; }) => AudioHandle+playTrack: (url: string, config?: PlayConfig) => Promise<AudioHandle | null>preload: (url: string) => Promise<void>preloadAll: (urls: string[]) => Promise<void>preloadFromData: (url: string, data: ArrayBuffer) => Promise<void>diff --git a/sdk/etc/index.node.api.md b/sdk/etc/index.node.api.mdindex cdd39eb09..fc5c6aaf1 100644--- a/sdk/etc/index.node.api.md+++ b/sdk/etc/index.node.api.md@@ -557,6 +557,7 @@ invalidateBuffer: (url: string) => booleanmuteBus: (busName: string, muted: boolean) => voidplayBGM: (url: string, config?: { volume?: number; fadeIn?: number; crossFade?: number; }) => voidplaySFX: (url: string, config?: { volume?: number; pitch?: number; pan?: number; priority?: number; }) => AudioHandle+playTrack: (url: string, config?: PlayConfig) => Promise<AudioHandle | null>preload: (url: string) => Promise<void>preloadAll: (urls: string[]) => Promise<void>preloadFromData: (url: string, data: ArrayBuffer) => Promise<void>diff --git a/sdk/src/audio/Audio.ts b/sdk/src/audio/Audio.tsindex 079490c1d..b965237a8 100644--- a/sdk/src/audio/Audio.ts+++ b/sdk/src/audio/Audio.ts@@ -1,6 +1,6 @@// SPDX-License-Identifier: Apache-2.0// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team-import type { PlatformAudioBackend, AudioBufferHandle, AudioHandle } from './PlatformAudioBackend';+import type { PlatformAudioBackend, AudioBufferHandle, AudioHandle, PlayConfig } from './PlatformAudioBackend';import type { AudioMixer, BusDuckRule } from './AudioMixer';import type { BusEffectDef } from './BusEffects';import { defineResource } from '../resource';@@ -239,6 +239,30 @@ export class AudioAPI {if (!this.bufferCache_.has(url)) this.insertEntry_(url, buffer);}+ /**+ * Play a fully-RESOLVED clip URL on an explicit bus (created on demand+ * under master). Unlike playSFX/playBGM this applies no ref resolution —+ * the caller already holds the final URL (the video system's audio track+ * derives it from the resolved video source). Null when the clip can't+ * load; loaded buffers share the residency cache.+ */+ async playTrack(url: string, config: PlayConfig = {}): Promise<AudioHandle | null> {+ if (this.disposed_) return null;+ try {+ if (!this.bufferCache_.has(url)) {+ const buffer = await this.backend_.loadBuffer(url);+ if (!this.bufferCache_.has(url)) this.insertEntry_(url, buffer);+ }+ } catch {+ return null;+ }+ if (this.disposed_) return null;+ const buffer = this.lookupBuffer_(url);+ if (!buffer) return null;+ if (config.bus) this.ensureBus(config.bus);+ return this.backend_.play(buffer, config);+ }+playSFX(url: string, config?: {volume?: number;pitch?: number;diff --git a/sdk/src/video/PlatformVideoBackend.ts b/sdk/src/video/PlatformVideoBackend.tsindex 5e8fa8cdf..82f7f2d5c 100644--- a/sdk/src/video/PlatformVideoBackend.ts+++ b/sdk/src/video/PlatformVideoBackend.ts@@ -48,10 +48,14 @@ export interface PlatformVideoBackend {}/**- * What the app hands a platform's backend factory. `sideModules` is a lazy- * getter — the host may attach to the app after plugins build, so backends- * resolve it per stream, not at construction.+ * What the app hands a platform's backend factory. Both accessors are lazy —+ * the side-module host and the audio service may attach to the app after+ * plugins build, so backends resolve them per stream, not at construction.*/export interface VideoBackendContext {sideModules(): import('../sideModules/host').SideModuleHost | null;+ /** The app's audio service, when the audio plugin is installed. The wasm+ * backend plays the cook-demuxed `.m4a` audio track through it and slaves+ * the video clock to the track's playhead. */+ audio?(): import('../audio/Audio').AudioAPI | null;}diff --git a/sdk/src/video/VideoPlugin.ts b/sdk/src/video/VideoPlugin.tsindex c6b867c57..d10894ece 100644--- a/sdk/src/video/VideoPlugin.ts+++ b/sdk/src/video/VideoPlugin.ts@@ -14,6 +14,7 @@ import type { UIVisualData } from '../component.generated';import { UIVisual } from '../ui/core/ui-visual';import type { VideoStreamHandle } from './PlatformVideoBackend';import { NullVideoBackend } from './NullVideoBackend';+import { Audio } from '../audio/Audio';import { getPlatform } from '../platform/base';import { isEditor, isPlayMode } from '../env';import { log } from '../logger';@@ -65,10 +66,15 @@ export class VideoPlugin implements Plugin {build(app: App): void {// createVideoBackend is optional; fall back to the silent Null backend.- // The context's lazy sideModules getter lets the wasm backend resolve the- // host per stream (it may attach to the app after plugins build).- const backend = getPlatform().createVideoBackend?.({ sideModules: () => app.sideModules })- ?? new NullVideoBackend();+ // The context's getters are lazy so the wasm backend resolves the+ // side-module host / audio service per stream (they may attach to the+ // app after plugins build).+ const backend = getPlatform().createVideoBackend?.({+ sideModules: () => app.sideModules,+ audio: () => {+ try { return app.getResource(Audio); } catch { return null; }+ },+ }) ?? new NullVideoBackend();const video = new VideoAPI(backend);this.video_ = video;app.insertResource(VideoPlayer, video);diff --git a/sdk/src/video/WasmVideoBackend.ts b/sdk/src/video/WasmVideoBackend.tsindex 7a9000b1c..af936777a 100644--- a/sdk/src/video/WasmVideoBackend.ts+++ b/sdk/src/video/WasmVideoBackend.ts@@ -5,9 +5,18 @@// pump as every other backend. Runs wherever the engine's wasm runs — the// guaranteed path on platforms without a reliable native decoder (WeChat, headless)// — and owns the clock, so seek/currentTime/duration/rate all work uniformly.+//+// Audio: the cook demuxes the source's audio track into an `<staged>.m4a`+// sibling played through the audio pipeline (bus-routed, so mixer/ducking+// apply). When a track exists, IT is the clock — the video decodes toward the+// track's playhead — since audio glitches are far more audible than a video+// frame arriving late. Without a track (or when it ends first), the engine+// wall clock drives.import type { ESEngineModule } from '../wasm';import type { PlatformVideoBackend, VideoBackendContext, VideoStreamHandle, VideoStreamOptions } from './PlatformVideoBackend';import type { SideModule } from '../sideModules/host';+import type { AudioAPI } from '../audio/Audio';+import type { AudioHandle } from '../audio/PlatformAudioBackend';import { createTextureFromPixels, updateTextureSubregion } from '../runtimeAssets';import { requireResourceManager } from '../resourceManager';import { getPlatform } from '../platform/base';@@ -36,10 +45,23 @@ export interface VideoWasmModule {readonly HEAPU8: Uint8Array;}+/** What the wasm stream needs from its backend (module + optional audio). */+interface WasmStreamDeps {+ acquire(): Promise<SideModule | null>;+ audio(): AudioAPI | null;+}+// Longest catch-up decoded in one pump. MPEG-1 has no frame-skip (P/B frames// need every predecessor), so a huge dt after a hidden window would stall the// frame decoding it all; clamping instead slows playback to real decode speed.const MAX_PUMP_DT = 0.25;+// An audio clock this far BEHIND the video is a loop wrap or backward seek —+// exact-seek the video instead of waiting for the clock to catch up.+const CLOCK_BACK_JUMP = 0.25;+// Bus the audio track routes through (created on demand under master).+const VIDEO_AUDIO_BUS = 'video';+// Video extensions whose cooked `.m4a` audio-track sibling is worth probing.+const AUDIO_SIBLING_RE = /\.(esv|mpg|mpeg)$/i;class WasmVideoStreamHandle implements VideoStreamHandle {readonly id = nextId_++;@@ -61,20 +83,28 @@ class WasmVideoStreamHandle implements VideoStreamHandle {private playing_: boolean;private loop_: boolean;private rate_: number;+ private volume_: number;+ private muted_: boolean;private lastNow_ = -1;private frameDirty_ = false;private pendingSeek_ = -1;+ private audioApi_: AudioAPI | null = null;+ private audioUrl_: string | null = null;+ private audio_: AudioHandle | null = null;+ private audioGen_ = 0;- constructor(url: string, options: VideoStreamOptions, acquire: () => Promise<SideModule | null>) {+ constructor(url: string, options: VideoStreamOptions, deps: WasmStreamDeps) {this.playing_ = options.autoplay ?? true;this.loop_ = options.loop ?? false;this.rate_ = options.playbackRate ?? 1;- void this.start_(url, acquire);+ this.volume_ = options.volume ?? 1;+ this.muted_ = options.muted ?? false;+ void this.start_(url, deps);}- private async start_(url: string, acquire: () => Promise<SideModule | null>): Promise<void> {+ private async start_(url: string, deps: WasmStreamDeps): Promise<void> {try {- const [bytes, mod] = await Promise.all([this.loadBytes_(url), acquire()]);+ const [bytes, mod] = await Promise.all([this.loadBytes_(url), deps.acquire()]);if (this.disposed_) return;if (!mod) throw new Error('the "videodec" side module is unavailable');const video = mod as unknown as VideoWasmModule;@@ -96,11 +126,16 @@ class WasmVideoStreamHandle implements VideoStreamHandle {this.framePtr_ = video._malloc(this.frameBytes_);if (!this.framePtr_) throw new Error('videodec out of memory');video._es_video_set_loop(decoder, this.loop_ ? 1 : 0);+ let startAt = 0;if (this.pendingSeek_ >= 0) {- video._es_video_seek(decoder, this.pendingSeek_);+ if (video._es_video_seek(decoder, this.pendingSeek_)) this.frameDirty_ = true;+ startAt = this.pendingSeek_;this.pendingSeek_ = -1;- this.frameDirty_ = true;}++ this.audioApi_ = deps.audio();+ if (this.audioApi_ && AUDIO_SIBLING_RE.test(url)) this.audioUrl_ = `${url}.m4a`;+ if (this.playing_) void this.startAudio_(startAt);} catch (err) {if (this.disposed_) return;log.error('video', `wasm decode failed for "${url}"`, err);@@ -119,6 +154,53 @@ class WasmVideoStreamHandle implements VideoStreamHandle {return platform.readFile(url);}+ // === audio track ========================================================++ /** (Re)start the audio-track sibling at `offset` seconds. A missing track+ * resolves to null once and disables further probes (engine clock drives). */+ private async startAudio_(offset: number): Promise<void> {+ const api = this.audioApi_;+ const url = this.audioUrl_;+ if (!api || !url) return;+ const gen = ++this.audioGen_;+ const handle = await api.playTrack(url, {+ bus: VIDEO_AUDIO_BUS,+ volume: this.muted_ ? 0 : this.volume_,+ loop: this.loop_,+ playbackRate: this.rate_,+ startOffset: offset,+ });+ if (!handle) {+ if (gen === this.audioGen_) this.audioUrl_ = null; // no track — engine clock+ return;+ }+ if (this.disposed_ || gen !== this.audioGen_) {+ handle.stop();+ return;+ }+ // The audio track is the clock now; the video must not free-wrap on its+ // own (the wrap is driven by the track's playhead jumping back).+ this.mod_?._es_video_set_loop(this.decoder_, 0);+ // A non-looping track that ends BEFORE the last video frame hands the+ // clock back to the engine so the tail still plays out.+ handle.onEnd = () => {+ if (this.audio_ === handle && !this.loop_) {+ this.audio_ = null;+ this.lastNow_ = -1;+ }+ };+ if (!this.playing_) handle.pause();+ this.audio_ = handle;+ }++ private stopAudio_(): void {+ this.audioGen_++;+ if (this.audio_) {+ this.audio_.stop();+ this.audio_ = null;+ }+ }+// === texture pump =======================================================pump(module: ESEngineModule): void {@@ -126,15 +208,33 @@ class WasmVideoStreamHandle implements VideoStreamHandle {if (this.disposed_ || !video) return;if (this.playing_ && !this.ended_) {- const now = getPlatform().now();- const dt = this.lastNow_ < 0 ? 0 : Math.min((now - this.lastNow_) / 1000, MAX_PUMP_DT);- this.lastNow_ = now;- if (dt > 0 && video._es_video_advance(this.decoder_, dt * this.rate_)) {- this.frameDirty_ = true;+ if (this.audio_) {+ // Audio-track clock: decode toward the track's playhead.+ const target = this.audio_.currentTime;+ const cur = video._es_video_time(this.decoder_);+ const dt = target - cur;+ if (dt > 0.001) {+ if (video._es_video_advance(this.decoder_, Math.min(dt, MAX_PUMP_DT))) {+ this.frameDirty_ = true;+ }+ } else if (dt < -CLOCK_BACK_JUMP) {+ // The track wrapped (loop) or jumped backwards — follow it.+ if (video._es_video_seek(this.decoder_, Math.max(target, 0))) {+ this.frameDirty_ = true;+ }+ }+ } else {+ const now = getPlatform().now();+ const dt = this.lastNow_ < 0 ? 0 : Math.min((now - this.lastNow_) / 1000, MAX_PUMP_DT);+ this.lastNow_ = now;+ if (dt > 0 && video._es_video_advance(this.decoder_, dt * this.rate_)) {+ this.frameDirty_ = true;+ }}if (!this.loop_ && video._es_video_has_ended(this.decoder_)) {this.ended_ = true;this.playing_ = false;+ this.stopAudio_();this.onEnded?.();}}@@ -176,10 +276,16 @@ class WasmVideoStreamHandle implements VideoStreamHandle {if (this.ended_) {this.ended_ = false;this.seek(0);+ return;}+ if (this.audio_) this.audio_.resume();+ else if (this.mod_) void this.startAudio_(this.currentTime);}- pause(): void { this.playing_ = false; }+ pause(): void {+ this.playing_ = false;+ this.audio_?.pause();+ }seek(timeSeconds: number): void {const t = timeSeconds > 0 ? timeSeconds : 0;@@ -189,23 +295,37 @@ class WasmVideoStreamHandle implements VideoStreamHandle {}this.ended_ = false;if (this.mod_._es_video_seek(this.decoder_, t)) this.frameDirty_ = true;+ // AudioHandles have no in-place seek — restart the track at the offset.+ this.stopAudio_();+ if (this.playing_) void this.startAudio_(t);}- // The wasm path decodes video only — the cook demuxes the audio track for- // the audio pipeline, so per-stream volume/mute live there, not here.- setVolume(_volume: number): void {}- setMuted(_muted: boolean): void {}+ setVolume(volume: number): void {+ this.volume_ = volume;+ this.audio_?.setVolume(this.muted_ ? 0 : volume);+ }++ setMuted(muted: boolean): void {+ this.muted_ = muted;+ this.audio_?.setVolume(muted ? 0 : this.volume_);+ }setLoop(loop: boolean): void {this.loop_ = loop;- this.mod_?._es_video_set_loop(this.decoder_, loop ? 1 : 0);+ this.audio_?.setLoop(loop);+ // Without an audio clock the decoder wraps itself.+ if (!this.audio_) this.mod_?._es_video_set_loop(this.decoder_, loop ? 1 : 0);}- setPlaybackRate(rate: number): void { this.rate_ = rate > 0 ? rate : 0; }+ setPlaybackRate(rate: number): void {+ this.rate_ = rate > 0 ? rate : 0;+ this.audio_?.setPlaybackRate(this.rate_);+ }stop(): void {if (this.disposed_) return;this.disposed_ = true;+ this.stopAudio_();if (this.mod_) {this.mod_._es_video_close(this.decoder_);if (this.framePtr_) this.mod_._free(this.framePtr_);@@ -226,10 +346,13 @@ export class WasmVideoBackend implements PlatformVideoBackend {constructor(private readonly ctx_: VideoBackendContext) {}createStream(url: string, options: VideoStreamOptions): VideoStreamHandle {- return new WasmVideoStreamHandle(url, options, () => {- const host = this.ctx_.sideModules();- if (!host) return Promise.resolve(null);- return host.acquire('videodec');+ return new WasmVideoStreamHandle(url, options, {+ acquire: () => {+ const host = this.ctx_.sideModules();+ if (!host) return Promise.resolve(null);+ return host.acquire('videodec');+ },+ audio: () => this.ctx_.audio?.() ?? null,});}diff --git a/sdk/tests/audio-api.test.ts b/sdk/tests/audio-api.test.tsindex 99b39b957..c5f23a691 100644--- a/sdk/tests/audio-api.test.ts+++ b/sdk/tests/audio-api.test.ts@@ -281,4 +281,36 @@ describe('AudioAPI', () => {expect(backend2.loadBuffer).toHaveBeenCalledTimes(1);});});++ describe('playTrack', () => {+ it('loads the URL verbatim (no ref resolution) and plays it on the bus', async () => {+ const backend = createMockBackend();+ const audio = new AudioAPI(backend, createMockMixer());+ audio.setRefResolver(() => { throw new Error('must not resolve'); });+ const handle = await audio.playTrack('assets/clip.esv.m4a', { bus: 'video', startOffset: 1.5, volume: 0.5 });+ expect(handle).not.toBeNull();+ expect(backend.loadBuffer).toHaveBeenCalledWith('assets/clip.esv.m4a');+ expect(backend.play).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({+ bus: 'video', startOffset: 1.5, volume: 0.5,+ }));+ });++ it('creates the bus on demand and caches the buffer across plays', async () => {+ const backend = createMockBackend();+ const mixer = createMockMixer();+ (mixer.getBus as ReturnType<typeof vi.fn>).mockReturnValue(undefined);+ const audio = new AudioAPI(backend, mixer);+ await audio.playTrack('t.m4a', { bus: 'video' });+ expect(mixer.createBus).toHaveBeenCalledWith(expect.objectContaining({ name: 'video' }));+ await audio.playTrack('t.m4a', { bus: 'video' });+ expect(backend.loadBuffer).toHaveBeenCalledTimes(1);+ });++ it('resolves null when the clip cannot load', async () => {+ const backend = createMockBackend();+ (backend.loadBuffer as ReturnType<typeof vi.fn>).mockRejectedValue(new Error('404'));+ const audio = new AudioAPI(backend, createMockMixer());+ expect(await audio.playTrack('missing.m4a')).toBeNull();+ });+ });});diff --git a/sdk/tests/wasm-video-backend.test.ts b/sdk/tests/wasm-video-backend.test.tsindex 385b65dea..b0ba63b9e 100644--- a/sdk/tests/wasm-video-backend.test.ts+++ b/sdk/tests/wasm-video-backend.test.ts@@ -5,6 +5,8 @@import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';import { WasmVideoBackend, type VideoWasmModule } from '../src/video/WasmVideoBackend';import type { SideModuleHost } from '../src/sideModules/host';+import type { AudioAPI } from '../src/audio/Audio';+import type { AudioHandle } from '../src/audio/PlatformAudioBackend';import { setPlatform } from '../src/platform/base';import { initResourceManager, shutdownResourceManager } from '../src/resourceManager';import type { PlatformAdapter } from '../src/platform/types';@@ -234,3 +236,133 @@ describe('WasmVideoBackend', () => {expect(() => handle.pump(engineModule)).not.toThrow();});});++// === audio-track clock ======================================================++function mockAudio() {+ const handles: FakeAudioHandle[] = [];+ class FakeAudioHandle {+ time = 0;+ playing = true;+ onEnd?: () => void;+ stop = vi.fn(() => { this.playing = false; });+ pause = vi.fn(() => { this.playing = false; });+ resume = vi.fn(() => { this.playing = true; });+ setVolume = vi.fn();+ setPan = vi.fn();+ setLoop = vi.fn();+ setPlaybackRate = vi.fn();+ get isPlaying() { return this.playing; }+ get currentTime() { return this.time; }+ get duration() { return 2.5; }+ readonly id = handles.length + 1;+ }+ const playTrack = vi.fn(async (_url: string, cfg: { startOffset?: number }) => {+ const h = new FakeAudioHandle();+ h.time = cfg.startOffset ?? 0;+ handles.push(h);+ return h as unknown as AudioHandle;+ });+ const api = { playTrack } as unknown as AudioAPI;+ return { api, playTrack, handles };+}++describe('WasmVideoBackend audio-track clock', () => {+ it('starts the .m4a sibling on the video bus and slaves the video clock to it', async () => {+ const { mod, raw } = mockVideoModule();+ const { api, playTrack, handles } = mockAudio();+ const backend = new WasmVideoBackend({ sideModules: () => hostFor(mod), audio: () => api });+ const handle = backend.createStream('videos/clip.esv', { volume: 0.5, loop: false });+ await flush();++ expect(playTrack).toHaveBeenCalledWith('videos/clip.esv.m4a', expect.objectContaining({+ bus: 'video', volume: 0.5, loop: false, startOffset: 0,+ }));+ // The decoder must not free-wrap while the track drives the clock.+ expect(raw._es_video_set_loop).toHaveBeenLastCalledWith(1, 0);++ handles[0].time = 0.1;+ handle.pump(engineModule);+ expect((raw._es_video_advance.mock.calls.at(-1) as number[])[1]).toBeCloseTo(0.1);++ // Audio clock jumping far backwards (loop wrap) exact-seeks the video.+ raw._es_video_advance(1, 2.0); // decoder time → ~2.1+ handles[0].time = 0.02;+ handle.pump(engineModule);+ expect(raw._es_video_seek).toHaveBeenCalledWith(1, 0.02);+ handle.stop();+ expect(handles[0].stop).toHaveBeenCalled();+ });++ it('muted stream starts its track at volume 0; volume/mute changes forward', async () => {+ const { mod } = mockVideoModule();+ const { api, playTrack, handles } = mockAudio();+ const backend = new WasmVideoBackend({ sideModules: () => hostFor(mod), audio: () => api });+ const handle = backend.createStream('clip.esv', { muted: true, volume: 0.8 });+ await flush();+ expect(playTrack).toHaveBeenCalledWith('clip.esv.m4a', expect.objectContaining({ volume: 0 }));+ handle.setMuted(false);+ expect(handles[0].setVolume).toHaveBeenLastCalledWith(0.8);+ handle.setVolume(0.3);+ expect(handles[0].setVolume).toHaveBeenLastCalledWith(0.3);+ handle.stop();+ });++ it('seek restarts the track at the offset; pause/play suspend and resume it', async () => {+ const { mod } = mockVideoModule();+ const { api, playTrack, handles } = mockAudio();+ const backend = new WasmVideoBackend({ sideModules: () => hostFor(mod), audio: () => api });+ const handle = backend.createStream('clip.esv', {});+ await flush();+ handle.seek(1.5);+ await flush();+ expect(handles[0].stop).toHaveBeenCalled();+ expect(playTrack).toHaveBeenLastCalledWith('clip.esv.m4a', expect.objectContaining({ startOffset: 1.5 }));+ handle.pause();+ expect(handles[1].pause).toHaveBeenCalled();+ handle.play();+ expect(handles[1].resume).toHaveBeenCalled();+ handle.stop();+ });++ it('a missing track falls back to the engine clock', async () => {+ const { mod, raw } = mockVideoModule();+ const playTrack = vi.fn(async () => null);+ const api = { playTrack } as unknown as AudioAPI;+ const backend = new WasmVideoBackend({ sideModules: () => hostFor(mod), audio: () => api });+ const handle = backend.createStream('clip.esv', {});+ await flush();+ expect(playTrack).toHaveBeenCalledTimes(1);+ nowMs = 16;+ handle.pump(engineModule);+ nowMs = 32;+ handle.pump(engineModule);+ expect(raw._es_video_advance).toHaveBeenCalled();+ handle.stop();+ });++ it('a non-looping track ending early hands the clock back to the engine', async () => {+ const { mod, raw } = mockVideoModule();+ const { api, handles } = mockAudio();+ const backend = new WasmVideoBackend({ sideModules: () => hostFor(mod), audio: () => api });+ const handle = backend.createStream('clip.esv', { loop: false });+ await flush();+ handles[0].onEnd?.();+ nowMs = 16;+ handle.pump(engineModule); // clock base+ nowMs = 32;+ handle.pump(engineModule);+ expect(raw._es_video_advance).toHaveBeenCalled(); // engine clock drove+ handle.stop();+ });++ it('never probes a sibling for non-esv sources', async () => {+ const { mod } = mockVideoModule();+ const { api, playTrack } = mockAudio();+ const backend = new WasmVideoBackend({ sideModules: () => hostFor(mod), audio: () => api });+ const handle = backend.createStream('clip.bin', {});+ await flush();+ expect(playTrack).not.toHaveBeenCalled();+ handle.stop();+ });+});