Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

feat(video): cook transcodes to .esv + m4a audio track slaved as the clock (V2)

esengine7dbbff069b81
diff --git a/NOTICE b/NOTICE
index 905527f50..db8f87be3 100644
--- a/NOTICE
+++ b/NOTICE
@@ -35,6 +35,13 @@ license text is included in its directory under third_party/.
pl_mpeg third_party/pl_mpeg MIT
Spine Runtimes third_party/spine-runtimes-* Spine Runtimes License
+The desktop editor additionally bundles an FFmpeg binary (via the ffmpeg-static
+npm package; FFmpeg is a trademark of Fabrice Bellard). It is licensed under the
+GPL (see the LICENSE file inside the ffmpeg-static package) and is invoked by
+the editor's asset cook strictly as a separate external process — it is not
+linked into, nor does its license extend to, Estella itself. It does not ship
+in exported games. Set ESTELLA_FFMPEG to use a system FFmpeg instead.
+
IMPORTANT — Spine Runtimes are NOT open source. Relicensing Estella to Apache-2.0
does NOT change this: the Spine Runtimes remain governed by the Spine Runtimes
License Agreement from Esoteric Software (see third_party/spine-runtimes-*/LICENSE).
diff --git a/desktop/electron-builder.yml b/desktop/electron-builder.yml
index 4d5327a18..18dc88a6e 100644
--- a/desktop/electron-builder.yml
+++ b/desktop/electron-builder.yml
@@ -25,6 +25,9 @@ files:
# directory copy, which cannot source from inside an asar.
asarUnpack:
- "**/node_modules/{esbuild,@esbuild}/**"
+ # ffmpeg (video cook) spawns as a subprocess — executables cannot run from
+ # inside the asar (videoCook resolves the app.asar.unpacked twin).
+ - "**/node_modules/ffmpeg-static/**"
- dist-electron/hosts/**
# The MCP front: an MCP client runs this with plain node (node cannot read
# inside an asar), which spawns/attaches the editor as the MCP host.
diff --git a/desktop/electron/cookAssets.ts b/desktop/electron/cookAssets.ts
index b369a9924..099abafca 100644
--- a/desktop/electron/cookAssets.ts
+++ b/desktop/electron/cookAssets.ts
@@ -197,13 +197,14 @@ export async function cookAssets(
opts: {
entryScenes: string[]; outDir: string;
contentAddressed?: boolean; compressTextures?: boolean; atlasTextures?: boolean;
- compressAudio?: boolean;
+ compressAudio?: boolean; transcodeVideo?: boolean;
},
): Promise<CookResult> {
const contentAddressed = opts.contentAddressed ?? false;
const compressTextures = opts.compressTextures ?? false;
const atlasTextures = opts.atlasTextures ?? false;
const compressAudio = opts.compressAudio ?? false;
+ const transcodeVideo = opts.transcodeVideo ?? false;
const { index } = await scanAssetDatabase(root, { write: false, adopt: false });
const byUuid = new Map(index.entries.map((e) => [e.uuid, e]));
const byPath = new Map(index.entries.map((e) => [e.path, e]));
@@ -267,6 +268,12 @@ export async function cookAssets(
if (compressAudio) {
audioEnc = await import('./audioCook');
}
+ // Video → MPEG-1 `.esv` + `.m4a` audio sibling, for targets whose only video
+ // path is the wasm decoder (WeChat). ffmpeg loads lazily the same way.
+ let videoEnc: typeof import('./videoCook') | null = null;
+ if (transcodeVideo) {
+ videoEnc = await import('./videoCook');
+ }
const manifestEntries: CookManifestEntry[] = [];
const staged = new Set<string>(); // staged output paths, for content-addressed dedup
@@ -391,6 +398,21 @@ export async function cookAssets(
}
}
}
+ // Video → `.esv` (MPEG-1 for the wasm decoder) + audio-track sibling.
+ // `.esv` sources are already in the cooked format and pass through.
+ let videoAudio: Uint8Array | null = null;
+ if (videoEnc && entry.type === 'video' && ext.toLowerCase() !== '.esv') {
+ const settings = videoEnc.videoImportSettings(entry.importer);
+ const res = await videoEnc.transcodeVideoForWasm(path.join(root, entry.path), settings);
+ warnings.push(...res.warnings.map((w) => `${entry.path}: ${w}`));
+ if (res.esv) {
+ data = res.esv;
+ ext = '.esv';
+ videoAudio = res.audio;
+ } else {
+ warnings.push(`${entry.path}: shipped untranscoded — this video will NOT play on the wasm decode path (WeChat)`);
+ }
+ }
// Materials: relative refs → logical project paths (see rewriteMaterialRefs).
if (ext.toLowerCase() === '.esmaterial') {
try {
@@ -450,6 +472,27 @@ export async function cookAssets(
group,
...(compressedFormats ? { compressedFormats } : {}),
});
+ // The video's audio track ships as an `<staged>.m4a` SIBLING of the
+ // `.esv` — appended (not extension-swapped) so it can never collide with
+ // a real project `.m4a`, and the runtime derives it from the resolved
+ // video URL with no metadata channel. Its own manifest entry keeps
+ // addressable/CDN flows aware of the file.
+ if (videoAudio) {
+ const audioOutRel = `${outRel}.m4a`;
+ if (!staged.has(audioOutRel)) {
+ await writeFile(path.join(absOut, audioOutRel), videoAudio);
+ staged.add(audioOutRel);
+ }
+ manifestEntries.push({
+ uuid: `${entry.uuid}-audio`,
+ path: audioOutRel,
+ sourcePath: `${entry.path}.m4a`,
+ type: 'audio',
+ contentHash: contentHashHex(videoAudio),
+ size: videoAudio.byteLength,
+ group,
+ });
+ }
} catch (err) {
warnings.push(`copy failed ${entry.path}: ${err instanceof Error ? err.message : String(err)}`);
}
diff --git a/desktop/electron/exportWeChat.ts b/desktop/electron/exportWeChat.ts
index 3f8e481ff..454287acc 100644
--- a/desktop/electron/exportWeChat.ts
+++ b/desktop/electron/exportWeChat.ts
@@ -303,7 +303,7 @@ export async function exportWeChat(opts: {
// manifest. KTX2 textures are fine here: the scan below sees the staged
// .ktx2 files and ships the Basis transcoder side module with them.
progress({ phase: 'Cooking assets' });
- const cook = await cookAssets(opts.root, { entryScenes: scenes.map((s) => s.path), outDir: absOut, contentAddressed: opts.contentAddressed, compressTextures: opts.compressTextures, compressAudio: opts.compressAudio, atlasTextures: opts.atlasTextures });
+ const cook = await cookAssets(opts.root, { entryScenes: scenes.map((s) => s.path), outDir: absOut, contentAddressed: opts.contentAddressed, compressTextures: opts.compressTextures, compressAudio: opts.compressAudio, atlasTextures: opts.atlasTextures, transcodeVideo: true });
warnings.push(...cook.warnings);
// 1a. Restage for WeChat's code-package suffix whitelist (it has no `ktx2`
diff --git a/desktop/electron/videoCook.ts b/desktop/electron/videoCook.ts
new file mode 100644
index 000000000..9da5fd615
--- /dev/null
+++ b/desktop/electron/videoCook.ts
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * @file videoCook.ts — cook-time video transcode for the wasm decode path.
+ * WeChat's only video backend is the engine-owned MPEG-1 decoder
+ * (videodec side module), so the cook turns each authored video into a
+ * codec-tagged `.esv` (MPEG-1 Program Stream) plus an `.m4a` audio-track
+ * sibling played through the audio pipeline. Transcoding shells out to
+ * the bundled ffmpeg binary (ffmpeg-static) — a separate process, loaded
+ * lazily and only when a cook actually ships video.
+ */
+import { spawn } from 'node:child_process';
+import { mkdtemp, readFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+/** Resolve the ffmpeg binary: env override first, then the bundled static
+ * build. Inside a packaged app the binary must run from the unpacked twin of
+ * the asar (executables cannot spawn from an archive). Null if unavailable. */
+export async function resolveFfmpeg(): Promise<string | null> {
+ if (process.env.ESTELLA_FFMPEG) return process.env.ESTELLA_FFMPEG;
+ try {
+ const mod = await import('ffmpeg-static');
+ const p = (mod.default ?? mod) as unknown as string | null;
+ return p ? p.replace(`app.asar${path.sep}`, `app.asar.unpacked${path.sep}`) : null;
+ } catch {
+ return null;
+ }
+}
+
+function run(bin: string, args: string[]): Promise<{ code: number; stderr: string }> {
+ return new Promise((resolve, reject) => {
+ const child = spawn(bin, args, { stdio: ['ignore', 'ignore', 'pipe'] });
+ let stderr = '';
+ child.stderr.on('data', (d: Buffer) => { stderr += d.toString(); });
+ child.on('error', reject);
+ child.on('close', (code) => resolve({ code: code ?? -1, stderr }));
+ });
+}
+
+export interface VideoImportSettings {
+ /** mpeg1video quantizer (2 = near-lossless, larger = smaller file). */
+ quality: number;
+ audioBitrateKbps: number;
+}
+
+/** Per-asset video import settings from the `.meta` importer block (tolerant). */
+export function videoImportSettings(importer: Record<string, unknown> | undefined): VideoImportSettings {
+ const q = importer?.quality;
+ const kbps = importer?.audioBitrateKbps;
+ return {
+ quality: typeof q === 'number' && q >= 2 && q <= 31 ? Math.round(q) : 4,
+ audioBitrateKbps: kbps === 96 || kbps === 128 || kbps === 192 ? (kbps as number) : 128,
+ };
+}
+
+export interface VideoCookResult {
+ /** MPEG-1 PS bytes, or null when the transcode failed. */
+ esv: Uint8Array | null;
+ /** AAC (m4a) audio track, or null when the source has none. */
+ audio: Uint8Array | null;
+ warnings: string[];
+}
+
+// MPEG-1 dimensions must be even (macroblock chroma), and its frame rate is a
+// fixed table — sources with a rate outside it are resampled on retry.
+const EVEN_SCALE = 'scale=trunc(iw/2)*2:trunc(ih/2)*2';
+
+/**
+ * Transcode `srcAbs` into the wasm decode path's artifacts. Video failures are
+ * reported (esv null); a missing audio track is normal and silent.
+ */
+export async function transcodeVideoForWasm(
+ srcAbs: string,
+ settings: VideoImportSettings,
+): Promise<VideoCookResult> {
+ const warnings: string[] = [];
+ const ffmpeg = await resolveFfmpeg();
+ if (!ffmpeg) {
+ return { esv: null, audio: null, warnings: ['ffmpeg unavailable — cannot transcode video for the wasm decoder'] };
+ }
+ const tmp = await mkdtemp(path.join(tmpdir(), 'estella-video-'));
+ try {
+ const esvPath = path.join(tmp, 'out.esv');
+ const videoArgs = (extraFilter: string) => [
+ '-y', '-i', srcAbs, '-an',
+ '-c:v', 'mpeg1video', '-q:v', String(settings.quality),
+ '-vf', extraFilter ? `${EVEN_SCALE},${extraFilter}` : EVEN_SCALE,
+ '-f', 'mpeg', esvPath,
+ ];
+ let res = await run(ffmpeg, videoArgs(''));
+ if (res.code !== 0 && /fps|frame ?rate/i.test(res.stderr)) {
+ // Source frame rate is outside MPEG-1's table (or variable) — resample.
+ res = await run(ffmpeg, videoArgs('fps=30'));
+ }
+ let esv: Uint8Array | null = null;
+ if (res.code === 0) {
+ esv = new Uint8Array(await readFile(esvPath));
+ } else {
+ warnings.push(`video transcode failed: ${lastLine(res.stderr)}`);
+ }
+
+ const m4aPath = path.join(tmp, 'out.m4a');
+ const audioRes = await run(ffmpeg, [
+ '-y', '-i', srcAbs, '-vn',
+ '-c:a', 'aac', '-b:a', `${settings.audioBitrateKbps}k`,
+ '-f', 'ipod', m4aPath,
+ ]);
+ let audio: Uint8Array | null = null;
+ if (audioRes.code === 0) {
+ audio = new Uint8Array(await readFile(m4aPath));
+ } else if (!/does not contain any stream|Stream map .* matches no streams|no audio/i.test(audioRes.stderr)) {
+ warnings.push(`audio track extract failed: ${lastLine(audioRes.stderr)}`);
+ }
+ return { esv, audio, warnings };
+ } finally {
+ await rm(tmp, { recursive: true, force: true });
+ }
+}
+
+function lastLine(stderr: string): string {
+ const lines = stderr.trim().split('\n');
+ return lines[lines.length - 1] ?? 'unknown ffmpeg error';
+}
diff --git a/desktop/package.json b/desktop/package.json
index d934ddcda..2a3f59fa4 100644
--- a/desktop/package.json
+++ b/desktop/package.json
@@ -68,6 +68,7 @@
"dockview": "^6.6.1",
"esbuild": "^0.27.0",
"esengine": "workspace:*",
+ "ffmpeg-static": "^5.3.0",
"lucide-react": "^1.21.0",
"pngjs": "^7.0.0",
"react": "^19.2.0",
diff --git a/desktop/tests/video-cook.test.ts b/desktop/tests/video-cook.test.ts
new file mode 100644
index 000000000..488953dce
--- /dev/null
+++ b/desktop/tests/video-cook.test.ts
@@ -0,0 +1,159 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
+/**
+ * Cook-time video: mp4 → `.esv` (MPEG-1 PS for the wasm decoder) + `.m4a`
+ * audio-track sibling, through the real bundled ffmpeg. Sources are generated
+ * at test time with the same binary (lavfi), so no binary fixtures ship.
+ * When the videodec wasm is built locally, the cooked `.esv` is decoded
+ * end-to-end and its frames asserted.
+ */
+import { describe, it, expect, beforeAll } from 'vitest';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { existsSync, readFileSync } from 'node:fs';
+import { mkdtemp } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { resolveFfmpeg, transcodeVideoForWasm, videoImportSettings } from '../electron/videoCook';
+import { cookAssets } from '../electron/cookAssets';
+
+const execFileAsync = promisify(execFile);
+
+const WASM_DIR = path.resolve(__dirname, '../../build/wasm/web');
+const HAS_VIDEODEC = existsSync(path.join(WASM_DIR, 'videodec.wasm'));
+
+const ffmpeg = await resolveFfmpeg();
+let tmp = '';
+
+beforeAll(async () => {
+ tmp = await mkdtemp(path.join(tmpdir(), 'estella-videocook-test-'));
+});
+
+/** Generate a source clip: solid lime video, optional 440Hz tone, ODD width. */
+async function makeSource(name: string, withAudio: boolean): Promise<string> {
+ const out = path.join(tmp, name);
+ const args = ['-y', '-f', 'lavfi', '-i', 'color=c=lime:s=63x32:r=25:d=1'];
+ if (withAudio) args.push('-f', 'lavfi', '-i', 'sine=frequency=440:duration=1', '-c:a', 'aac', '-shortest');
+ args.push('-c:v', 'libx264', '-pix_fmt', 'yuv420p', out);
+ await execFileAsync(ffmpeg!, args);
+ return out;
+}
+
+describe('videoImportSettings', () => {
+ it('defaults and clamps', () => {
+ expect(videoImportSettings(undefined)).toEqual({ quality: 4, audioBitrateKbps: 128 });
+ expect(videoImportSettings({ quality: 8, audioBitrateKbps: 192 })).toEqual({ quality: 8, audioBitrateKbps: 192 });
+ expect(videoImportSettings({ quality: 99, audioBitrateKbps: 7 })).toEqual({ quality: 4, audioBitrateKbps: 128 });
+ });
+});
+
+describe.skipIf(!ffmpeg)('transcodeVideoForWasm', () => {
+ it('produces an MPEG-PS .esv and an .m4a audio track', async () => {
+ const src = await makeSource('with-audio.mp4', true);
+ const res = await transcodeVideoForWasm(src, videoImportSettings(undefined));
+ expect(res.warnings).toEqual([]);
+ expect(res.esv).not.toBeNull();
+ // MPEG-PS pack header: 00 00 01 BA.
+ expect(Array.from(res.esv!.subarray(0, 4))).toEqual([0, 0, 1, 0xba]);
+ expect(res.audio).not.toBeNull();
+ // MP4/M4A container: 'ftyp' at offset 4.
+ expect(Buffer.from(res.audio!.subarray(4, 8)).toString('ascii')).toBe('ftyp');
+ });
+
+ it('a silent source yields esv with no audio sibling and no warnings', async () => {
+ const src = await makeSource('silent.mp4', false);
+ const res = await transcodeVideoForWasm(src, videoImportSettings(undefined));
+ expect(res.warnings).toEqual([]);
+ expect(res.esv).not.toBeNull();
+ expect(res.audio).toBeNull();
+ });
+
+ it('reports failure for an undecodable source', async () => {
+ const bogus = path.join(tmp, 'bogus.mp4');
+ const fs = await import('node:fs/promises');
+ await fs.writeFile(bogus, Buffer.alloc(256, 0x42));
+ const res = await transcodeVideoForWasm(bogus, videoImportSettings(undefined));
+ expect(res.esv).toBeNull();
+ expect(res.warnings.length).toBeGreaterThan(0);
+ });
+
+ it.skipIf(!HAS_VIDEODEC)('cooked .esv decodes in the real videodec module with evened dimensions', async () => {
+ const src = await makeSource('roundtrip.mp4', false);
+ const res = await transcodeVideoForWasm(src, videoImportSettings(undefined));
+ expect(res.esv).not.toBeNull();
+
+ const factory = (await import(path.join(WASM_DIR, 'videodec.js'))).default;
+ const mod = await factory({ wasmBinary: readFileSync(path.join(WASM_DIR, 'videodec.wasm')) });
+ const ptr = mod._malloc(res.esv!.length);
+ mod.HEAPU8.set(res.esv!, ptr);
+ const h = mod._es_video_open(ptr, res.esv!.length);
+ expect(h).toBeGreaterThan(0);
+ // The odd 63px source width is evened for MPEG-1 macroblocks.
+ expect(mod._es_video_width(h)).toBe(62);
+ expect(mod._es_video_height(h)).toBe(32);
+
+ expect(mod._es_video_advance(h, 0.05)).toBe(1);
+ const out = mod._malloc(62 * 32 * 4);
+ expect(mod._es_video_frame_rgba(h, out, 62 * 32 * 4)).toBe(1);
+ const px = out + ((16 * 62) + 31) * 4;
+ const [r, g, b, a] = Array.from(mod.HEAPU8.subarray(px, px + 4)) as number[];
+ expect(g).toBeGreaterThan(200);
+ expect(r).toBeLessThan(60);
+ expect(b).toBeLessThan(60);
+ expect(a).toBe(255);
+ mod._free(out);
+ mod._es_video_close(h);
+ });
+});
+
+describe.skipIf(!ffmpeg)('cookAssets video wiring', () => {
+ const VIDEO_UUID = 'dddddddd-dddd-4ddd-8ddd-dddddddddddd';
+ const SCENE_UUID = 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee';
+
+ async function makeProject(): Promise<string> {
+ const fs = await import('node:fs');
+ const root = await mkdtemp(path.join(tmpdir(), 'estella-videowire-'));
+ const write = (rel: string, body: string | Uint8Array, meta: object) => {
+ const abs = path.join(root, rel);
+ fs.mkdirSync(path.dirname(abs), { recursive: true });
+ fs.writeFileSync(abs, body);
+ fs.writeFileSync(`${abs}.meta`, JSON.stringify({ version: '2.0', importer: {}, ...meta }));
+ };
+ const clip = readFileSync(await makeSource('wire-src.mp4', true));
+ write('assets/videos/clip.mp4', clip, { uuid: VIDEO_UUID, type: 'video' });
+ write('assets/scenes/main.esscene', JSON.stringify({
+ version: '1.0', name: 'v',
+ entities: [{ id: 1, name: 'V', parent: null, children: [], components: [
+ { type: 'Sprite', data: { texture: 0 } },
+ { type: 'Video', data: { source: `@uuid:${VIDEO_UUID}` } },
+ ] }],
+ }), { uuid: SCENE_UUID, type: 'scene' });
+ return root;
+ }
+
+ it('transcodeVideo stages .esv plus the .m4a sibling, both in the manifest', async () => {
+ const root = await makeProject();
+ const res = await cookAssets(root, {
+ entryScenes: ['assets/scenes/main.esscene'], outDir: 'build', transcodeVideo: true,
+ });
+ expect(res.warnings).toEqual([]);
+ const manifest = JSON.parse(readFileSync(path.join(root, 'build/assets.manifest.json'), 'utf8'));
+ const video = manifest.entries.find((e: { uuid: string }) => e.uuid === VIDEO_UUID);
+ expect(video.path).toBe('assets/videos/clip.esv');
+ expect(video.sourcePath).toBe('assets/videos/clip.mp4');
+ const audio = manifest.entries.find((e: { uuid: string }) => e.uuid === `${VIDEO_UUID}-audio`);
+ expect(audio.path).toBe('assets/videos/clip.esv.m4a');
+ expect(audio.type).toBe('audio');
+ expect(existsSync(path.join(root, 'build/assets/videos/clip.esv'))).toBe(true);
+ expect(existsSync(path.join(root, 'build/assets/videos/clip.esv.m4a'))).toBe(true);
+ });
+
+ it('without transcodeVideo the mp4 passes through untouched', async () => {
+ const root = await makeProject();
+ await cookAssets(root, { entryScenes: ['assets/scenes/main.esscene'], outDir: 'build' });
+ const manifest = JSON.parse(readFileSync(path.join(root, 'build/assets.manifest.json'), 'utf8'));
+ const video = manifest.entries.find((e: { uuid: string }) => e.uuid === VIDEO_UUID);
+ expect(video.path).toBe('assets/videos/clip.mp4');
+ expect(manifest.entries.find((e: { uuid: string }) => e.uuid === `${VIDEO_UUID}-audio`)).toBeUndefined();
+ });
+});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 538a5d1f9..871bb8119 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -35,6 +35,9 @@ importers:
esengine:
specifier: workspace:*
version: link:../sdk
+ ffmpeg-static:
+ specifier: ^5.3.0
+ version: 5.3.0
lucide-react:
specifier: ^1.21.0
version: 1.21.0([email protected])
@@ -206,6 +209,10 @@ packages:
resolution: {integrity: sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==}
engines: {node: '>=18'}
+ '@derhuerst/[email protected]':
+ resolution: {integrity: sha512-F9rL9k9Xjf5blCz8HsJRO4diy111cayL2vkY2XE4r4t3n0yPXVYy3KD3nJ1qbrSn9743UWSXH4IwuCa/HWlGFw==}
+ engines: {node: '>=6.0.0'}
+
'@electron-internal/[email protected]':
resolution: {integrity: sha512-OjKpjB7gohtEjZiq6nDx1egqjZJhGPN1iFOIED+NFhB/MMkXw/XRcHjh1DGXKT5z2W9eW7Jy2UKU3gpjvusFTQ==}
engines: {node: '>=22.12.0'}
@@ -951,6 +958,9 @@ packages:
resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==}
+
resolution: {integrity: sha512-fRa09kZTgu8o71KFcDjUFuc7F+dEbZYZmkI0mg5YBTRs0yMKjYHsq/c0urDKeDb+D5qVgXOdFcuu+DZPKOITwA==}
@@ -1052,6 +1062,10 @@ packages:
engines: {node: '>=0.4.0'}
hasBin: true
+ resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
+ engines: {node: '>= 6.0.0'}
+
resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==}
engines: {node: '>= 14'}
@@ -1204,6 +1218,9 @@ packages:
resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==}
engines: {node: '>= 0.4'}
+ resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==}
+
resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==}
engines: {node: '>=18'}
@@ -1287,6 +1304,10 @@ packages:
resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
+ resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==}
+ engines: {'0': node >= 6.0}
+
resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==}
@@ -1579,6 +1600,10 @@ packages:
resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==}
+ resolution: {integrity: sha512-H+K6sW6TiIX6VGend0KQwthe+kaceeH/luE8dIZyOP35ik7ahYojDuqlTV1bOrtEwl01sy2HFNGQfi5IDJvotg==}
+ engines: {node: '>=16'}
+
resolution: {integrity: sha512-5giy2PkLYY1cP39p17Ech+2xlpTRL9HLspOfEgm0L6CwBXBTgsK5ou0JtzYuepxkaQ/tvhCFIJ5uXo0OrM2DxA==}
@@ -1745,10 +1770,17 @@ packages:
resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==}
engines: {node: '>= 14'}
+ resolution: {integrity: sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==}
+
resolution: {integrity: sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==}
engines: {node: '>=10.19.0'}
+ resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}
+ engines: {node: '>= 6'}
+
resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==}
engines: {node: '>= 14'}
@@ -2209,6 +2241,9 @@ packages:
resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==}
+ resolution: {integrity: sha512-60zvsJReQPX5/QP0Kzfd/VrpjScIQ7SHBW6bFCYfEP+fp0Eppr1SHhIO5nd1PjZtvclzSzES9D/p5nFJurwfWg==}
+
resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==}
@@ -2372,6 +2407,10 @@ packages:
resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==}
+ resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
+ engines: {node: '>= 6'}
+
resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
engines: {node: '>=8.10.0'}
@@ -2716,6 +2755,9 @@ packages:
resolution: {integrity: sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==}
engines: {node: '>= 18'}
+ resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==}
+
resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==}
engines: {node: '>=14.17'}
@@ -3081,6 +3123,13 @@ snapshots:
'@csstools/[email protected]':
optional: true
+ '@derhuerst/[email protected]':
+ dependencies:
+ caseless: 0.12.0
+ concat-stream: 2.0.0
+ http-response-object: 3.0.2
+ parse-cache-control: 1.0.1
+
'@electron-internal/[email protected]': {}
'@electron/[email protected]':
@@ -3646,6 +3695,8 @@ snapshots:
'@types/[email protected]': {}
+ '@types/[email protected]': {}
+
dependencies:
undici-types: 7.18.2
@@ -3761,6 +3812,12 @@ snapshots:
+ dependencies:
+ debug: 4.4.3
+ transitivePeerDependencies:
+ - supports-color
+
@@ -3961,6 +4018,8 @@ snapshots:
call-bind-apply-helpers: 1.0.2
get-intrinsic: 1.3.0
+
dependencies:
assertion-error: 2.0.1
@@ -4037,6 +4096,13 @@ snapshots:
+ dependencies:
+ buffer-from: 1.1.2
+ inherits: 2.0.4
+ readable-stream: 3.6.2
+ typedarray: 0.0.6
+
@@ -4401,6 +4467,15 @@ snapshots:
+ dependencies:
+ '@derhuerst/http-basic': 8.2.4
+ env-paths: 2.2.1
+ https-proxy-agent: 5.0.1
+ progress: 2.0.3
+ transitivePeerDependencies:
+ - supports-color
+
dependencies:
minimatch: 5.1.9
@@ -4618,11 +4693,22 @@ snapshots:
transitivePeerDependencies:
- supports-color
+ dependencies:
+ '@types/node': 10.17.60
+
dependencies:
quick-lru: 5.1.1
resolve-alpn: 1.2.1
+ dependencies:
+ agent-base: 6.0.2
+ debug: 4.4.3
+ transitivePeerDependencies:
+ - supports-color
+
dependencies:
agent-base: 7.1.4
@@ -5036,6 +5122,8 @@ snapshots:
+
dependencies:
entities: 6.0.1
@@ -5200,6 +5288,12 @@ snapshots:
string_decoder: 1.1.1
util-deprecate: 1.0.2
+ dependencies:
+ inherits: 2.0.4
+ string_decoder: 1.1.1
+ util-deprecate: 1.0.2
+
dependencies:
picomatch: 2.3.1
@@ -5610,6 +5704,8 @@ snapshots:
media-typer: 1.1.0
mime-types: 3.0.2
+
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index f9b6e0144..6ac9f3e3a 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -4,3 +4,4 @@ packages:
onlyBuiltDependencies:
- esbuild
- electron
+ - ffmpeg-static
diff --git a/sdk/etc/index.api.md b/sdk/etc/index.api.md
index 22f20fe72..d47bac42a 100644
--- a/sdk/etc/index.api.md
+++ b/sdk/etc/index.api.md
@@ -562,6 +562,7 @@ invalidateBuffer: (url: string) => boolean
muteBus: (busName: string, muted: boolean) => void
playBGM: (url: string, config?: { volume?: number; fadeIn?: number; crossFade?: number; }) => void
playSFX: (url: string, config?: { volume?: number; pitch?: number; pan?: number; priority?: number; }) => AudioHandle
+playTrack: (url: string, config?: PlayConfig) => Promise<AudioHandle | null>
preload: (url: string) => Promise<void>
preloadAll: (urls: string[]) => Promise<void>
preloadFromData: (url: string, data: ArrayBuffer) => Promise<void>
diff --git a/sdk/etc/index.node.api.md b/sdk/etc/index.node.api.md
index cdd39eb09..fc5c6aaf1 100644
--- a/sdk/etc/index.node.api.md
+++ b/sdk/etc/index.node.api.md
@@ -557,6 +557,7 @@ invalidateBuffer: (url: string) => boolean
muteBus: (busName: string, muted: boolean) => void
playBGM: (url: string, config?: { volume?: number; fadeIn?: number; crossFade?: number; }) => void
playSFX: (url: string, config?: { volume?: number; pitch?: number; pan?: number; priority?: number; }) => AudioHandle
+playTrack: (url: string, config?: PlayConfig) => Promise<AudioHandle | null>
preload: (url: string) => Promise<void>
preloadAll: (urls: string[]) => Promise<void>
preloadFromData: (url: string, data: ArrayBuffer) => Promise<void>
diff --git a/sdk/src/audio/Audio.ts b/sdk/src/audio/Audio.ts
index 079490c1d..b965237a8 100644
--- a/sdk/src/audio/Audio.ts
+++ b/sdk/src/audio/Audio.ts
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
-import type { PlatformAudioBackend, AudioBufferHandle, AudioHandle } from './PlatformAudioBackend';
+import type { PlatformAudioBackend, AudioBufferHandle, AudioHandle, PlayConfig } from './PlatformAudioBackend';
import type { AudioMixer, BusDuckRule } from './AudioMixer';
import type { BusEffectDef } from './BusEffects';
import { defineResource } from '../resource';
@@ -239,6 +239,30 @@ export class AudioAPI {
if (!this.bufferCache_.has(url)) this.insertEntry_(url, buffer);
}
+ /**
+ * Play a fully-RESOLVED clip URL on an explicit bus (created on demand
+ * under master). Unlike playSFX/playBGM this applies no ref resolution —
+ * the caller already holds the final URL (the video system's audio track
+ * derives it from the resolved video source). Null when the clip can't
+ * load; loaded buffers share the residency cache.
+ */
+ async playTrack(url: string, config: PlayConfig = {}): Promise<AudioHandle | null> {
+ if (this.disposed_) return null;
+ try {
+ if (!this.bufferCache_.has(url)) {
+ const buffer = await this.backend_.loadBuffer(url);
+ if (!this.bufferCache_.has(url)) this.insertEntry_(url, buffer);
+ }
+ } catch {
+ return null;
+ }
+ if (this.disposed_) return null;
+ const buffer = this.lookupBuffer_(url);
+ if (!buffer) return null;
+ if (config.bus) this.ensureBus(config.bus);
+ return this.backend_.play(buffer, config);
+ }
+
playSFX(url: string, config?: {
volume?: number;
pitch?: number;
diff --git a/sdk/src/video/PlatformVideoBackend.ts b/sdk/src/video/PlatformVideoBackend.ts
index 5e8fa8cdf..82f7f2d5c 100644
--- a/sdk/src/video/PlatformVideoBackend.ts
+++ b/sdk/src/video/PlatformVideoBackend.ts
@@ -48,10 +48,14 @@ export interface PlatformVideoBackend {
}
/**
- * What the app hands a platform's backend factory. `sideModules` is a lazy
- * getter — the host may attach to the app after plugins build, so backends
- * resolve it per stream, not at construction.
+ * What the app hands a platform's backend factory. Both accessors are lazy —
+ * the side-module host and the audio service may attach to the app after
+ * plugins build, so backends resolve them per stream, not at construction.
*/
export interface VideoBackendContext {
sideModules(): import('../sideModules/host').SideModuleHost | null;
+ /** The app's audio service, when the audio plugin is installed. The wasm
+ * backend plays the cook-demuxed `.m4a` audio track through it and slaves
+ * the video clock to the track's playhead. */
+ audio?(): import('../audio/Audio').AudioAPI | null;
}
diff --git a/sdk/src/video/VideoPlugin.ts b/sdk/src/video/VideoPlugin.ts
index c6b867c57..d10894ece 100644
--- a/sdk/src/video/VideoPlugin.ts
+++ b/sdk/src/video/VideoPlugin.ts
@@ -14,6 +14,7 @@ import type { UIVisualData } from '../component.generated';
import { UIVisual } from '../ui/core/ui-visual';
import type { VideoStreamHandle } from './PlatformVideoBackend';
import { NullVideoBackend } from './NullVideoBackend';
+import { Audio } from '../audio/Audio';
import { getPlatform } from '../platform/base';
import { isEditor, isPlayMode } from '../env';
import { log } from '../logger';
@@ -65,10 +66,15 @@ export class VideoPlugin implements Plugin {
build(app: App): void {
// createVideoBackend is optional; fall back to the silent Null backend.
- // The context's lazy sideModules getter lets the wasm backend resolve the
- // host per stream (it may attach to the app after plugins build).
- const backend = getPlatform().createVideoBackend?.({ sideModules: () => app.sideModules })
- ?? new NullVideoBackend();
+ // The context's getters are lazy so the wasm backend resolves the
+ // side-module host / audio service per stream (they may attach to the
+ // app after plugins build).
+ const backend = getPlatform().createVideoBackend?.({
+ sideModules: () => app.sideModules,
+ audio: () => {
+ try { return app.getResource(Audio); } catch { return null; }
+ },
+ }) ?? new NullVideoBackend();
const video = new VideoAPI(backend);
this.video_ = video;
app.insertResource(VideoPlayer, video);
diff --git a/sdk/src/video/WasmVideoBackend.ts b/sdk/src/video/WasmVideoBackend.ts
index 7a9000b1c..af936777a 100644
--- a/sdk/src/video/WasmVideoBackend.ts
+++ b/sdk/src/video/WasmVideoBackend.ts
@@ -5,9 +5,18 @@
// pump as every other backend. Runs wherever the engine's wasm runs — the
// guaranteed path on platforms without a reliable native decoder (WeChat, headless)
// — and owns the clock, so seek/currentTime/duration/rate all work uniformly.
+//
+// Audio: the cook demuxes the source's audio track into an `<staged>.m4a`
+// sibling played through the audio pipeline (bus-routed, so mixer/ducking
+// apply). When a track exists, IT is the clock — the video decodes toward the
+// track's playhead — since audio glitches are far more audible than a video
+// frame arriving late. Without a track (or when it ends first), the engine
+// wall clock drives.
import type { ESEngineModule } from '../wasm';
import type { PlatformVideoBackend, VideoBackendContext, VideoStreamHandle, VideoStreamOptions } from './PlatformVideoBackend';
import type { SideModule } from '../sideModules/host';
+import type { AudioAPI } from '../audio/Audio';
+import type { AudioHandle } from '../audio/PlatformAudioBackend';
import { createTextureFromPixels, updateTextureSubregion } from '../runtimeAssets';
import { requireResourceManager } from '../resourceManager';
import { getPlatform } from '../platform/base';
@@ -36,10 +45,23 @@ export interface VideoWasmModule {
readonly HEAPU8: Uint8Array;
}
+/** What the wasm stream needs from its backend (module + optional audio). */
+interface WasmStreamDeps {
+ acquire(): Promise<SideModule | null>;
+ audio(): AudioAPI | null;
+}
+
// Longest catch-up decoded in one pump. MPEG-1 has no frame-skip (P/B frames
// need every predecessor), so a huge dt after a hidden window would stall the
// frame decoding it all; clamping instead slows playback to real decode speed.
const MAX_PUMP_DT = 0.25;
+// An audio clock this far BEHIND the video is a loop wrap or backward seek —
+// exact-seek the video instead of waiting for the clock to catch up.
+const CLOCK_BACK_JUMP = 0.25;
+// Bus the audio track routes through (created on demand under master).
+const VIDEO_AUDIO_BUS = 'video';
+// Video extensions whose cooked `.m4a` audio-track sibling is worth probing.
+const AUDIO_SIBLING_RE = /\.(esv|mpg|mpeg)$/i;
class WasmVideoStreamHandle implements VideoStreamHandle {
readonly id = nextId_++;
@@ -61,20 +83,28 @@ class WasmVideoStreamHandle implements VideoStreamHandle {
private playing_: boolean;
private loop_: boolean;
private rate_: number;
+ private volume_: number;
+ private muted_: boolean;
private lastNow_ = -1;
private frameDirty_ = false;
private pendingSeek_ = -1;
+ private audioApi_: AudioAPI | null = null;
+ private audioUrl_: string | null = null;
+ private audio_: AudioHandle | null = null;
+ private audioGen_ = 0;
- constructor(url: string, options: VideoStreamOptions, acquire: () => Promise<SideModule | null>) {
+ constructor(url: string, options: VideoStreamOptions, deps: WasmStreamDeps) {
this.playing_ = options.autoplay ?? true;
this.loop_ = options.loop ?? false;
this.rate_ = options.playbackRate ?? 1;
- void this.start_(url, acquire);
+ this.volume_ = options.volume ?? 1;
+ this.muted_ = options.muted ?? false;
+ void this.start_(url, deps);
}
- private async start_(url: string, acquire: () => Promise<SideModule | null>): Promise<void> {
+ private async start_(url: string, deps: WasmStreamDeps): Promise<void> {
try {
- const [bytes, mod] = await Promise.all([this.loadBytes_(url), acquire()]);
+ const [bytes, mod] = await Promise.all([this.loadBytes_(url), deps.acquire()]);
if (this.disposed_) return;
if (!mod) throw new Error('the "videodec" side module is unavailable');
const video = mod as unknown as VideoWasmModule;
@@ -96,11 +126,16 @@ class WasmVideoStreamHandle implements VideoStreamHandle {
this.framePtr_ = video._malloc(this.frameBytes_);
if (!this.framePtr_) throw new Error('videodec out of memory');
video._es_video_set_loop(decoder, this.loop_ ? 1 : 0);
+ let startAt = 0;
if (this.pendingSeek_ >= 0) {
- video._es_video_seek(decoder, this.pendingSeek_);
+ if (video._es_video_seek(decoder, this.pendingSeek_)) this.frameDirty_ = true;
+ startAt = this.pendingSeek_;
this.pendingSeek_ = -1;
- this.frameDirty_ = true;
}
+
+ this.audioApi_ = deps.audio();
+ if (this.audioApi_ && AUDIO_SIBLING_RE.test(url)) this.audioUrl_ = `${url}.m4a`;
+ if (this.playing_) void this.startAudio_(startAt);
} catch (err) {
if (this.disposed_) return;
log.error('video', `wasm decode failed for "${url}"`, err);
@@ -119,6 +154,53 @@ class WasmVideoStreamHandle implements VideoStreamHandle {
return platform.readFile(url);
}
+ // === audio track ========================================================
+
+ /** (Re)start the audio-track sibling at `offset` seconds. A missing track
+ * resolves to null once and disables further probes (engine clock drives). */
+ private async startAudio_(offset: number): Promise<void> {
+ const api = this.audioApi_;
+ const url = this.audioUrl_;
+ if (!api || !url) return;
+ const gen = ++this.audioGen_;
+ const handle = await api.playTrack(url, {
+ bus: VIDEO_AUDIO_BUS,
+ volume: this.muted_ ? 0 : this.volume_,
+ loop: this.loop_,
+ playbackRate: this.rate_,
+ startOffset: offset,
+ });
+ if (!handle) {
+ if (gen === this.audioGen_) this.audioUrl_ = null; // no track — engine clock
+ return;
+ }
+ if (this.disposed_ || gen !== this.audioGen_) {
+ handle.stop();
+ return;
+ }
+ // The audio track is the clock now; the video must not free-wrap on its
+ // own (the wrap is driven by the track's playhead jumping back).
+ this.mod_?._es_video_set_loop(this.decoder_, 0);
+ // A non-looping track that ends BEFORE the last video frame hands the
+ // clock back to the engine so the tail still plays out.
+ handle.onEnd = () => {
+ if (this.audio_ === handle && !this.loop_) {
+ this.audio_ = null;
+ this.lastNow_ = -1;
+ }
+ };
+ if (!this.playing_) handle.pause();
+ this.audio_ = handle;
+ }
+
+ private stopAudio_(): void {
+ this.audioGen_++;
+ if (this.audio_) {
+ this.audio_.stop();
+ this.audio_ = null;
+ }
+ }
+
// === texture pump =======================================================
pump(module: ESEngineModule): void {
@@ -126,15 +208,33 @@ class WasmVideoStreamHandle implements VideoStreamHandle {
if (this.disposed_ || !video) return;
if (this.playing_ && !this.ended_) {
- const now = getPlatform().now();
- const dt = this.lastNow_ < 0 ? 0 : Math.min((now - this.lastNow_) / 1000, MAX_PUMP_DT);
- this.lastNow_ = now;
- if (dt > 0 && video._es_video_advance(this.decoder_, dt * this.rate_)) {
- this.frameDirty_ = true;
+ if (this.audio_) {
+ // Audio-track clock: decode toward the track's playhead.
+ const target = this.audio_.currentTime;
+ const cur = video._es_video_time(this.decoder_);
+ const dt = target - cur;
+ if (dt > 0.001) {
+ if (video._es_video_advance(this.decoder_, Math.min(dt, MAX_PUMP_DT))) {
+ this.frameDirty_ = true;
+ }
+ } else if (dt < -CLOCK_BACK_JUMP) {
+ // The track wrapped (loop) or jumped backwards — follow it.
+ if (video._es_video_seek(this.decoder_, Math.max(target, 0))) {
+ this.frameDirty_ = true;
+ }
+ }
+ } else {
+ const now = getPlatform().now();
+ const dt = this.lastNow_ < 0 ? 0 : Math.min((now - this.lastNow_) / 1000, MAX_PUMP_DT);
+ this.lastNow_ = now;
+ if (dt > 0 && video._es_video_advance(this.decoder_, dt * this.rate_)) {
+ this.frameDirty_ = true;
+ }
}
if (!this.loop_ && video._es_video_has_ended(this.decoder_)) {
this.ended_ = true;
this.playing_ = false;
+ this.stopAudio_();
this.onEnded?.();
}
}
@@ -176,10 +276,16 @@ class WasmVideoStreamHandle implements VideoStreamHandle {
if (this.ended_) {
this.ended_ = false;
this.seek(0);
+ return;
}
+ if (this.audio_) this.audio_.resume();
+ else if (this.mod_) void this.startAudio_(this.currentTime);
}
- pause(): void { this.playing_ = false; }
+ pause(): void {
+ this.playing_ = false;
+ this.audio_?.pause();
+ }
seek(timeSeconds: number): void {
const t = timeSeconds > 0 ? timeSeconds : 0;
@@ -189,23 +295,37 @@ class WasmVideoStreamHandle implements VideoStreamHandle {
}
this.ended_ = false;
if (this.mod_._es_video_seek(this.decoder_, t)) this.frameDirty_ = true;
+ // AudioHandles have no in-place seek — restart the track at the offset.
+ this.stopAudio_();
+ if (this.playing_) void this.startAudio_(t);
}
- // The wasm path decodes video only — the cook demuxes the audio track for
- // the audio pipeline, so per-stream volume/mute live there, not here.
- setVolume(_volume: number): void {}
- setMuted(_muted: boolean): void {}
+ setVolume(volume: number): void {
+ this.volume_ = volume;
+ this.audio_?.setVolume(this.muted_ ? 0 : volume);
+ }
+
+ setMuted(muted: boolean): void {
+ this.muted_ = muted;
+ this.audio_?.setVolume(muted ? 0 : this.volume_);
+ }
setLoop(loop: boolean): void {
this.loop_ = loop;
- this.mod_?._es_video_set_loop(this.decoder_, loop ? 1 : 0);
+ this.audio_?.setLoop(loop);
+ // Without an audio clock the decoder wraps itself.
+ if (!this.audio_) this.mod_?._es_video_set_loop(this.decoder_, loop ? 1 : 0);
}
- setPlaybackRate(rate: number): void { this.rate_ = rate > 0 ? rate : 0; }
+ setPlaybackRate(rate: number): void {
+ this.rate_ = rate > 0 ? rate : 0;
+ this.audio_?.setPlaybackRate(this.rate_);
+ }
stop(): void {
if (this.disposed_) return;
this.disposed_ = true;
+ this.stopAudio_();
if (this.mod_) {
this.mod_._es_video_close(this.decoder_);
if (this.framePtr_) this.mod_._free(this.framePtr_);
@@ -226,10 +346,13 @@ export class WasmVideoBackend implements PlatformVideoBackend {
constructor(private readonly ctx_: VideoBackendContext) {}
createStream(url: string, options: VideoStreamOptions): VideoStreamHandle {
- return new WasmVideoStreamHandle(url, options, () => {
- const host = this.ctx_.sideModules();
- if (!host) return Promise.resolve(null);
- return host.acquire('videodec');
+ return new WasmVideoStreamHandle(url, options, {
+ acquire: () => {
+ const host = this.ctx_.sideModules();
+ if (!host) return Promise.resolve(null);
+ return host.acquire('videodec');
+ },
+ audio: () => this.ctx_.audio?.() ?? null,
});
}
diff --git a/sdk/tests/audio-api.test.ts b/sdk/tests/audio-api.test.ts
index 99b39b957..c5f23a691 100644
--- a/sdk/tests/audio-api.test.ts
+++ b/sdk/tests/audio-api.test.ts
@@ -281,4 +281,36 @@ describe('AudioAPI', () => {
expect(backend2.loadBuffer).toHaveBeenCalledTimes(1);
});
});
+
+ describe('playTrack', () => {
+ it('loads the URL verbatim (no ref resolution) and plays it on the bus', async () => {
+ const backend = createMockBackend();
+ const audio = new AudioAPI(backend, createMockMixer());
+ audio.setRefResolver(() => { throw new Error('must not resolve'); });
+ const handle = await audio.playTrack('assets/clip.esv.m4a', { bus: 'video', startOffset: 1.5, volume: 0.5 });
+ expect(handle).not.toBeNull();
+ expect(backend.loadBuffer).toHaveBeenCalledWith('assets/clip.esv.m4a');
+ expect(backend.play).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({
+ bus: 'video', startOffset: 1.5, volume: 0.5,
+ }));
+ });
+
+ it('creates the bus on demand and caches the buffer across plays', async () => {
+ const backend = createMockBackend();
+ const mixer = createMockMixer();
+ (mixer.getBus as ReturnType<typeof vi.fn>).mockReturnValue(undefined);
+ const audio = new AudioAPI(backend, mixer);
+ await audio.playTrack('t.m4a', { bus: 'video' });
+ expect(mixer.createBus).toHaveBeenCalledWith(expect.objectContaining({ name: 'video' }));
+ await audio.playTrack('t.m4a', { bus: 'video' });
+ expect(backend.loadBuffer).toHaveBeenCalledTimes(1);
+ });
+
+ it('resolves null when the clip cannot load', async () => {
+ const backend = createMockBackend();
+ (backend.loadBuffer as ReturnType<typeof vi.fn>).mockRejectedValue(new Error('404'));
+ const audio = new AudioAPI(backend, createMockMixer());
+ expect(await audio.playTrack('missing.m4a')).toBeNull();
+ });
+ });
});
diff --git a/sdk/tests/wasm-video-backend.test.ts b/sdk/tests/wasm-video-backend.test.ts
index 385b65dea..b0ba63b9e 100644
--- a/sdk/tests/wasm-video-backend.test.ts
+++ b/sdk/tests/wasm-video-backend.test.ts
@@ -5,6 +5,8 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { WasmVideoBackend, type VideoWasmModule } from '../src/video/WasmVideoBackend';
import type { SideModuleHost } from '../src/sideModules/host';
+import type { AudioAPI } from '../src/audio/Audio';
+import type { AudioHandle } from '../src/audio/PlatformAudioBackend';
import { setPlatform } from '../src/platform/base';
import { initResourceManager, shutdownResourceManager } from '../src/resourceManager';
import type { PlatformAdapter } from '../src/platform/types';
@@ -234,3 +236,133 @@ describe('WasmVideoBackend', () => {
expect(() => handle.pump(engineModule)).not.toThrow();
});
});
+
+// === audio-track clock ======================================================
+
+function mockAudio() {
+ const handles: FakeAudioHandle[] = [];
+ class FakeAudioHandle {
+ time = 0;
+ playing = true;
+ onEnd?: () => void;
+ stop = vi.fn(() => { this.playing = false; });
+ pause = vi.fn(() => { this.playing = false; });
+ resume = vi.fn(() => { this.playing = true; });
+ setVolume = vi.fn();
+ setPan = vi.fn();
+ setLoop = vi.fn();
+ setPlaybackRate = vi.fn();
+ get isPlaying() { return this.playing; }
+ get currentTime() { return this.time; }
+ get duration() { return 2.5; }
+ readonly id = handles.length + 1;
+ }
+ const playTrack = vi.fn(async (_url: string, cfg: { startOffset?: number }) => {
+ const h = new FakeAudioHandle();
+ h.time = cfg.startOffset ?? 0;
+ handles.push(h);
+ return h as unknown as AudioHandle;
+ });
+ const api = { playTrack } as unknown as AudioAPI;
+ return { api, playTrack, handles };
+}
+
+describe('WasmVideoBackend audio-track clock', () => {
+ it('starts the .m4a sibling on the video bus and slaves the video clock to it', async () => {
+ const { mod, raw } = mockVideoModule();
+ const { api, playTrack, handles } = mockAudio();
+ const backend = new WasmVideoBackend({ sideModules: () => hostFor(mod), audio: () => api });
+ const handle = backend.createStream('videos/clip.esv', { volume: 0.5, loop: false });
+ await flush();
+
+ expect(playTrack).toHaveBeenCalledWith('videos/clip.esv.m4a', expect.objectContaining({
+ bus: 'video', volume: 0.5, loop: false, startOffset: 0,
+ }));
+ // The decoder must not free-wrap while the track drives the clock.
+ expect(raw._es_video_set_loop).toHaveBeenLastCalledWith(1, 0);
+
+ handles[0].time = 0.1;
+ handle.pump(engineModule);
+ expect((raw._es_video_advance.mock.calls.at(-1) as number[])[1]).toBeCloseTo(0.1);
+
+ // Audio clock jumping far backwards (loop wrap) exact-seeks the video.
+ raw._es_video_advance(1, 2.0); // decoder time → ~2.1
+ handles[0].time = 0.02;
+ handle.pump(engineModule);
+ expect(raw._es_video_seek).toHaveBeenCalledWith(1, 0.02);
+ handle.stop();
+ expect(handles[0].stop).toHaveBeenCalled();
+ });
+
+ it('muted stream starts its track at volume 0; volume/mute changes forward', async () => {
+ const { mod } = mockVideoModule();
+ const { api, playTrack, handles } = mockAudio();
+ const backend = new WasmVideoBackend({ sideModules: () => hostFor(mod), audio: () => api });
+ const handle = backend.createStream('clip.esv', { muted: true, volume: 0.8 });
+ await flush();
+ expect(playTrack).toHaveBeenCalledWith('clip.esv.m4a', expect.objectContaining({ volume: 0 }));
+ handle.setMuted(false);
+ expect(handles[0].setVolume).toHaveBeenLastCalledWith(0.8);
+ handle.setVolume(0.3);
+ expect(handles[0].setVolume).toHaveBeenLastCalledWith(0.3);
+ handle.stop();
+ });
+
+ it('seek restarts the track at the offset; pause/play suspend and resume it', async () => {
+ const { mod } = mockVideoModule();
+ const { api, playTrack, handles } = mockAudio();
+ const backend = new WasmVideoBackend({ sideModules: () => hostFor(mod), audio: () => api });
+ const handle = backend.createStream('clip.esv', {});
+ await flush();
+ handle.seek(1.5);
+ await flush();
+ expect(handles[0].stop).toHaveBeenCalled();
+ expect(playTrack).toHaveBeenLastCalledWith('clip.esv.m4a', expect.objectContaining({ startOffset: 1.5 }));
+ handle.pause();
+ expect(handles[1].pause).toHaveBeenCalled();
+ handle.play();
+ expect(handles[1].resume).toHaveBeenCalled();
+ handle.stop();
+ });
+
+ it('a missing track falls back to the engine clock', async () => {
+ const { mod, raw } = mockVideoModule();
+ const playTrack = vi.fn(async () => null);
+ const api = { playTrack } as unknown as AudioAPI;
+ const backend = new WasmVideoBackend({ sideModules: () => hostFor(mod), audio: () => api });
+ const handle = backend.createStream('clip.esv', {});
+ await flush();
+ expect(playTrack).toHaveBeenCalledTimes(1);
+ nowMs = 16;
+ handle.pump(engineModule);
+ nowMs = 32;
+ handle.pump(engineModule);
+ expect(raw._es_video_advance).toHaveBeenCalled();
+ handle.stop();
+ });
+
+ it('a non-looping track ending early hands the clock back to the engine', async () => {
+ const { mod, raw } = mockVideoModule();
+ const { api, handles } = mockAudio();
+ const backend = new WasmVideoBackend({ sideModules: () => hostFor(mod), audio: () => api });
+ const handle = backend.createStream('clip.esv', { loop: false });
+ await flush();
+ handles[0].onEnd?.();
+ nowMs = 16;
+ handle.pump(engineModule); // clock base
+ nowMs = 32;
+ handle.pump(engineModule);
+ expect(raw._es_video_advance).toHaveBeenCalled(); // engine clock drove
+ handle.stop();
+ });
+
+ it('never probes a sibling for non-esv sources', async () => {
+ const { mod } = mockVideoModule();
+ const { api, playTrack } = mockAudio();
+ const backend = new WasmVideoBackend({ sideModules: () => hostFor(mod), audio: () => api });
+ const handle = backend.createStream('clip.bin', {});
+ await flush();
+ expect(playTrack).not.toHaveBeenCalled();
+ handle.stop();
+ });
+});