A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.
TypeScript 2d-game-engine3d3d-game-engineandroidcppecsgame-editorgame-enginegamedeviostypescriptwebassemblywebglwebgpuwechat-minigame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
* @file videoCook.ts — cook-time video transcode for the wasm decode path.
* WeChat's only video backend is the engine-owned MPEG-1 decoder
* (videodec side module), so the cook turns each authored video into a
* codec-tagged `.esv` (MPEG-1 Program Stream) plus an `.m4a` audio-track
* sibling played through the audio pipeline. Transcoding shells out to
* the bundled ffmpeg binary (ffmpeg-static) — a separate process, loaded
* lazily and only when a cook actually ships video.
*/
import { spawn } from 'node:child_process';
import { existsSync, readdirSync } from 'node:fs';
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
/** Resolve the ffmpeg binary: env override first, then the bundled static
* build. Inside a packaged app the binary must run from the unpacked twin of
* the asar (executables cannot spawn from an archive). Null if unavailable —
* including when the resolved path dangles (e.g. ffmpeg-static was bundled,
* which rebases its __dirname-derived path onto the bundle dir). */
export async function resolveFfmpeg(): Promise<string | null> {
if (process.env.ESTELLA_FFMPEG) return process.env.ESTELLA_FFMPEG;
try {
const mod = await import('ffmpeg-static');
const p = (mod.default ?? mod) as unknown as string | null;
const real = p ? p.replace(`app.asar${path.sep}`, `app.asar.unpacked${path.sep}`) : null;
if (real && existsSync(real)) return real;
} catch { /* not resolvable from here — try the workspace below */ }
return resolveFfmpegFromWorkspace();
}
/**
* The binary the workspace installed, found by walking up from the working
* directory. The bundled import above answers with a path derived from its own
* __dirname, which a bundle rebases onto the bundle's directory — so in the
* headless export (esbuild) it dangles, and the cook would ship an untranscoded
* video that cannot play. This is the same binary, located the way a shell
* would.
*/
function resolveFfmpegFromWorkspace(): string | null {
const exe = process.platform === 'win32' ? 'ffmpeg.exe' : 'ffmpeg';
for (let dir = process.cwd(); ; dir = path.dirname(dir)) {
const modules = path.join(dir, 'node_modules');
const direct = path.join(modules, 'ffmpeg-static', exe);
if (existsSync(direct)) return direct;
// pnpm keeps the real package under .pnpm/<name>@<version>/node_modules/.
const store = path.join(modules, '.pnpm');
if (existsSync(store)) {
for (const entry of readdirSync(store)) {
if (!entry.startsWith('ffmpeg-static@')) continue;
const nested = path.join(store, entry, 'node_modules', 'ffmpeg-static', exe);
if (existsSync(nested)) return nested;
}
}
const parent = path.dirname(dir);
if (parent === dir) return null;
}
}
function run(bin: string, args: string[]): Promise<{ code: number; stderr: string }> {
return new Promise((resolve, reject) => {
const child = spawn(bin, args, { stdio: ['ignore', 'ignore', 'pipe'] });
let stderr = '';
child.stderr.on('data', (d: Buffer) => { stderr += d.toString(); });
child.on('error', reject);
child.on('close', (code) => resolve({ code: code ?? -1, stderr }));
});
}
export interface VideoImportSettings {
/** mpeg1video quantizer (2 = near-lossless, larger = smaller file). */
quality: number;
audioBitrateKbps: number;
}
/** Per-asset video import settings from the `.meta` importer block (tolerant). */
export function videoImportSettings(importer: Record<string, unknown> | undefined): VideoImportSettings {
const q = importer?.quality;
const kbps = importer?.audioBitrateKbps;
return {
quality: typeof q === 'number' && q >= 2 && q <= 31 ? Math.round(q) : 4,
audioBitrateKbps: kbps === 96 || kbps === 128 || kbps === 192 ? (kbps as number) : 128,
};
}
export interface VideoCookResult {
/** MPEG-1 PS bytes, or null when the transcode failed. */
esv: Uint8Array | null;
/** AAC (m4a) audio track, or null when the source has none. */
audio: Uint8Array | null;
warnings: string[];
}
// MPEG-1 dimensions must be even (macroblock chroma), and its frame rate is a
// fixed table — sources with a rate outside it are resampled on retry.
const EVEN_SCALE = 'scale=trunc(iw/2)*2:trunc(ih/2)*2';
/**
* Transcode `srcAbs` into the wasm decode path's artifacts. Video failures are
* reported (esv null); a missing audio track is normal and silent.
*/
export async function transcodeVideoForWasm(
srcAbs: string,
settings: VideoImportSettings,
): Promise<VideoCookResult> {
const warnings: string[] = [];
const ffmpeg = await resolveFfmpeg();
if (!ffmpeg) {
return { esv: null, audio: null, warnings: ['ffmpeg unavailable — cannot transcode video for the wasm decoder'] };
}
const tmp = await mkdtemp(path.join(tmpdir(), 'estella-video-'));
try {
const esvPath = path.join(tmp, 'out.esv');
const m4aPath = path.join(tmp, 'out.m4a');
const videoArgs = (extraFilter: string) => [
'-y', '-i', srcAbs, '-an',
'-c:v', 'mpeg1video', '-q:v', String(settings.quality),
'-vf', extraFilter ? `${EVEN_SCALE},${extraFilter}` : EVEN_SCALE,
'-f', 'mpeg', esvPath,
];
// The two passes are independent (-an vs -vn, separate outputs) — overlap them.
const [res, audioRes] = await Promise.all([
(async () => {
const first = await run(ffmpeg, videoArgs(''));
if (first.code === 0 || !/fps|frame ?rate/i.test(first.stderr)) return first;
// Source frame rate is outside MPEG-1's table (or variable) — resample.
return run(ffmpeg, videoArgs('fps=30'));
})(),
run(ffmpeg, [
'-y', '-i', srcAbs, '-vn',
'-c:a', 'aac', '-b:a', `${settings.audioBitrateKbps}k`,
'-f', 'ipod', m4aPath,
]),
]);
let esv: Uint8Array | null = null;
if (res.code === 0) {
esv = new Uint8Array(await readFile(esvPath));
} else {
warnings.push(`video transcode failed: ${lastLine(res.stderr)}`);
}
let audio: Uint8Array | null = null;
if (audioRes.code === 0) {
audio = new Uint8Array(await readFile(m4aPath));
} else if (!/does not contain any stream|Stream map .* matches no streams|no audio/i.test(audioRes.stderr)) {
warnings.push(`audio track extract failed: ${lastLine(audioRes.stderr)}`);
}
return { esv, audio, warnings };
} catch (err) {
// spawn-level failure (ENOENT, EPERM) — report, don't abort the cook loop.
warnings.push(`ffmpeg failed to run: ${err instanceof Error ? err.message : String(err)}`);
return { esv: null, audio: null, warnings };
} finally {
await rm(tmp, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
}
}
function lastLine(stderr: string): string {
const lines = stderr.trim().split('\n');
return lines[lines.length - 1] ?? 'unknown ffmpeg error';
}