Skip to content

A fast 2D and 3D game engine — TypeScript SDK, C++/WebAssembly core, visual editor. Ship one project to web, desktop, WeChat MiniGames, playable ads, and native Android / iOS.

6.9 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright (c) 2024-present ESEngine Team
/**
 * @file  videoCook.ts — cook-time video transcode for the wasm decode path.
 *        WeChat's only video backend is the engine-owned MPEG-1 decoder
 *        (videodec side module), so the cook turns each authored video into a
 *        codec-tagged `.esv` (MPEG-1 Program Stream) plus an `.m4a` audio-track
 *        sibling played through the audio pipeline. Transcoding shells out to
 *        the bundled ffmpeg binary (ffmpeg-static) — a separate process, loaded
 *        lazily and only when a cook actually ships video.
 */
import { spawn } from 'node:child_process';
import { existsSync, readdirSync } from 'node:fs';
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';

/** Resolve the ffmpeg binary: env override first, then the bundled static
 *  build. Inside a packaged app the binary must run from the unpacked twin of
 *  the asar (executables cannot spawn from an archive). Null if unavailable —
 *  including when the resolved path dangles (e.g. ffmpeg-static was bundled,
 *  which rebases its __dirname-derived path onto the bundle dir). */
export async function resolveFfmpeg(): Promise<string | null> {
  if (process.env.ESTELLA_FFMPEG) return process.env.ESTELLA_FFMPEG;
  try {
    const mod = await import('ffmpeg-static');
    const p = (mod.default ?? mod) as unknown as string | null;
    const real = p ? p.replace(`app.asar${path.sep}`, `app.asar.unpacked${path.sep}`) : null;
    if (real && existsSync(real)) return real;
  } catch { /* not resolvable from here — try the workspace below */ }
  return resolveFfmpegFromWorkspace();
}

/**
 * The binary the workspace installed, found by walking up from the working
 * directory. The bundled import above answers with a path derived from its own
 * __dirname, which a bundle rebases onto the bundle's directory — so in the
 * headless export (esbuild) it dangles, and the cook would ship an untranscoded
 * video that cannot play. This is the same binary, located the way a shell
 * would.
 */
function resolveFfmpegFromWorkspace(): string | null {
  const exe = process.platform === 'win32' ? 'ffmpeg.exe' : 'ffmpeg';
  for (let dir = process.cwd(); ; dir = path.dirname(dir)) {
    const modules = path.join(dir, 'node_modules');
    const direct = path.join(modules, 'ffmpeg-static', exe);
    if (existsSync(direct)) return direct;
    // pnpm keeps the real package under .pnpm/<name>@<version>/node_modules/.
    const store = path.join(modules, '.pnpm');
    if (existsSync(store)) {
      for (const entry of readdirSync(store)) {
        if (!entry.startsWith('ffmpeg-static@')) continue;
        const nested = path.join(store, entry, 'node_modules', 'ffmpeg-static', exe);
        if (existsSync(nested)) return nested;
      }
    }
    const parent = path.dirname(dir);
    if (parent === dir) return null;
  }
}

function run(bin: string, args: string[]): Promise<{ code: number; stderr: string }> {
  return new Promise((resolve, reject) => {
    const child = spawn(bin, args, { stdio: ['ignore', 'ignore', 'pipe'] });
    let stderr = '';
    child.stderr.on('data', (d: Buffer) => { stderr += d.toString(); });
    child.on('error', reject);
    child.on('close', (code) => resolve({ code: code ?? -1, stderr }));
  });
}

export interface VideoImportSettings {
  /** mpeg1video quantizer (2 = near-lossless, larger = smaller file). */
  quality: number;
  audioBitrateKbps: number;
}

/** Per-asset video import settings from the `.meta` importer block (tolerant). */
export function videoImportSettings(importer: Record<string, unknown> | undefined): VideoImportSettings {
  const q = importer?.quality;
  const kbps = importer?.audioBitrateKbps;
  return {
    quality: typeof q === 'number' && q >= 2 && q <= 31 ? Math.round(q) : 4,
    audioBitrateKbps: kbps === 96 || kbps === 128 || kbps === 192 ? (kbps as number) : 128,
  };
}

export interface VideoCookResult {
  /** MPEG-1 PS bytes, or null when the transcode failed. */
  esv: Uint8Array | null;
  /** AAC (m4a) audio track, or null when the source has none. */
  audio: Uint8Array | null;
  warnings: string[];
}

// MPEG-1 dimensions must be even (macroblock chroma), and its frame rate is a
// fixed table — sources with a rate outside it are resampled on retry.
const EVEN_SCALE = 'scale=trunc(iw/2)*2:trunc(ih/2)*2';

/**
 * Transcode `srcAbs` into the wasm decode path's artifacts. Video failures are
 * reported (esv null); a missing audio track is normal and silent.
 */
export async function transcodeVideoForWasm(
  srcAbs: string,
  settings: VideoImportSettings,
): Promise<VideoCookResult> {
  const warnings: string[] = [];
  const ffmpeg = await resolveFfmpeg();
  if (!ffmpeg) {
    return { esv: null, audio: null, warnings: ['ffmpeg unavailable — cannot transcode video for the wasm decoder'] };
  }
  const tmp = await mkdtemp(path.join(tmpdir(), 'estella-video-'));
  try {
    const esvPath = path.join(tmp, 'out.esv');
    const m4aPath = path.join(tmp, 'out.m4a');
    const videoArgs = (extraFilter: string) => [
      '-y', '-i', srcAbs, '-an',
      '-c:v', 'mpeg1video', '-q:v', String(settings.quality),
      '-vf', extraFilter ? `${EVEN_SCALE},${extraFilter}` : EVEN_SCALE,
      '-f', 'mpeg', esvPath,
    ];
    // The two passes are independent (-an vs -vn, separate outputs) — overlap them.
    const [res, audioRes] = await Promise.all([
      (async () => {
        const first = await run(ffmpeg, videoArgs(''));
        if (first.code === 0 || !/fps|frame ?rate/i.test(first.stderr)) return first;
        // Source frame rate is outside MPEG-1's table (or variable) — resample.
        return run(ffmpeg, videoArgs('fps=30'));
      })(),
      run(ffmpeg, [
        '-y', '-i', srcAbs, '-vn',
        '-c:a', 'aac', '-b:a', `${settings.audioBitrateKbps}k`,
        '-f', 'ipod', m4aPath,
      ]),
    ]);

    let esv: Uint8Array | null = null;
    if (res.code === 0) {
      esv = new Uint8Array(await readFile(esvPath));
    } else {
      warnings.push(`video transcode failed: ${lastLine(res.stderr)}`);
    }
    let audio: Uint8Array | null = null;
    if (audioRes.code === 0) {
      audio = new Uint8Array(await readFile(m4aPath));
    } else if (!/does not contain any stream|Stream map .* matches no streams|no audio/i.test(audioRes.stderr)) {
      warnings.push(`audio track extract failed: ${lastLine(audioRes.stderr)}`);
    }
    return { esv, audio, warnings };
  } catch (err) {
    // spawn-level failure (ENOENT, EPERM) — report, don't abort the cook loop.
    warnings.push(`ffmpeg failed to run: ${err instanceof Error ? err.message : String(err)}`);
    return { esv: null, audio: null, warnings };
  } finally {
    await rm(tmp, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
  }
}

function lastLine(stderr: string): string {
  const lines = stderr.trim().split('\n');
  return lines[lines.length - 1] ?? 'unknown ffmpeg error';
}