Open source CapCut alternative.
TypeScript
fix(cut): select playable media streams during conversion [rebuild]
David Le3af045b3a7dc
diff --git a/site/src/cut/lib/mediaRead.ts b/site/src/cut/lib/mediaRead.tsindex a3f0f7bb..af753888 100644--- a/site/src/cut/lib/mediaRead.ts+++ b/site/src/cut/lib/mediaRead.ts@@ -68,9 +68,7 @@ export interface FrameSize {}export class UnreadableMediaError extends Error {- /** The container parsed and the file carries video — this browser just has- * no decoder for it. Callers that can convert branch on this rather than on- * the message. */+ /** The container parsed, but a media stream needs a decoder this runtime lacks. */readonly undecodable: boolean;constructor(message = "Cut can't read this media file.", undecodable = false) {@@ -362,9 +360,9 @@ export async function hasUndecodableVideo(input: Input): Promise<boolean> {/** The primary audio track, or null — same decodability rule as video. */export async function audioTrackOf(input: Input): Promise<InputAudioTrack | null> {- const track = await tracked(input.getPrimaryAudioTrack());- if (!track) return null;- return headless() || (await track.canDecode()) ? track : null;+ return tracked(input.getPrimaryAudioTrack({+ filter: (track) => headless() ? track.codec !== null : track.canDecode(),+ }));}/** Read what a file is: how long, whether it carries picture or sound, and at@@ -384,6 +382,9 @@ export async function probeMediaFile(src: string | Blob): Promise<MediaProbe> {true);}+ if (!audio && (await tracked(input.getAudioTracks())).length) {+ throw new UnreadableMediaError("This file's audio needs conversion.", true);+ }if (!video && !audio) throw new UnreadableMediaError();const [duration, bytes] = await Promise.all([input.computeDuration(), input.source.getSizeOrNull()]);const size = bytes === null ? {} : { sizeBytes: bytes };diff --git a/site/src/cut/server/ai/headlessTools.ts b/site/src/cut/server/ai/headlessTools.tsindex fa3556f2..dfb18e21 100644--- a/site/src/cut/server/ai/headlessTools.ts+++ b/site/src/cut/server/ai/headlessTools.ts@@ -80,9 +80,10 @@ setHostMediaStore({const handle = { tmpDir: "", outPath, progress: 0, log: [] as string[] };const outcome = await convertToMp4(handle, src, outPath, { maxHeight: opts.maxHeight });return {- fileName,+ fileName: outcome.unchanged ? source.fileName : fileName,+ ...(outcome.unchanged ? { unchanged: true } : {}),reencoded: outcome.transcodedVideo || outcome.transcodedAudio,- duration: await probeDuration(outPath),+ duration: await probeDuration(outcome.unchanged ? src : outPath),...(outcome.width !== undefined ? { width: outcome.width, height: outcome.height } : {}),};},diff --git a/site/src/cut/server/convert.test.ts b/site/src/cut/server/convert.test.tsnew file mode 100644index 00000000..020c1b07--- /dev/null+++ b/site/src/cut/server/convert.test.ts@@ -0,0 +1,74 @@+import { afterAll, beforeAll, describe, expect, test } from "bun:test";+import { execFileSync } from "node:child_process";+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { convertToMp4, selectConversionStreams, streamCodecs } from "./convert";+import { audioTrackOf, openMedia } from "@/cut/lib/mediaRead";++let dir: string;+beforeAll(async () => { dir = await mkdtemp(path.join(os.tmpdir(), "cut-convert-test-")); });+afterAll(async () => { await rm(dir, { recursive: true, force: true }); });+const ffmpeg = (args: string[]) => execFileSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", ...args], { timeout: 30_000 });+const handle = (out: string) => ({ tmpDir: dir, outPath: out, progress: 0, log: [] as string[] });++describe("import conversion", () => {+ test("selects stereo AAC when an unsupported spatial track comes first", () => {+ expect(selectConversionStreams([+ { index: 0, codec_type: "video", codec_name: "hevc" },+ { index: 1, codec_type: "audio", codec_tag_string: "apac", disposition: { default: 1 } },+ { index: 2, codec_type: "audio", codec_name: "aac", codec_tag_string: "mp4a" },+ ])).toMatchObject({ video: "hevc", audio: "aac", audioIndex: 2, remux: true });+ });++ test("a file with only unsupported sound fails explicitly", () => {+ expect(() => selectConversionStreams([+ { index: 0, codec_type: "video", codec_name: "hevc" },+ { index: 1, codec_type: "audio", codec_tag_string: "apac" },+ ])).toThrow("no supported audio track");+ });++ test("converts HEVC video and PCM audio to playable H.264/AAC", async () => {+ const src = path.join(dir, "phone.mov");+ const out = path.join(dir, "phone.mp4");+ ffmpeg(["-f", "lavfi", "-i", "color=c=red:s=64x64:r=10:d=0.3", "-f", "lavfi", "-i", "sine=frequency=440:duration=0.3",+ "-c:v", "libx265", "-x265-params", "log-level=error:pools=1", "-pix_fmt", "yuv420p10le", "-tag:v", "hvc1", "-c:a", "pcm_s16le", "-shortest", src]);+ const before = await readFile(src);+ expect(await convertToMp4(handle(out), src, out)).toMatchObject({ transcodedVideo: true, transcodedAudio: true, width: 64, height: 64 });+ expect(await streamCodecs(out)).toMatchObject({ video: "h264", audio: "aac", videoNeedsEncoding: false });+ expect(await readFile(src)).toEqual(before);+ }, 30_000);++ test("remuxes a MOV with an unknown first audio track using its second AAC track", async () => {+ const src = path.join(dir, "spatial.mov");+ const out = path.join(dir, "stereo.mp4");+ ffmpeg(["-f", "lavfi", "-i", "color=s=64x64:r=10:d=0.3", "-f", "lavfi", "-i", "sine=duration=0.3",+ "-map", "0:v", "-map", "1:a", "-map", "1:a", "-c:v", "libx264", "-c:a", "aac", "-shortest", src]);+ const bytes = await readFile(src);+ const audioEntry = bytes.indexOf(Buffer.from("mp4a"));+ expect(audioEntry).toBeGreaterThan(0);+ bytes.write("apac", audioEntry, "ascii");+ await writeFile(src, bytes);+ const input = openMedia(new Blob([bytes]));+ try {+ expect((await audioTrackOf(input))?.codec).toBe("aac");+ } finally { input.dispose(); }+ expect(await streamCodecs(src)).toMatchObject({ audio: "aac", audioIndex: 2, remux: true });+ expect(await convertToMp4(handle(out), src, out)).toMatchObject({ transcodedVideo: false, transcodedAudio: false });+ expect(await streamCodecs(out)).toMatchObject({ audio: "aac", remux: false });+ }, 30_000);++ test("already playable MP4 stays unchanged", async () => {+ const src = path.join(dir, "ready.mp4");+ ffmpeg(["-f", "lavfi", "-i", "color=s=64x64:r=10:d=0.2", "-c:v", "libx264", src]);+ expect(await convertToMp4(handle(path.join(dir, "unused.mp4")), src, path.join(dir, "unused.mp4"))).toMatchObject({ unchanged: true });+ });++ test("audio-only phone media becomes playable AAC without a video stream", async () => {+ const src = path.join(dir, "recording.mov");+ const out = path.join(dir, "recording.m4a");+ ffmpeg(["-f", "lavfi", "-i", "sine=duration=0.2", "-c:a", "pcm_s16le", src]);+ expect(await convertToMp4(handle(out), src, out)).toMatchObject({ transcodedVideo: false, transcodedAudio: true });+ expect(await streamCodecs(out)).toMatchObject({ video: undefined, audio: "aac" });+ });+});diff --git a/site/src/cut/server/convert.ts b/site/src/cut/server/convert.tsindex 938d70e7..83699d15 100644--- a/site/src/cut/server/convert.ts+++ b/site/src/cut/server/convert.ts@@ -44,53 +44,67 @@ const MP4_EXT = /\.(mp4|m4v)$/i;* to leave it alone. */export const fitsAlready = (file: string,- codecs: { video?: string; audio?: string },+ codecs: { video?: string; audio?: string; remux?: boolean; videoNeedsEncoding?: boolean },work: { shrink: boolean; sdr: boolean }) =>MP4_EXT.test(file) &&!work.shrink &&!work.sdr &&+ !codecs.remux &&+ !codecs.videoNeedsEncoding &&(!codecs.video || KEEPS_VIDEO.has(codecs.video)) &&(!codecs.audio || KEEPS_AUDIO.has(codecs.audio));-/** The first video and audio stream's codec names, or undefined for a file- * that carries no such stream (or a probe that fails — the conversion then- * takes the re-encode path, which is correct for anything). */-export function streamCodecs(file: string): Promise<{ video?: string; audio?: string }> {- return new Promise((resolve) => {+export type MediaStream = {+ index: number;+ codec_type?: string;+ codec_name?: string;+ codec_tag_string?: string;+ pix_fmt?: string;+ disposition?: { default?: number; attached_pic?: number };+};++/** Choose the playable audio rendition carried beside optional spatial tracks. */+export function selectConversionStreams(streams: MediaStream[]) {+ const video = streams.find((s) => s.codec_type === "video" && !s.disposition?.attached_pic);+ const audioTracks = streams.filter((s) => s.codec_type === "audio");+ const knownAudio = audioTracks.filter((s) => s.codec_name && s.codec_name !== "unknown" && s.codec_tag_string !== "apac");+ const audio = knownAudio.find((s) => s.disposition?.default) ?? knownAudio[0];+ if (audioTracks.length && !audio) throw new Error("This file has no supported audio track for conversion.");+ if (video && (!video.codec_name || video.codec_name === "unknown")) throw new Error("This file has no supported video track for conversion.");+ return {+ video: video?.codec_name,+ audio: audio?.codec_name,+ videoIndex: video?.index,+ videoNeedsEncoding: !!video && (video.codec_name !== "h264" || (!!video.pix_fmt && video.pix_fmt !== "yuv420p" && video.pix_fmt !== "yuvj420p")),+ audioIndex: audio?.index,+ remux: audioTracks.length > 1,+ };+}++/** Probe stream indexes as well as codecs so conversion maps the selected rendition. */+export function streamCodecs(file: string): Promise<ReturnType<typeof selectConversionStreams>> {+ return new Promise((resolve, reject) => {const p = spawn("ffprobe", [- "-v", "error",- "-show_entries", "stream=codec_type,codec_name",- "-of", "json",- file,+ "-v", "error", "-show_entries",+ "stream=index,codec_type,codec_name,codec_tag_string,pix_fmt:stream_disposition=default,attached_pic",+ "-of", "json", file,]);let out = "";const timer = setTimeout(() => {p.kill("SIGKILL");- resolve({});+ reject(new Error("Media inspection timed out."));}, 30_000);timer.unref();p.stdout.on("data", (d) => (out += d));p.on("close", (code) => {clearTimeout(timer);- if (code !== 0) return resolve({});+ if (code !== 0) return reject(new Error("Could not inspect this media file."));try {- const streams = (JSON.parse(out).streams ?? []) as {- codec_type?: string;- codec_name?: string;- }[];- resolve({- video: streams.find((s) => s.codec_type === "video")?.codec_name,- audio: streams.find((s) => s.codec_type === "audio")?.codec_name,- });- } catch {- resolve({});- }- });- p.on("error", () => {- clearTimeout(timer);- resolve({});+ resolve(selectConversionStreams((JSON.parse(out).streams ?? []) as MediaStream[]));+ } catch (error) { reject(error); }});+ p.on("error", (error) => { clearTimeout(timer); reject(error); });});}@@ -112,7 +126,7 @@ export async function convertToMp4(const shrink = !!(cap && dims && dims.height > cap);const color = codecs.video ? await videoColorInfo(src) : null;const sdr = sdrConvert(color, "yuv420p");- const transcodedVideo = !!codecs.video && (!KEEPS_VIDEO.has(codecs.video) || shrink || !!sdr);+ const transcodedVideo = !!codecs.video && (codecs.videoNeedsEncoding || shrink || !!sdr);const transcodedAudio = !!codecs.audio && !KEEPS_AUDIO.has(codecs.audio);if (fitsAlready(src, codecs, { shrink, sdr: !!sdr })) {return {@@ -125,7 +139,7 @@ export async function convertToMp4(const video: string[] = [];if (codecs.video) {- video.push("-map", "0:v:0");+ video.push("-map", `0:${codecs.videoIndex}`);if (!transcodedVideo) {video.push("-c:v", "copy");} else {@@ -146,7 +160,7 @@ export async function convertToMp4(const audio: string[] = [];if (codecs.audio) {- audio.push("-map", "0:a:0");+ audio.push("-map", `0:${codecs.audioIndex}`);audio.push(...(transcodedAudio ? ["-c:a", "aac", "-b:a", "192k"] : ["-c:a", "copy"]));}diff --git a/site/src/cut/server/http/convert.ts b/site/src/cut/server/http/convert.tsindex dd75ee01..a08016f1 100644--- a/site/src/cut/server/http/convert.ts+++ b/site/src/cut/server/http/convert.ts@@ -24,10 +24,10 @@ export const convertApi = {const out = path.join(tmp, mp4NameFor(path.basename(file.name) || "media"));await writeFile(src, new Uint8Array(await file.arrayBuffer()));const handle = { tmpDir: tmp, outPath: out, progress: 0, log: [] as string[] };- await convertToMp4(handle, src, out, {+ const outcome = await convertToMp4(handle, src, out, {maxHeight: Number.isFinite(maxHeight) && maxHeight > 0 ? maxHeight : undefined,});- const bytes = await readFile(out);+ const bytes = await readFile(outcome.unchanged ? src : out);return new Response(new Uint8Array(bytes), {headers: { "Content-Type": "video/mp4", "Content-Length": String(bytes.length) },});