Skip to content

Open source CapCut alternative.

TypeScript

fix(cut): select playable media streams during conversion [rebuild]

David Le3af045b3a7dc
diff --git a/site/src/cut/lib/mediaRead.ts b/site/src/cut/lib/mediaRead.ts
index a3f0f7bb..af753888 100644
--- a/site/src/cut/lib/mediaRead.ts
+++ b/site/src/cut/lib/mediaRead.ts
@@ -68,9 +68,7 @@ export interface FrameSize {
}
export class UnreadableMediaError extends Error {
- /** The container parsed and the file carries video — this browser just has
- * no decoder for it. Callers that can convert branch on this rather than on
- * the message. */
+ /** The container parsed, but a media stream needs a decoder this runtime lacks. */
readonly undecodable: boolean;
constructor(message = "Cut can't read this media file.", undecodable = false) {
@@ -362,9 +360,9 @@ export async function hasUndecodableVideo(input: Input): Promise<boolean> {
/** The primary audio track, or null — same decodability rule as video. */
export async function audioTrackOf(input: Input): Promise<InputAudioTrack | null> {
- const track = await tracked(input.getPrimaryAudioTrack());
- if (!track) return null;
- return headless() || (await track.canDecode()) ? track : null;
+ return tracked(input.getPrimaryAudioTrack({
+ filter: (track) => headless() ? track.codec !== null : track.canDecode(),
+ }));
}
/** Read what a file is: how long, whether it carries picture or sound, and at
@@ -384,6 +382,9 @@ export async function probeMediaFile(src: string | Blob): Promise<MediaProbe> {
true
);
}
+ if (!audio && (await tracked(input.getAudioTracks())).length) {
+ throw new UnreadableMediaError("This file's audio needs conversion.", true);
+ }
if (!video && !audio) throw new UnreadableMediaError();
const [duration, bytes] = await Promise.all([input.computeDuration(), input.source.getSizeOrNull()]);
const size = bytes === null ? {} : { sizeBytes: bytes };
diff --git a/site/src/cut/server/ai/headlessTools.ts b/site/src/cut/server/ai/headlessTools.ts
index fa3556f2..dfb18e21 100644
--- a/site/src/cut/server/ai/headlessTools.ts
+++ b/site/src/cut/server/ai/headlessTools.ts
@@ -80,9 +80,10 @@ setHostMediaStore({
const handle = { tmpDir: "", outPath, progress: 0, log: [] as string[] };
const outcome = await convertToMp4(handle, src, outPath, { maxHeight: opts.maxHeight });
return {
- fileName,
+ fileName: outcome.unchanged ? source.fileName : fileName,
+ ...(outcome.unchanged ? { unchanged: true } : {}),
reencoded: outcome.transcodedVideo || outcome.transcodedAudio,
- duration: await probeDuration(outPath),
+ duration: await probeDuration(outcome.unchanged ? src : outPath),
...(outcome.width !== undefined ? { width: outcome.width, height: outcome.height } : {}),
};
},
diff --git a/site/src/cut/server/convert.test.ts b/site/src/cut/server/convert.test.ts
new file mode 100644
index 00000000..020c1b07
--- /dev/null
+++ b/site/src/cut/server/convert.test.ts
@@ -0,0 +1,74 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { execFileSync } from "node:child_process";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { convertToMp4, selectConversionStreams, streamCodecs } from "./convert";
+import { audioTrackOf, openMedia } from "@/cut/lib/mediaRead";
+
+let dir: string;
+beforeAll(async () => { dir = await mkdtemp(path.join(os.tmpdir(), "cut-convert-test-")); });
+afterAll(async () => { await rm(dir, { recursive: true, force: true }); });
+const ffmpeg = (args: string[]) => execFileSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", ...args], { timeout: 30_000 });
+const handle = (out: string) => ({ tmpDir: dir, outPath: out, progress: 0, log: [] as string[] });
+
+describe("import conversion", () => {
+ test("selects stereo AAC when an unsupported spatial track comes first", () => {
+ expect(selectConversionStreams([
+ { index: 0, codec_type: "video", codec_name: "hevc" },
+ { index: 1, codec_type: "audio", codec_tag_string: "apac", disposition: { default: 1 } },
+ { index: 2, codec_type: "audio", codec_name: "aac", codec_tag_string: "mp4a" },
+ ])).toMatchObject({ video: "hevc", audio: "aac", audioIndex: 2, remux: true });
+ });
+
+ test("a file with only unsupported sound fails explicitly", () => {
+ expect(() => selectConversionStreams([
+ { index: 0, codec_type: "video", codec_name: "hevc" },
+ { index: 1, codec_type: "audio", codec_tag_string: "apac" },
+ ])).toThrow("no supported audio track");
+ });
+
+ test("converts HEVC video and PCM audio to playable H.264/AAC", async () => {
+ const src = path.join(dir, "phone.mov");
+ const out = path.join(dir, "phone.mp4");
+ ffmpeg(["-f", "lavfi", "-i", "color=c=red:s=64x64:r=10:d=0.3", "-f", "lavfi", "-i", "sine=frequency=440:duration=0.3",
+ "-c:v", "libx265", "-x265-params", "log-level=error:pools=1", "-pix_fmt", "yuv420p10le", "-tag:v", "hvc1", "-c:a", "pcm_s16le", "-shortest", src]);
+ const before = await readFile(src);
+ expect(await convertToMp4(handle(out), src, out)).toMatchObject({ transcodedVideo: true, transcodedAudio: true, width: 64, height: 64 });
+ expect(await streamCodecs(out)).toMatchObject({ video: "h264", audio: "aac", videoNeedsEncoding: false });
+ expect(await readFile(src)).toEqual(before);
+ }, 30_000);
+
+ test("remuxes a MOV with an unknown first audio track using its second AAC track", async () => {
+ const src = path.join(dir, "spatial.mov");
+ const out = path.join(dir, "stereo.mp4");
+ ffmpeg(["-f", "lavfi", "-i", "color=s=64x64:r=10:d=0.3", "-f", "lavfi", "-i", "sine=duration=0.3",
+ "-map", "0:v", "-map", "1:a", "-map", "1:a", "-c:v", "libx264", "-c:a", "aac", "-shortest", src]);
+ const bytes = await readFile(src);
+ const audioEntry = bytes.indexOf(Buffer.from("mp4a"));
+ expect(audioEntry).toBeGreaterThan(0);
+ bytes.write("apac", audioEntry, "ascii");
+ await writeFile(src, bytes);
+ const input = openMedia(new Blob([bytes]));
+ try {
+ expect((await audioTrackOf(input))?.codec).toBe("aac");
+ } finally { input.dispose(); }
+ expect(await streamCodecs(src)).toMatchObject({ audio: "aac", audioIndex: 2, remux: true });
+ expect(await convertToMp4(handle(out), src, out)).toMatchObject({ transcodedVideo: false, transcodedAudio: false });
+ expect(await streamCodecs(out)).toMatchObject({ audio: "aac", remux: false });
+ }, 30_000);
+
+ test("already playable MP4 stays unchanged", async () => {
+ const src = path.join(dir, "ready.mp4");
+ ffmpeg(["-f", "lavfi", "-i", "color=s=64x64:r=10:d=0.2", "-c:v", "libx264", src]);
+ expect(await convertToMp4(handle(path.join(dir, "unused.mp4")), src, path.join(dir, "unused.mp4"))).toMatchObject({ unchanged: true });
+ });
+
+ test("audio-only phone media becomes playable AAC without a video stream", async () => {
+ const src = path.join(dir, "recording.mov");
+ const out = path.join(dir, "recording.m4a");
+ ffmpeg(["-f", "lavfi", "-i", "sine=duration=0.2", "-c:a", "pcm_s16le", src]);
+ expect(await convertToMp4(handle(out), src, out)).toMatchObject({ transcodedVideo: false, transcodedAudio: true });
+ expect(await streamCodecs(out)).toMatchObject({ video: undefined, audio: "aac" });
+ });
+});
diff --git a/site/src/cut/server/convert.ts b/site/src/cut/server/convert.ts
index 938d70e7..83699d15 100644
--- a/site/src/cut/server/convert.ts
+++ b/site/src/cut/server/convert.ts
@@ -44,53 +44,67 @@ const MP4_EXT = /\.(mp4|m4v)$/i;
* to leave it alone. */
export const fitsAlready = (
file: string,
- codecs: { video?: string; audio?: string },
+ codecs: { video?: string; audio?: string; remux?: boolean; videoNeedsEncoding?: boolean },
work: { shrink: boolean; sdr: boolean }
) =>
MP4_EXT.test(file) &&
!work.shrink &&
!work.sdr &&
+ !codecs.remux &&
+ !codecs.videoNeedsEncoding &&
(!codecs.video || KEEPS_VIDEO.has(codecs.video)) &&
(!codecs.audio || KEEPS_AUDIO.has(codecs.audio));
-/** The first video and audio stream's codec names, or undefined for a file
- * that carries no such stream (or a probe that fails — the conversion then
- * takes the re-encode path, which is correct for anything). */
-export function streamCodecs(file: string): Promise<{ video?: string; audio?: string }> {
- return new Promise((resolve) => {
+export type MediaStream = {
+ index: number;
+ codec_type?: string;
+ codec_name?: string;
+ codec_tag_string?: string;
+ pix_fmt?: string;
+ disposition?: { default?: number; attached_pic?: number };
+};
+
+/** Choose the playable audio rendition carried beside optional spatial tracks. */
+export function selectConversionStreams(streams: MediaStream[]) {
+ const video = streams.find((s) => s.codec_type === "video" && !s.disposition?.attached_pic);
+ const audioTracks = streams.filter((s) => s.codec_type === "audio");
+ const knownAudio = audioTracks.filter((s) => s.codec_name && s.codec_name !== "unknown" && s.codec_tag_string !== "apac");
+ const audio = knownAudio.find((s) => s.disposition?.default) ?? knownAudio[0];
+ if (audioTracks.length && !audio) throw new Error("This file has no supported audio track for conversion.");
+ if (video && (!video.codec_name || video.codec_name === "unknown")) throw new Error("This file has no supported video track for conversion.");
+ return {
+ video: video?.codec_name,
+ audio: audio?.codec_name,
+ videoIndex: video?.index,
+ videoNeedsEncoding: !!video && (video.codec_name !== "h264" || (!!video.pix_fmt && video.pix_fmt !== "yuv420p" && video.pix_fmt !== "yuvj420p")),
+ audioIndex: audio?.index,
+ remux: audioTracks.length > 1,
+ };
+}
+
+/** Probe stream indexes as well as codecs so conversion maps the selected rendition. */
+export function streamCodecs(file: string): Promise<ReturnType<typeof selectConversionStreams>> {
+ return new Promise((resolve, reject) => {
const p = spawn("ffprobe", [
- "-v", "error",
- "-show_entries", "stream=codec_type,codec_name",
- "-of", "json",
- file,
+ "-v", "error", "-show_entries",
+ "stream=index,codec_type,codec_name,codec_tag_string,pix_fmt:stream_disposition=default,attached_pic",
+ "-of", "json", file,
]);
let out = "";
const timer = setTimeout(() => {
p.kill("SIGKILL");
- resolve({});
+ reject(new Error("Media inspection timed out."));
}, 30_000);
timer.unref();
p.stdout.on("data", (d) => (out += d));
p.on("close", (code) => {
clearTimeout(timer);
- if (code !== 0) return resolve({});
+ if (code !== 0) return reject(new Error("Could not inspect this media file."));
try {
- const streams = (JSON.parse(out).streams ?? []) as {
- codec_type?: string;
- codec_name?: string;
- }[];
- resolve({
- video: streams.find((s) => s.codec_type === "video")?.codec_name,
- audio: streams.find((s) => s.codec_type === "audio")?.codec_name,
- });
- } catch {
- resolve({});
- }
- });
- p.on("error", () => {
- clearTimeout(timer);
- resolve({});
+ resolve(selectConversionStreams((JSON.parse(out).streams ?? []) as MediaStream[]));
+ } catch (error) { reject(error); }
});
+ p.on("error", (error) => { clearTimeout(timer); reject(error); });
});
}
@@ -112,7 +126,7 @@ export async function convertToMp4(
const shrink = !!(cap && dims && dims.height > cap);
const color = codecs.video ? await videoColorInfo(src) : null;
const sdr = sdrConvert(color, "yuv420p");
- const transcodedVideo = !!codecs.video && (!KEEPS_VIDEO.has(codecs.video) || shrink || !!sdr);
+ const transcodedVideo = !!codecs.video && (codecs.videoNeedsEncoding || shrink || !!sdr);
const transcodedAudio = !!codecs.audio && !KEEPS_AUDIO.has(codecs.audio);
if (fitsAlready(src, codecs, { shrink, sdr: !!sdr })) {
return {
@@ -125,7 +139,7 @@ export async function convertToMp4(
const video: string[] = [];
if (codecs.video) {
- video.push("-map", "0:v:0");
+ video.push("-map", `0:${codecs.videoIndex}`);
if (!transcodedVideo) {
video.push("-c:v", "copy");
} else {
@@ -146,7 +160,7 @@ export async function convertToMp4(
const audio: string[] = [];
if (codecs.audio) {
- audio.push("-map", "0:a:0");
+ audio.push("-map", `0:${codecs.audioIndex}`);
audio.push(...(transcodedAudio ? ["-c:a", "aac", "-b:a", "192k"] : ["-c:a", "copy"]));
}
diff --git a/site/src/cut/server/http/convert.ts b/site/src/cut/server/http/convert.ts
index dd75ee01..a08016f1 100644
--- a/site/src/cut/server/http/convert.ts
+++ b/site/src/cut/server/http/convert.ts
@@ -24,10 +24,10 @@ export const convertApi = {
const out = path.join(tmp, mp4NameFor(path.basename(file.name) || "media"));
await writeFile(src, new Uint8Array(await file.arrayBuffer()));
const handle = { tmpDir: tmp, outPath: out, progress: 0, log: [] as string[] };
- await convertToMp4(handle, src, out, {
+ const outcome = await convertToMp4(handle, src, out, {
maxHeight: Number.isFinite(maxHeight) && maxHeight > 0 ? maxHeight : undefined,
});
- const bytes = await readFile(out);
+ const bytes = await readFile(outcome.unchanged ? src : out);
return new Response(new Uint8Array(bytes), {
headers: { "Content-Type": "video/mp4", "Content-Length": String(bytes.length) },
});