Skip to content

Open source CapCut alternative.

TypeScript

fix(cut): build compat proxies where an encoder can [rebuild]

David Lebd5494d93a16
diff --git a/site/src/cut/lib/mediaProxy.test.ts b/site/src/cut/lib/mediaProxy.test.ts
index d3899ef3..7bf93c15 100644
--- a/site/src/cut/lib/mediaProxy.test.ts
+++ b/site/src/cut/lib/mediaProxy.test.ts
@@ -300,6 +300,26 @@ describe("compat proxy", () => {
expect(useEditor.getState().assets[0].proxy?.fileName).toBe("a.mp4.compat.webm");
});
+ test("a Mac project's picture cover is built by the worker and stored on the Mac", async () => {
+ // The Mac's ffmpeg carries no VP9 encoder, so only a sound cover is the engine's.
+ compatNeeds = { a: "video" };
+ features = new Set(["media.proxy", "media.compat"]);
+ const asset = h264("a", { url: "http://127.0.0.1:41417/file?p=a.mp4" });
+ useEditor.setState({ projectId: "p", assets: [asset] });
+ const b = backend("local", () => Response.json({}));
+ const realFetch = globalThis.fetch;
+ globalThis.fetch = (async () => new Response("master")) as unknown as typeof fetch;
+ try {
+ await ensureProxy("p", asset, b.api);
+ } finally {
+ globalThis.fetch = realFetch;
+ }
+ expect(b.calls).toHaveLength(0);
+ expect(staged).toEqual([{ name: "a.mp4", cover: "video" }]);
+ expect(stored).toEqual(["a.mp4.compat.webm"]);
+ expect(useEditor.getState().assets[0].proxy?.fileName).toBe("a.mp4.compat.webm");
+ });
+
test("a cover already on the asset is kept; a VP9 cover carries the sound too", async () => {
compatNeeds = { a: "audio" };
const b = backend("cloud", () => Response.json({}));
diff --git a/site/src/cut/lib/mediaProxy.ts b/site/src/cut/lib/mediaProxy.ts
index b02f2bb1..e57e7dd3 100644
--- a/site/src/cut/lib/mediaProxy.ts
+++ b/site/src/cut/lib/mediaProxy.ts
@@ -320,9 +320,10 @@ async function requestProxy(
return { fileName: result.fileName, sizeBytes: result.sizeBytes };
}
-/** Build the compat proxy through the residency that can. A browser project
- * hands the master to the Mac for a sound cover, and otherwise to the worker
- * by way of the cloud shelf: the Mac's ffmpeg carries no VP9 encoder. */
+/** Build the compat proxy through the residency that can. The Mac's ffmpeg
+ * carries no VP9 encoder, so it builds only a sound cover; a picture cover for
+ * a browser or Mac project goes to the worker by way of the cloud shelf and is
+ * stored back in the project. */
async function buildCompat(
projectId: string,
asset: MediaAsset,
@@ -331,7 +332,7 @@ async function buildCompat(
onProgress: (p: number) => void
): Promise<Landed | null> {
const { proxyMaxHeight, proxyCrf } = cutColor();
- if (backend.kind === "local") {
+ if (backend.kind === "local" && ask.cover === "audio") {
if (!(await engineFeatures()).has("media.compat")) return null;
return requestProxy(projectId, backend, { file: asset.fileName, maxHeight: proxyMaxHeight, crf: proxyCrf, compat: ask.cover }, onProgress);
}
diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.ts
index ac1d9671..ad7765a5 100644
--- a/site/src/cut/server/exportPipeline.ts
+++ b/site/src/cut/server/exportPipeline.ts
@@ -464,6 +464,12 @@ function ffmpegEncoders(): Promise<Set<string>> {
}));
}
+/** Whether this ffmpeg writes VP9: the worker's build does, the Mac's LGPL
+ * build does not. */
+export async function hasVp9Encoder(): Promise<boolean> {
+ return (await ffmpegEncoders()).has("libvpx-vp9");
+}
+
export async function h264Encoder(): Promise<"libx264" | "h264_videotoolbox"> {
return (await ffmpegEncoders()).has("libx264") ? "libx264" : "h264_videotoolbox";
}
diff --git a/site/src/cut/server/proxy.test.ts b/site/src/cut/server/proxy.test.ts
index e8920ef8..374bc651 100644
--- a/site/src/cut/server/proxy.test.ts
+++ b/site/src/cut/server/proxy.test.ts
@@ -189,6 +189,16 @@ describe("compat proxy", () => {
expect(packets(out)).toBe(packets(src));
}, 60_000);
+ test("a Rec.709 picture MP4 can't hold is encoded when only the sound needs Opus", async () => {
+ const src = path.join(dir, "master.mov");
+ ffmpeg(["-y", "-f", "lavfi", "-i", "testsrc=size=320x240:rate=24:duration=1", "-f", "lavfi", "-i", "sine=duration=1",
+ "-c:v", "prores_ks", "-colorspace", "bt709", "-color_range", "tv", "-c:a", "aac", "-shortest", src]);
+ const out = path.join(dir, "master.mov.compat.mp4");
+ await makeCompatProxy(handle(out), src, out, { cover: "audio", maxHeight: 1080, crf: 20 });
+
+ expect(codecsOf(out)).toEqual(["h264,video", "opus,audio"]);
+ }, 60_000);
+
test("a picture outside Rec.709 limited is carried into it", async () => {
const src = phoneClip("fullrange.mp4", ["-colorspace", "smpte170m", "-color_range", "pc"]);
const out = path.join(dir, "fullrange.mp4.compat.mp4");
diff --git a/site/src/cut/server/proxy.ts b/site/src/cut/server/proxy.ts
index 4c89ba4e..de9685fa 100644
--- a/site/src/cut/server/proxy.ts
+++ b/site/src/cut/server/proxy.ts
@@ -1,6 +1,6 @@
import { spawn } from "node:child_process";
import { stat } from "node:fs/promises";
-import { runFfmpeg, videoEncoder, vtQuality, type RenderHandle } from "./exportPipeline";
+import { hasVp9Encoder, runFfmpeg, videoEncoder, vtQuality, type RenderHandle } from "./exportPipeline";
import { assertGraphSafe } from "./filterGraph";
import { streamCodecs } from "./convert";
import { videoDimensions } from "./util";
@@ -206,6 +206,10 @@ export function vp9Crf(crf: number) {
/** The compat proxy's sound: Opus, which every browser decodes. */
const OPUS_KBPS = 128;
+/** Picture codecs an MP4 holds as they are, so a sound-only cover can copy
+ * them. Anything else (ProRes, DNxHD) is encoded to H.264. */
+const MP4_PICTURES = new Set(["h264", "hevc", "av1", "vp9", "mpeg4"]);
+
/** A keyframe every second of the master's time, counted from the last
* forced one, as the HEVC proxy places them. */
const KEY_EVERY_SECOND = "expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+1))";
@@ -227,7 +231,7 @@ export interface CompatArgs {
export function compatFfmpegArgs(
src: string,
out: string,
- opts: CompatArgs & { source: ProxySource | null; hasAudio: boolean; h264: string }
+ opts: CompatArgs & { source: ProxySource | null; videoCodec: string | null; hasAudio: boolean; h264: string }
): string[] {
const { source } = opts;
const cap = Math.max(2, Math.round(opts.maxHeight));
@@ -240,8 +244,10 @@ export function compatFfmpegArgs(
// A sound-only cover keeps the master's size unless it is a light one.
const shrink = source.height > cap && (opts.cover === "video" || !!kbps);
// A Rec.709 limited picture already is what a proxy holds, so a
- // sound-only cover copies it packet for packet.
- const copy = opts.cover === "audio" && source.matrix === "bt709" && !source.fullRange && !shrink && !kbps;
+ // sound-only cover copies it packet for packet when the MP4 can hold it.
+ const copy =
+ opts.cover === "audio" && source.matrix === "bt709" && !source.fullRange && !shrink && !kbps &&
+ MP4_PICTURES.has(opts.videoCodec ?? "");
if (copy) {
video.push("-c:v", "copy");
} else {
@@ -289,13 +295,14 @@ export async function makeCompatProxy(
): Promise<ProxyOutcome> {
const codecs = await streamCodecs(src);
if (opts.cover === "video" && !codecs.video) throw new Error("That file carries no video.");
+ if (opts.cover === "video" && !(await hasVp9Encoder())) throw new Error("This machine's ffmpeg has no VP9 encoder.");
const [source, dims, h264] = await Promise.all([
codecs.video ? probeProxySource(src) : Promise.resolve(null),
codecs.video ? videoDimensions(src) : Promise.resolve(null),
videoEncoder("h264"),
]);
if (source && dims) source.height = dims.height;
- await runFfmpeg(handle, compatFfmpegArgs(src, out, { ...opts, source, hasAudio: !!codecs.audio, h264 }), opts.onProgress);
+ await runFfmpeg(handle, compatFfmpegArgs(src, out, { ...opts, source, videoCodec: codecs.video ?? null, hasAudio: !!codecs.audio, h264 }), opts.onProgress);
const [info, outDims] = await Promise.all([stat(out), source ? videoDimensions(out) : Promise.resolve(null)]);
return { sizeBytes: info.size, ...(outDims ? { width: outDims.width, height: outDims.height } : {}) };
}