Open source CapCut alternative.
TypeScript
fix(cut): build compat proxies where an encoder can [rebuild]
David Lebd5494d93a16
diff --git a/site/src/cut/lib/mediaProxy.test.ts b/site/src/cut/lib/mediaProxy.test.tsindex d3899ef3..7bf93c15 100644--- a/site/src/cut/lib/mediaProxy.test.ts+++ b/site/src/cut/lib/mediaProxy.test.ts@@ -300,6 +300,26 @@ describe("compat proxy", () => {expect(useEditor.getState().assets[0].proxy?.fileName).toBe("a.mp4.compat.webm");});+ test("a Mac project's picture cover is built by the worker and stored on the Mac", async () => {+ // The Mac's ffmpeg carries no VP9 encoder, so only a sound cover is the engine's.+ compatNeeds = { a: "video" };+ features = new Set(["media.proxy", "media.compat"]);+ const asset = h264("a", { url: "http://127.0.0.1:41417/file?p=a.mp4" });+ useEditor.setState({ projectId: "p", assets: [asset] });+ const b = backend("local", () => Response.json({}));+ const realFetch = globalThis.fetch;+ globalThis.fetch = (async () => new Response("master")) as unknown as typeof fetch;+ try {+ await ensureProxy("p", asset, b.api);+ } finally {+ globalThis.fetch = realFetch;+ }+ expect(b.calls).toHaveLength(0);+ expect(staged).toEqual([{ name: "a.mp4", cover: "video" }]);+ expect(stored).toEqual(["a.mp4.compat.webm"]);+ expect(useEditor.getState().assets[0].proxy?.fileName).toBe("a.mp4.compat.webm");+ });+test("a cover already on the asset is kept; a VP9 cover carries the sound too", async () => {compatNeeds = { a: "audio" };const b = backend("cloud", () => Response.json({}));diff --git a/site/src/cut/lib/mediaProxy.ts b/site/src/cut/lib/mediaProxy.tsindex b02f2bb1..e57e7dd3 100644--- a/site/src/cut/lib/mediaProxy.ts+++ b/site/src/cut/lib/mediaProxy.ts@@ -320,9 +320,10 @@ async function requestProxy(return { fileName: result.fileName, sizeBytes: result.sizeBytes };}-/** Build the compat proxy through the residency that can. A browser project- * hands the master to the Mac for a sound cover, and otherwise to the worker- * by way of the cloud shelf: the Mac's ffmpeg carries no VP9 encoder. */+/** Build the compat proxy through the residency that can. The Mac's ffmpeg+ * carries no VP9 encoder, so it builds only a sound cover; a picture cover for+ * a browser or Mac project goes to the worker by way of the cloud shelf and is+ * stored back in the project. */async function buildCompat(projectId: string,asset: MediaAsset,@@ -331,7 +332,7 @@ async function buildCompat(onProgress: (p: number) => void): Promise<Landed | null> {const { proxyMaxHeight, proxyCrf } = cutColor();- if (backend.kind === "local") {+ if (backend.kind === "local" && ask.cover === "audio") {if (!(await engineFeatures()).has("media.compat")) return null;return requestProxy(projectId, backend, { file: asset.fileName, maxHeight: proxyMaxHeight, crf: proxyCrf, compat: ask.cover }, onProgress);}diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.tsindex ac1d9671..ad7765a5 100644--- a/site/src/cut/server/exportPipeline.ts+++ b/site/src/cut/server/exportPipeline.ts@@ -464,6 +464,12 @@ function ffmpegEncoders(): Promise<Set<string>> {}));}+/** Whether this ffmpeg writes VP9: the worker's build does, the Mac's LGPL+ * build does not. */+export async function hasVp9Encoder(): Promise<boolean> {+ return (await ffmpegEncoders()).has("libvpx-vp9");+}+export async function h264Encoder(): Promise<"libx264" | "h264_videotoolbox"> {return (await ffmpegEncoders()).has("libx264") ? "libx264" : "h264_videotoolbox";}diff --git a/site/src/cut/server/proxy.test.ts b/site/src/cut/server/proxy.test.tsindex e8920ef8..374bc651 100644--- a/site/src/cut/server/proxy.test.ts+++ b/site/src/cut/server/proxy.test.ts@@ -189,6 +189,16 @@ describe("compat proxy", () => {expect(packets(out)).toBe(packets(src));}, 60_000);+ test("a Rec.709 picture MP4 can't hold is encoded when only the sound needs Opus", async () => {+ const src = path.join(dir, "master.mov");+ ffmpeg(["-y", "-f", "lavfi", "-i", "testsrc=size=320x240:rate=24:duration=1", "-f", "lavfi", "-i", "sine=duration=1",+ "-c:v", "prores_ks", "-colorspace", "bt709", "-color_range", "tv", "-c:a", "aac", "-shortest", src]);+ const out = path.join(dir, "master.mov.compat.mp4");+ await makeCompatProxy(handle(out), src, out, { cover: "audio", maxHeight: 1080, crf: 20 });++ expect(codecsOf(out)).toEqual(["h264,video", "opus,audio"]);+ }, 60_000);+test("a picture outside Rec.709 limited is carried into it", async () => {const src = phoneClip("fullrange.mp4", ["-colorspace", "smpte170m", "-color_range", "pc"]);const out = path.join(dir, "fullrange.mp4.compat.mp4");diff --git a/site/src/cut/server/proxy.ts b/site/src/cut/server/proxy.tsindex 4c89ba4e..de9685fa 100644--- a/site/src/cut/server/proxy.ts+++ b/site/src/cut/server/proxy.ts@@ -1,6 +1,6 @@import { spawn } from "node:child_process";import { stat } from "node:fs/promises";-import { runFfmpeg, videoEncoder, vtQuality, type RenderHandle } from "./exportPipeline";+import { hasVp9Encoder, runFfmpeg, videoEncoder, vtQuality, type RenderHandle } from "./exportPipeline";import { assertGraphSafe } from "./filterGraph";import { streamCodecs } from "./convert";import { videoDimensions } from "./util";@@ -206,6 +206,10 @@ export function vp9Crf(crf: number) {/** The compat proxy's sound: Opus, which every browser decodes. */const OPUS_KBPS = 128;+/** Picture codecs an MP4 holds as they are, so a sound-only cover can copy+ * them. Anything else (ProRes, DNxHD) is encoded to H.264. */+const MP4_PICTURES = new Set(["h264", "hevc", "av1", "vp9", "mpeg4"]);+/** A keyframe every second of the master's time, counted from the last* forced one, as the HEVC proxy places them. */const KEY_EVERY_SECOND = "expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+1))";@@ -227,7 +231,7 @@ export interface CompatArgs {export function compatFfmpegArgs(src: string,out: string,- opts: CompatArgs & { source: ProxySource | null; hasAudio: boolean; h264: string }+ opts: CompatArgs & { source: ProxySource | null; videoCodec: string | null; hasAudio: boolean; h264: string }): string[] {const { source } = opts;const cap = Math.max(2, Math.round(opts.maxHeight));@@ -240,8 +244,10 @@ export function compatFfmpegArgs(// A sound-only cover keeps the master's size unless it is a light one.const shrink = source.height > cap && (opts.cover === "video" || !!kbps);// A Rec.709 limited picture already is what a proxy holds, so a- // sound-only cover copies it packet for packet.- const copy = opts.cover === "audio" && source.matrix === "bt709" && !source.fullRange && !shrink && !kbps;+ // sound-only cover copies it packet for packet when the MP4 can hold it.+ const copy =+ opts.cover === "audio" && source.matrix === "bt709" && !source.fullRange && !shrink && !kbps &&+ MP4_PICTURES.has(opts.videoCodec ?? "");if (copy) {video.push("-c:v", "copy");} else {@@ -289,13 +295,14 @@ export async function makeCompatProxy(): Promise<ProxyOutcome> {const codecs = await streamCodecs(src);if (opts.cover === "video" && !codecs.video) throw new Error("That file carries no video.");+ if (opts.cover === "video" && !(await hasVp9Encoder())) throw new Error("This machine's ffmpeg has no VP9 encoder.");const [source, dims, h264] = await Promise.all([codecs.video ? probeProxySource(src) : Promise.resolve(null),codecs.video ? videoDimensions(src) : Promise.resolve(null),videoEncoder("h264"),]);if (source && dims) source.height = dims.height;- await runFfmpeg(handle, compatFfmpegArgs(src, out, { ...opts, source, hasAudio: !!codecs.audio, h264 }), opts.onProgress);+ await runFfmpeg(handle, compatFfmpegArgs(src, out, { ...opts, source, videoCodec: codecs.video ?? null, hasAudio: !!codecs.audio, h264 }), opts.onProgress);const [info, outDims] = await Promise.all([stat(out), source ? videoDimensions(out) : Promise.resolve(null)]);return { sizeBytes: info.size, ...(outDims ? { width: outDims.width, height: outDims.height } : {}) };}