Open source CapCut alternative.
TypeScript
feat(app): export a ProRes 4444 master at full chroma [rebuild]
David Le0e5b093bb509
diff --git a/docs/guides/cut/README.md b/docs/guides/cut/README.mdindex 33aaf5b2..b7b55741 100644--- a/docs/guides/cut/README.md+++ b/docs/guides/cut/README.md@@ -52,7 +52,7 @@ Guides draw over the preview from the button beside the timeline zoom: thirds, c## The export-An export is the preview drawn again frame by frame into an MP4 by the same compositor and mixer, so the file matches what the person saw. The dialog names the file, runs at the footage's own frame rate unless told otherwise, can deliver just the span the selection covers, offers a marked 4K upscale because the platforms transcode a 4K upload at a higher bitrate, and can save the captions as an SRT to the browser's Downloads folder; H.264 files carry the High profile and every H.264 or HEVC file keys every two seconds. One thing the export draws better: a clip with smooth slow motion on has the frames between its source frames estimated from the motion between them, by a frame interpolation network in the tab and by ffmpeg's own interpolation on the ffmpeg pipeline, wherever its rate runs under 1×; the preview blends the two frames instead, and so does an export from a browser without WebGPU, the one surface that cannot run the network. The finished file goes to the project's home: the app's project folder for a Mac project, storage for a cloud one, the page's store for a browser one. A browser that cannot carry the render hands it to the machine behind the project, and the ffmpeg pipeline stays for ProRes, headless renders, share cards, and streaming ladders. The local-compute guide covers where each kind of work runs.+An export is the preview drawn again frame by frame into an MP4 by the same compositor and mixer, so the file matches what the person saw. The dialog names the file, runs at the footage's own frame rate unless told otherwise, can deliver just the span the selection covers, offers a marked 4K upscale because the platforms transcode a 4K upload at a higher bitrate, and can save the captions as an SRT to the browser's Downloads folder. The Master preset is ProRes 4444, composited at 4:4:4 end to end, so titles and captions land in the file with the color the preview drew. One thing the export draws better: a clip in smooth slow motion has its in-between frames estimated from the motion around them, where the preview blends the neighbors; a browser without WebGPU blends too. The finished file goes to the project's home: the app's project folder for a Mac project, storage for a cloud one, the page's store for a browser one. A browser that cannot carry the render hands it to the machine behind the project, and the ffmpeg pipeline stays for ProRes, headless renders, share cards, and streaming ladders. The local-compute guide covers where each kind of work runs.## Sharingdiff --git a/site/packages/effects-kit/src/effects.ts b/site/packages/effects-kit/src/effects.tsindex 0cb63d36..a6e08ea6 100644--- a/site/packages/effects-kit/src/effects.ts+++ b/site/packages/effects-kit/src/effects.ts@@ -558,6 +558,16 @@ export function applyEffectToCanvas(* out of quoted expressions). */const gate = (a: number, b: number) => `enable='gte(t,${fmt(a)})*lt(t,${fmt(b)})'`;+/** The pixel family a chain works in: the format its branches carry, and the+ * option that keeps `overlay` blending in the same one. A 4:2:0 graph is the+ * default, which is what `overlay` picks on its own. */+export interface ChainChroma {+ pixFmt: string;+ overlay: string;+}++export const CHROMA_420: ChainChroma = { pixFmt: "yuv420p", overlay: "" };+/*** ffmpeg filter_complex lines rendering one effect from `[inLabel]` into* `[outLabel]`, active only inside [start, end). Every filter used is in the@@ -575,7 +585,8 @@ export function effectFilterLines(height: number,tag: string,focus?: { x: number; y: number },- ramp?: number+ ramp?: number,+ chroma: ChainChroma = CHROMA_420): string[] | null {const custom = customEffects.get(effect);if (custom) return custom.filterLines(inLabel, outLabel, amount, start, end, width, height, tag);@@ -590,12 +601,12 @@ export function effectFilterLines(// The graded copy renders on its own branch and replaces the frame only// inside the window, the same shape the shake recipe uses — a look chain// is several filters deep and not all of them take a timeline gate.- const lines = lookFilterLines(`lkfi${tag}`, `lkfo${tag}`, look, k, height, "yuv420p", tag);+ const lines = lookFilterLines(`lkfi${tag}`, `lkfo${tag}`, look, k, height, chroma.pixFmt, tag);if (!lines) return null;return [`[${inLabel}]split[lkfb${tag}][lkfi${tag}]`,...lines,- `[lkfb${tag}][lkfo${tag}]overlay=0:0:${en}:eof_action=pass[${outLabel}]`,+ `[lkfb${tag}][lkfo${tag}]overlay=0:0:${en}:eof_action=pass${chroma.overlay}[${outLabel}]`,];}switch (effect as VisualEffectId) {@@ -615,7 +626,7 @@ export function effectFilterLines(`[${inLabel}]split[efb${tag}][efs${tag}]`,`[efs${tag}]scale=${zw}:${zh},crop=${width}:${height}:` +`${Math.round((zw - width) * f.x)}:${Math.round((zh - height) * f.y)}[efc${tag}]`,- `[efb${tag}][efc${tag}]overlay=0:0:${en}:eof_action=pass[${outLabel}]`,+ `[efb${tag}][efc${tag}]overlay=0:0:${en}:eof_action=pass${chroma.overlay}[${outLabel}]`,];}// Ramped: the branch is rescaled every frame along the same eased curve@@ -632,7 +643,7 @@ export function effectFilterLines(`[${inLabel}]split[efb${tag}][efs${tag}]`,`[efs${tag}]scale=w='trunc(iw*${grow}/2)*2':h='trunc(ih*${grow}/2)*2':eval=frame[efc${tag}]`,`[efb${tag}][efc${tag}]overlay=x='-(w-W)*${fmt(f.x)}':y='-(h-H)*${fmt(f.y)}':` +- `${en}:eof_action=pass[${outLabel}]`,+ `${en}:eof_action=pass${chroma.overlay}[${outLabel}]`,];}case "grain":@@ -708,7 +719,7 @@ export function effectFilterLines(`[${inLabel}]split[efb${tag}][efs${tag}]`,`[efs${tag}]scale=${zw}:${zh},crop=${width}:${height}:` +`x='(in_w-out_w)/2+${ax}*sin(${tl}*33)':y='(in_h-out_h)/2+${ay}*cos(${tl}*47)'[efc${tag}]`,- `[efb${tag}][efc${tag}]overlay=0:0:${en}:eof_action=pass[${outLabel}]`,+ `[efb${tag}][efc${tag}]overlay=0:0:${en}:eof_action=pass${chroma.overlay}[${outLabel}]`,];}default:diff --git a/site/src/cut/lib/engineFeatures.ts b/site/src/cut/lib/engineFeatures.tsindex 31af00c3..df78a9bb 100644--- a/site/src/cut/lib/engineFeatures.ts+++ b/site/src/cut/lib/engineFeatures.ts@@ -4,6 +4,6 @@* hands an engine work an older app would silently drop. A build from before* the list existed answers with none.*/-export const ENGINE_FEATURES = ["export.range", "export.name"] as const;+export const ENGINE_FEATURES = ["export.range", "export.name", "export.prores4444"] as const;export type EngineFeature = (typeof ENGINE_FEATURES)[number];diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.tsindex 1a5f53d5..12de7a86 100644--- a/site/src/cut/lib/exportClient.ts+++ b/site/src/cut/lib/exportClient.ts@@ -40,17 +40,20 @@ import type {} from "./types";import {+ deliveryCodec,deliveryContainer,deliverySpan,+ EXPORT_CODECS,EXPORT_CONTAINERS,exportBaseName,+ fixedRate,specMediaFiles,type ExportAudioCodec,type ExportCodec,type ExportContainer,type ExportRange,} from "./exportDelivery";-export { EXPORT_CONTAINERS };+export { deliveryCodec, EXPORT_CODECS, EXPORT_CONTAINERS, fixedRate };export type { ExportAudioCodec, ExportCodec, ExportContainer, ExportRange };export interface ExportSettings {@@ -75,22 +78,15 @@ export interface ExportSettings {export { DELIVERY_DEFAULTS, EXPORT_PRESETS } from "./exportPresets";-export const EXPORT_CODECS = [- { id: "h264", label: "H.264", detail: "plays everywhere" },- { id: "hevc", label: "HEVC", detail: "half the size · newer devices" },- { id: "prores", label: "ProRes 422 HQ", detail: "edit master · MOV" },-] as const satisfies readonly { id: ExportCodec; label: string; detail: string }[];-export const EXPORT_AUDIO = [{ id: "aac", label: "AAC", detail: "192 kbps" },{ id: "pcm", label: "PCM", detail: "uncompressed · MOV" },] as const satisfies readonly { id: ExportAudioCodec; label: string; detail: string }[];-/** What a container can carry: ProRes and PCM need MOV. */+/** What a container can carry: the codec's own demand, and PCM's MOV. */export function fitContainer(settings: ExportSettings): ExportSettings {- if (settings.codec === "prores" || settings.audioCodec === "pcm") {- return settings.container === "mov" ? settings : { ...settings, container: "mov" };- }+ const needed = deliveryCodec(settings.codec).container ?? (settings.audioCodec === "pcm" ? "mov" : null);+ if (needed && settings.container !== needed) return { ...settings, container: needed };return settings;}@@ -172,9 +168,9 @@ export const EXPORT_QUICK_PRESETS = [{id: "master",label: "Master",- detail: "ProRes 422 HQ · MOV · PCM",+ detail: "ProRes 4444 · MOV · PCM · nothing given up",short: "ProRes",- choice: { resolution: "source", fps: "source", quality: "high", codec: "prores", container: "mov", audioCodec: "pcm" },+ choice: { resolution: "source", fps: "source", quality: "high", codec: "prores4444", container: "mov", audioCodec: "pcm" },},] as const satisfies readonly { id: string; label: string; detail: string; short: string; choice: ExportChoice }[];@@ -215,8 +211,9 @@ export function choiceSettings(codec: choice.codec,container: choice.container,audioCodec: choice.audioCodec,- // ProRes has one rate per profile; a typed bitrate is for the other codecs.- ...(choice.bitrateMbps && choice.codec !== "prores"+ // A fixed-rate codec has one rate per profile; a typed bitrate is for+ // the others.+ ...(choice.bitrateMbps && !fixedRate(choice.codec)? { bitrate: Math.round(choice.bitrateMbps * 1_000_000) }: {}),});@@ -1462,6 +1459,10 @@ async function assertEngineCarries(settings: ExportSettings): Promise<void> {const wants: [EngineFeature, boolean, string][] = [["export.range", !!settings.range, "export a range"],["export.name", !!settings.name, "name the file"],+ // An engine that does not know the codec falls through to H.264, which+ // would hand back a .mov of exactly the picture the master was chosen to+ // avoid.+ ["export.prores4444", settings.codec === "prores4444", "export a ProRes 4444 master"],];const has = await engineFeatures();for (const [feature, wanted, what] of wants) {diff --git a/site/src/cut/lib/exportDelivery.ts b/site/src/cut/lib/exportDelivery.tsindex 10342bef..b254584a 100644--- a/site/src/cut/lib/exportDelivery.ts+++ b/site/src/cut/lib/exportDelivery.ts@@ -4,7 +4,7 @@* cloud routes and the worker all read the one table.*/-export type ExportCodec = "h264" | "hevc" | "prores";+export type ExportCodec = "h264" | "hevc" | "prores" | "prores4444";export type ExportContainer = "mp4" | "mov";export type ExportAudioCodec = "aac" | "pcm";@@ -20,6 +20,55 @@ export const EXPORT_CONTAINERS: readonly DeliveryContainer[] = [{ id: "mov", label: "MOV", ext: ".mov", mime: "video/quicktime" },];+/**+ * A codec a delivered file can carry: what the menu calls it, the container+ * it needs, the chroma it holds, and how it is rated.+ *+ * `bpp` marks a codec with no rate control: ProRes encodes at one rate per+ * profile, so the quality tier and a typed bitrate never reach it and its+ * size is bits-per-pixel times the frame. The others are rated by tier.+ */+export interface DeliveryCodec {+ id: ExportCodec;+ label: string;+ detail: string;+ /** The only container this codec rides; absent = any of them. */+ container?: ExportContainer;+ /** Full-chroma delivery: 4:4:4, no subsampling. */+ chroma444?: true;+ /** Bits a pixel, fixed, for a codec with no rate control. */+ bpp?: number;+}++/**+ * The delivery codecs, in menu order. ProRes 422 HQ writes ~220 Mbit/s at+ * 1080p30 and ProRes 4444 ~330, which is where the fixed bits-per-pixel come+ * from.+ */+export const EXPORT_CODECS: readonly DeliveryCodec[] = [+ { id: "h264", label: "H.264", detail: "plays everywhere" },+ { id: "hevc", label: "HEVC", detail: "half the size · newer devices" },+ { id: "prores", label: "ProRes 422 HQ", detail: "edit master · MOV", container: "mov", bpp: 3.54 },+ {+ id: "prores4444",+ label: "ProRes 4444",+ detail: "every pixel's own color · MOV",+ container: "mov",+ chroma444: true,+ bpp: 5.31,+ },+];++export function deliveryCodec(id: ExportCodec | undefined): DeliveryCodec {+ return EXPORT_CODECS.find((c) => c.id === id) ?? EXPORT_CODECS[0];+}++/** Whether a codec sets its own rate, so the quality tier and a typed+ * bitrate have nothing to act on. */+export function fixedRate(id: ExportCodec | undefined): boolean {+ return deliveryCodec(id).bpp !== undefined;+}+/** A window of the timeline to deliver, seconds; absent = the whole cut. */export interface ExportRange {start: number;@@ -73,8 +122,8 @@ export function isDeliveryName(name: string): boolean {/*** The video bitrate a setting asks for, in bits per second: H.264* bits-per-pixel halving every +6 CRF from a ~0.08 bpp anchor at CRF 23, HEVC- * landing the same tier in about 60% of the bits, ProRes 422 HQ a fixed ~3.5- * bits a pixel (220 Mbit/s at 1080p30). A bitrate the user typed wins.+ * landing the same tier in about 60% of the bits, and a fixed-rate codec the+ * bits a pixel its table row names. A bitrate the user typed wins.** WebCodecs and VideoToolbox encode to a bitrate while the tiers are written* in CRF; this is the one model they share with the dialog's size estimate, so@@ -89,7 +138,8 @@ export function videoBitrateFor(s: {bitrate?: number;}): number {const pixelsPerSec = s.width * s.height * s.fps;- if (s.codec === "prores") return Math.round(pixelsPerSec * 3.54);+ const fixed = deliveryCodec(s.codec).bpp;+ if (fixed) return Math.round(pixelsPerSec * fixed);if (s.bitrate) return s.bitrate;const bpp = 0.08 * 2 ** ((23 - s.crf) / 6) * (s.codec === "hevc" ? 0.6 : 1);return Math.max(200_000, Math.round(pixelsPerSec * bpp));diff --git a/site/src/cut/lib/exportRender.test.ts b/site/src/cut/lib/exportRender.test.tsindex 56c573ef..de88a342 100644--- a/site/src/cut/lib/exportRender.test.ts+++ b/site/src/cut/lib/exportRender.test.ts@@ -157,10 +157,12 @@ describe("bitrateFor by delivery", () => {test("HEVC asks for fewer bits than H.264 at the same tier", () => {expect(bitrateFor({ ...base, codec: "hevc" })).toBeLessThan(bitrateFor({ ...base, codec: "h264" }));});- test("ProRes is a fixed rate, above any tier", () => {+ test("ProRes is a fixed rate, above any tier, and 4444 above 422 HQ", () => {expect(bitrateFor({ ...base, codec: "prores" })).toBeGreaterThan(bitrateFor({ ...base, codec: "h264", crf: 10 }));+ expect(bitrateFor({ ...base, codec: "prores4444" })).toBeGreaterThan(bitrateFor({ ...base, codec: "prores" }));});- test("a typed bitrate wins", () => {+ test("a typed bitrate wins, except where the codec sets its own", () => {expect(bitrateFor({ ...base, codec: "h264", bitrate: 1_234_000 })).toBe(1_234_000);+ expect(bitrateFor({ ...base, codec: "prores4444", bitrate: 1_234_000 })).not.toBe(1_234_000);});});diff --git a/site/src/cut/server/convert.ts b/site/src/cut/server/convert.tsindex d9042c76..938d70e7 100644--- a/site/src/cut/server/convert.ts+++ b/site/src/cut/server/convert.ts@@ -111,7 +111,7 @@ export async function convertToMp4(const cap = opts.maxHeight && opts.maxHeight > 0 ? Math.round(opts.maxHeight) : 0;const shrink = !!(cap && dims && dims.height > cap);const color = codecs.video ? await videoColorInfo(src) : null;- const sdr = sdrConvert(color);+ const sdr = sdrConvert(color, "yuv420p");const transcodedVideo = !!codecs.video && (!KEEPS_VIDEO.has(codecs.video) || shrink || !!sdr);const transcodedAudio = !!codecs.audio && !KEEPS_AUDIO.has(codecs.audio);if (fitsAlready(src, codecs, { shrink, sdr: !!sdr })) {diff --git a/site/src/cut/server/exportPipeline.e2e.test.ts b/site/src/cut/server/exportPipeline.e2e.test.tsindex 173e9194..b0033930 100644--- a/site/src/cut/server/exportPipeline.e2e.test.ts+++ b/site/src/cut/server/exportPipeline.e2e.test.ts@@ -11,9 +11,33 @@ import { runExport, type ExportSpec, type RenderHandle } from "./exportPipeline"// transitions, edge animations, looks, grades, masks (painted and subject),// keyframed poses, borders, shadows, overlay videos, elements (static and// slideshow), effects, captions, an audio bed with ducking, whole-video-// fades, a background color, a gap — and renders it end to end. A filter-// ffmpeg rejects, a broken option value, or an unconnected pad fails here-// before it can fail an export.+// fades, a background color, a gap, a clip played backward — and renders it+// end to end. A filter ffmpeg rejects, a broken option value, or an+// unconnected pad fails here before it can fail an export.+//+// It runs once per delivery, because the graph is built in the chroma the+// delivery carries: a ProRes 4444 master composites at 4:4:4, where every+// filter and every `overlay` negotiates a different pixel family than the+// 4:2:0 the H.264 file is built in.+const DELIVERIES = [+ { name: "an H.264 MP4", ext: "mp4", codec: "h264", pixFmt: "yuv420p", over: {} },+ {+ // ProRes 4444's bitstream carries twelve bits, so the file reads back as+ // yuv444p12le whatever the encoder was handed.+ name: "a ProRes 4444 master",+ ext: "mov",+ codec: "prores",+ pixFmt: "yuv444p12le",+ over: { codec: "prores4444", container: "mov", audioCodec: "pcm" },+ },+] as const satisfies readonly {+ name: string;+ ext: string;+ codec: string;+ /** What the finished file decodes as. */+ pixFmt: string;+ over: Partial<ExportSpec>;+}[];const available = (cmd: string) =>spawnSync(cmd, ["-version"], { stdio: "ignore" }).status === 0;@@ -32,7 +56,7 @@ const probe = (file: string) => {["-v", "error","-show_entries", "format=duration",- "-show_entries", "stream=codec_type,width,height",+ "-show_entries", "stream=codec_type,codec_name,pix_fmt,width,height","-of", "json",file,],@@ -41,16 +65,16 @@ const probe = (file: string) => {if (r.status !== 0) throw new Error(`ffprobe failed:\n${r.stderr}`);return JSON.parse(r.stdout) as {format: { duration: string };- streams: { codec_type: string; width?: number; height?: number }[];+ streams: { codec_type: string; codec_name: string; pix_fmt?: string; width?: number; height?: number }[];};};const W = 320;const H = 568;-describe("export pipeline end to end", () => {+for (const delivery of DELIVERIES) describe("export pipeline end to end", () => {test.skipIf(!tools)(- "a project using every pipeline feature renders through real ffmpeg",+ `a project using every pipeline feature renders through real ffmpeg as ${delivery.name}`,async () => {const mediaDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-media-"));const tmpDir = await mkdtemp(path.join(os.tmpdir(), "cut-e2e-job-"));@@ -116,7 +140,7 @@ describe("export pipeline end to end", () => {},{file: "b.mp4", in: 0, out: 2, muted: false,- fit: "fit",+ fit: "fit", reverse: true,grade: { brightness: 5, contrast: 8, saturation: -10, temperature: 12, hue: 10 },mask: { file: "mask.png" },kf: [@@ -174,11 +198,12 @@ describe("export pipeline end to end", () => {{ file: "cap.png", start: 0.3, end: 1.2 },{ file: "cap.png", start: 1.4, end: 2.0 },],+ ...delivery.over,};const job: RenderHandle = {tmpDir,- outPath: path.join(tmpDir, "out.mp4"),+ outPath: path.join(tmpDir, `out.${delivery.ext}`),progress: 0,log: [],};@@ -192,6 +217,10 @@ describe("export pipeline end to end", () => {const video = meta.streams.find((s) => s.codec_type === "video");expect(video?.width).toBe(W);expect(video?.height).toBe(H);+ expect(video?.codec_name).toBe(delivery.codec);+ // The chroma the composite was built in is the chroma the file holds:+ // 4:4:4 all the way through for the master, 4:2:0 for the MP4.+ expect(video?.pix_fmt).toBe(delivery.pixFmt);expect(meta.streams.some((s) => s.codec_type === "audio")).toBe(true);} finally {await rm(mediaDir, { recursive: true, force: true });diff --git a/site/src/cut/server/exportPipeline.test.ts b/site/src/cut/server/exportPipeline.test.tsindex b4c1a4e6..b51ea44b 100644--- a/site/src/cut/server/exportPipeline.test.ts+++ b/site/src/cut/server/exportPipeline.test.ts@@ -28,7 +28,8 @@ const runsFor = async (over: Partial<ExportSpec>): Promise<string[][]> => {videoColorInfo: async () => null,videoDecodeCost: async () => null,mediaDuration: async (file) => produced.get(file) ?? 0,- videoEncoder: async (codec) => (codec === "hevc" ? "libx265" : codec === "prores" ? "prores_ks" : "libx264"),+ videoEncoder: async (codec) =>+ codec === "hevc" ? "libx265" : codec.startsWith("prores") ? "prores_ks" : "libx264",runFfmpeg: async (_job, args) => {ffmpegCalls.push(args);const piece = args[args.length - 1].match(/^(.*)\.\d+\.(?:mov|wav)$/);@@ -873,6 +874,29 @@ describe("delivery", () => {expect(args[args.length - 1].endsWith("encode.mov")).toBe(true);});+ test("a ProRes 4444 master writes 10-bit 4:4:4 and composites at full chroma", async () => {+ const args = await encodeRun({ codec: "prores4444", container: "mov", audioCodec: "pcm" });+ expect(arg(args, "-c:v")).toBe("prores_ks");+ expect(arg(args, "-profile:v")).toBe("4");+ expect(arg(args, "-pix_fmt")).toBe("yuv444p10le");+ expect(args).not.toContain("-crf");+ const graph = arg(args, "-filter_complex");+ // The composite itself holds the chroma; a 4:2:0 graph would have thrown+ // it away before prores_ks ever saw a frame.+ expect(graph).toContain("format=yuv444p");+ expect(graph).not.toContain("yuv420p");+ expect(graph).not.toContain("yuva420p");+ for (const stanza of graph.split(";").filter((f) => f.includes("overlay="))) {+ expect(stanza).toContain(":format=yuv444");+ }+ });++ test("an H.264 delivery composites at 4:2:0, where its encoder lands anyway", async () => {+ const graph = arg(await encodeRun({}), "-filter_complex");+ expect(graph).toContain("format=yuv420p");+ expect(graph).not.toContain("yuv444");+ });+test("a typed bitrate replaces the tier's CRF", async () => {const args = await encodeRun({ bitrate: 6_000_000 });expect(args).not.toContain("-crf");diff --git a/site/src/cut/server/exportPipeline.ts b/site/src/cut/server/exportPipeline.tsindex 9fa85f08..808ad67e 100644--- a/site/src/cut/server/exportPipeline.ts+++ b/site/src/cut/server/exportPipeline.ts@@ -1,5 +1,5 @@import { spawn, type ChildProcess } from "node:child_process";-import { deliveryContainer, deliverySpan, KEYFRAME_INTERVAL_S, videoBitrateFor, type ExportRange } from "../lib/exportDelivery";+import { deliveryCodec, deliveryContainer, deliverySpan, KEYFRAME_INTERVAL_S, videoBitrateFor, type ExportCodec, type ExportRange } from "../lib/exportDelivery";import { readFile, stat, unlink, writeFile } from "node:fs/promises";import path from "node:path";import { atempoChain, audioChannels, hasStream, mediaDuration, num, videoColorInfo, videoDecodeCost } from "./util";@@ -64,7 +64,7 @@ export interface ExportSpec {crf: number;preset: string;/** The delivery. A spec from before these existed is an H.264 + AAC MP4. */- codec?: "h264" | "hevc" | "prores";+ codec?: ExportCodec;container?: "mp4" | "mov";audioCodec?: "aac" | "pcm";/** A bitrate the user typed, bits per second; absent = the `crf` tier. */@@ -376,7 +376,7 @@ export type ExportVideoCodec = NonNullable<ExportSpec["codec"]>;*/export async function videoEncoder(codec: ExportVideoCodec): Promise<string> {const have = await ffmpegEncoders();- if (codec === "prores") return "prores_ks";+ if (codec === "prores" || codec === "prores4444") return "prores_ks";if (codec === "hevc") return have.has("libx265") ? "libx265" : "hevc_videotoolbox";return have.has("libx264") ? "libx264" : "h264_videotoolbox";}@@ -429,8 +429,15 @@ export function videoCodecArgs(enc: string, spec: ExportSpec): string[] {return ["-c:v", "libx265", "-preset", spec.preset, ...(spec.bitrate ? rate : ["-crf", String(spec.crf + 4)]), ...gop, "-x265-params", "log-level=error", "-tag:v", "hvc1", "-pix_fmt", "yuv420p"];case "hevc_videotoolbox":return ["-c:v", "hevc_videotoolbox", ...rate, ...gop, "-profile:v", "main", "-tag:v", "hvc1", "-pix_fmt", "yuv420p", "-allow_sw", "1"];- case "prores_ks":- return ["-c:v", "prores_ks", "-profile:v", "3", "-vendor", "apl0", "-pix_fmt", "yuv422p10le"];+ case "prores_ks": {+ // Profile 3 is 422 HQ, profile 4 is 4444. 4444 keeps the chroma the+ // graph composited at — every pixel its own color, which is what the+ // titles, captions and stickers drawn in RGBA were made of — and the+ // alpha-free pixel format spares the file a plane that is all opaque.+ const full = codec === "prores4444";+ return ["-c:v", "prores_ks", "-profile:v", full ? "4" : "3", "-vendor", "apl0",+ "-pix_fmt", full ? "yuv444p10le" : "yuv422p10le"];+ }default:throw new Error(`No encoder for ${codec}.`);}@@ -476,7 +483,7 @@ async function resolveMedia(* when the matrix itself is BT.2020, or when it's untagged and the wide* primaries/transfer tags are the only signal there is.*/-export function sdrConvert(c: Awaited<ReturnType<typeof videoColorInfo>>) {+export function sdrConvert(c: Awaited<ReturnType<typeof videoColorInfo>>, fmt: string) {if (c == null) return "";const matrix = c.matrix && c.matrix !== "unknown" ? c.matrix : null;const wide =@@ -485,7 +492,7 @@ export function sdrConvert(c: Awaited<ReturnType<typeof videoColorInfo>>) {(c.primaries === "bt2020" ||c.transfer === "arib-std-b67" ||c.transfer === "smpte2084"));- return wide ? "format=yuv420p10le,colorspace=all=bt709:iall=bt2020:format=yuv420p," : "";+ return wide ? `format=yuv420p10le,colorspace=all=bt709:iall=bt2020:format=${fmt},` : "";}/** A clip's effective playback rate (>0, default 1). */@@ -711,6 +718,25 @@ export async function runExport(let spec = narrowSpecToRange(given);const { width: W, height: H, fps } = spec;+ // The chroma the composite is built at. A full-chroma delivery composites+ // at 4:4:4, so the titles, captions, stickers and masks the graph draws in+ // RGBA reach the file with every pixel's own color — a 4:2:0 graph averages+ // that away before the encoder ever sees it, whatever the file is written+ // as afterward. H.264 and HEVC subsample at the encoder regardless, so their+ // graph stays 4:2:0 and costs what it always did.+ //+ // Eight bits a plane either way: the looks and effects carry 0–255+ // constants (`geq`, `lutyuv`), the sources decode to eight, and ProRes 4444+ // takes the exact widening to ten at the encoder.+ const full = deliveryCodec(spec.codec).chroma444 === true;+ const clipFmt = full ? "yuv444p" : "yuv420p";+ const alphaFmt = full ? "yuva444p" : "yuva420p";+ // `overlay` blends in its own pixel family, and its default is yuv420 —+ // left alone it would undo the graph's chroma at every composite.+ const ovl = full ? ":format=yuv444" : "";+ /** The same family, for the effect chains the kit writes. */+ const chroma = { pixFmt: clipFmt, overlay: ovl };+// A clip that plays backward is rendered off a turned copy of its span,// baked ahead of the graph in bounded chunks (turnMedia.ts): the clip// becomes a forward clip over that file, its curve's nodes carried along,@@ -749,7 +775,7 @@ export async function runExport(const video = !soundOnly && (await io.hasStream(src, "v"));const audio = await io.hasStream(src, "a");const mono = audio && (await io.audioChannels(src)) === 1;- const colorFix = video ? sdrConvert(await io.videoColorInfo(src)) : "";+ const colorFix = video ? sdrConvert(await io.videoColorInfo(src), clipFmt) : "";const decodeCost = video ? ((await io.videoDecodeCost(src)) ?? 0) : 0;const file = path.join(job.tmpDir, `turned_${tag}.${video ? "mov" : "wav"}`);const pivot = await bakeTurnedMedia(@@ -760,7 +786,7 @@ export async function runExport(duration: io.mediaDuration,},src,- { lo, hi, video, audio, colorFix, decodeCost },+ { lo, hi, video, audio, colorFix, decodeCost, fmt: clipFmt, master: full },file);turned.set(file, { video, audio, mono });@@ -781,7 +807,6 @@ export async function runExport(const overlayVideos = [...(spec.overlayVideos ?? [])].sort((a, b) => a.track - b.track || a.start - b.start);- const clipFmt = "yuv420p";// Every bare patch of frame in the graph — the letterbox around a fitted// clip, a gap on track 0, the backdrop behind an edge animation — is the// project's own background color. ffmpeg takes it as 0xRRGGBB.@@ -872,7 +897,7 @@ export async function runExport(const hasVideo = await io.hasStream(paths[i], "v", () => (videoProbeFailed = true));videoPresence.set(f, hasVideo || videoProbeFailed);// A failed color probe (null) means no conversion — SDR passthrough.- colorFix.set(f, sdrConvert(await io.videoColorInfo(paths[i])));+ colorFix.set(f, sdrConvert(await io.videoColorInfo(paths[i]), clipFmt));}));// The turned copies: already probed at the bake, already folded to SDR.@@ -1376,17 +1401,17 @@ export async function runExport(const scaleExpr = `scale=w='trunc(iw*(0.8+0.2*(${p}))/2)*2':h=-2:eval=frame`;if (fx.bg) {filters.push(- `[${inLab}]format=yuva420p,` ++ `[${inLab}]format=${alphaFmt},` +`fade=t=${side === "head" ? "in" : "out"}:st=0:d=${d}:alpha=1,${scaleExpr}[${sc}]`);filters.push(- `[${bg}][${sc}]overlay=x=(W-w)/2:y=(H-h)/2:shortest=1,format=${fmt}[${outLab}]`+ `[${bg}][${sc}]overlay=x=(W-w)/2:y=(H-h)/2:shortest=1${ovl},format=${fmt}[${outLab}]`);return;}filters.push(`[${inLab}]${scaleExpr}[${sc}]`);filters.push(- `[${bg}][${sc}]overlay=x=(W-w)/2:y=(H-h)/2:shortest=1,` ++ `[${bg}][${sc}]overlay=x=(W-w)/2:y=(H-h)/2:shortest=1${ovl},` +`fade=t=${side === "head" ? "in" : "out"}:st=0:d=${d},format=${fmt}[${outLab}]`);};@@ -1599,7 +1624,7 @@ export async function runExport(const brIdx = clipBorderInput.get(j);if (brIdx !== undefined) {filters.push(`${core}[cbi${j}]`);- filters.push(`[cbi${j}][${brIdx}:v]overlay=0:0:eof_action=pass,format=${segFmt}[cbo${j}]`);+ filters.push(`[cbi${j}][${brIdx}:v]overlay=0:0:eof_action=pass${ovl},format=${segFmt}[cbo${j}]`);core = `[cbo${j}]null`;}// A removal segment carries alpha, so its fades must ramp the alpha@@ -1630,7 +1655,7 @@ export async function runExport(// restores the constant-size opaque frame the join expects. The// multiply chains (alphaextract → blend → alphamerge) compose with// any alpha the segment carries.- pushEdgeFx(core, dur, headFx, tailFx, W, H, rmIn ? "yuva420p" : clipFmt, fades, `vmr${j}`, `c${j}`);+ pushEdgeFx(core, dur, headFx, tailFx, W, H, rmIn ? alphaFmt : clipFmt, fades, `vmr${j}`, `c${j}`);let cur = `vmr${j}`;if (mkIdx !== undefined) {filters.push(`[${mkIdx}:v]fps=${fps},scale=${W}:${H},setsar=1,format=gray[cmk${j}]`);@@ -1651,10 +1676,10 @@ export async function runExport(}const pos = posePositionExprs(c.kf!, 0);filters.push(- `[email protected]:s=${W}x${H}:r=${fps}:d=${num(dur)},format=yuva420p[ckb${j}]`+ `[email protected]:s=${W}x${H}:r=${fps}:d=${num(dur)},format=${alphaFmt}[ckb${j}]`);filters.push(- `[ckb${j}][${cur}]overlay=x=${pos.x}:y=${pos.y}:eof_action=pass[ckp${j}]`+ `[ckb${j}][${cur}]overlay=x=${pos.x}:y=${pos.y}:eof_action=pass${ovl}[ckp${j}]`);cur = `ckp${j}`;}@@ -1681,7 +1706,7 @@ export async function runExport();const flatOut = rmIn && c.look ? `vrl${j}` : segCore;filters.push(- `[cmb${j}][${cur}]overlay=0:0:shortest=1,format=${clipFmt},fps=${fps}[${flatOut}]`+ `[cmb${j}][${cur}]overlay=0:0:shortest=1${ovl},format=${clipFmt},fps=${fps}[${flatOut}]`);if (flatOut !== segCore) {// The removal clip's look, over the flattened opaque segment.@@ -1695,7 +1720,7 @@ export async function runExport(if (shIdx !== undefined) {filters.push(`[${shIdx}:v]fps=${fps},scale=${W}:${H},setsar=1,format=rgba[csh${j}]`);filters.push(- `[${segCore}][csh${j}]overlay=0:0:eof_action=pass,format=${clipFmt},fps=${fps}[${segOut}]`+ `[${segCore}][csh${j}]overlay=0:0:eof_action=pass${ovl},format=${clipFmt},fps=${fps}[${segOut}]`);}if (nFz > 0) {@@ -1958,12 +1983,12 @@ export async function runExport(const boxed =(ramped || maskIdx !== undefined || !!subjMask || keyed || !!oc.border) && !cover;const boxPad = boxed- ? `,format=yuva420p,pad=${boxW}:${boxH}:(ow-iw)/2:(oh-ih)/2:[email protected]`+ ? `,format=${alphaFmt},pad=${boxW}:${boxH}:(ow-iw)/2:(oh-ih)/2:[email protected]`: "";if (boxed) pos = region ? `${region.rx}:${region.ry}` : "0:0";- const fmt = hf > 0.01 || tf > 0.01 || boxed || orIn ? "yuva420p" : "yuv420p";+ const fmt = hf > 0.01 || tf > 0.01 || boxed || orIn ? alphaFmt : clipFmt;// The look chain reads the pre-pad picture, which is opaque.- const lookFmt = boxed ? "yuv420p" : fmt;+ const lookFmt = boxed ? clipFmt : fmt;const fades =(hf > 0.01 ? `,fade=t=in:st=0:d=${num(hf)}:alpha=1` : "") +(tf > 0.01 ? `,fade=t=out:st=${num(Math.max(0, olen - tf))}:d=${num(tf)}:alpha=1` : "");@@ -1984,7 +2009,7 @@ export async function runExport(? `[${idx}:v]setpts=PTS-STARTPTS`: `[${idx}:v]trim=${num(oc.in)}:${num(oc.out)},setpts=${retimedPts(ort)}`;}- let core = `${framedTimebase(timebase, framing, `o${k}`, orIn ? {} : oc, ort, fps, filters)},${orIn ? "" : colorFix.get(oc.file) ?? ""}${gradeChain(orIn ? undefined : oc.grade)}format=${orIn ? "yuva420p" : lookFmt}`;+ let core = `${framedTimebase(timebase, framing, `o${k}`, orIn ? {} : oc, ort, fps, filters)},${orIn ? "" : colorFix.get(oc.file) ?? ""}${gradeChain(orIn ? undefined : oc.grade)}format=${orIn ? alphaFmt : lookFmt}`;// Looks bake into footage overlays only: an image may carry alpha, which// the look chain's internal filters would flatten onto black over the// tracks beneath. The alpha fades stay safe: they apply after the look.@@ -2002,7 +2027,7 @@ export async function runExport(const obIdx = overlayBorderInput.get(oc);if (obIdx !== undefined) {filters.push(`${core}[obi${k}]`);- filters.push(`[obi${k}][${obIdx}:v]overlay=0:0:eof_action=pass,format=${fmt}[obo${k}]`);+ filters.push(`[obi${k}][${obIdx}:v]overlay=0:0:eof_action=pass${ovl},format=${fmt}[obo${k}]`);core = `[obo${k}]null`;}const pre = `ovp${k}`;@@ -2028,13 +2053,13 @@ export async function runExport(filters.push(`[${pre}]format=rgba,split[om0${k}][om1${k}]`);filters.push(`[om1${k}]alphaextract[oma${k}]`);filters.push(`[oma${k}][omk${k}]blend=all_mode=multiply[omm${k}]`);- filters.push(`[om0${k}][omm${k}]alphamerge,format=yuva420p[omc${k}]`);+ filters.push(`[om0${k}][omm${k}]alphamerge,format=${alphaFmt}[omc${k}]`);masked = `omc${k}`;}if (keyed) {const tf = poseTransformFilters(oc.kf!, boxW, boxH);if (tf) {- filters.push(`[${masked}]format=rgba${tf},format=yuva420p[okt${k}]`);+ filters.push(`[${masked}]format=rgba${tf},format=${alphaFmt}[okt${k}]`);masked = `okt${k}`;}}@@ -2054,7 +2079,7 @@ export async function runExport(`tpad=start_duration=${num(oc.start)}:[email protected][${lit}]`);const shOnto = `vovs${k}`;- filters.push(`[${onto}][${lit}]overlay=0:0:${enable}:eof_action=pass[${shOnto}]`);+ filters.push(`[${onto}][${lit}]overlay=0:0:${enable}:eof_action=pass${ovl}[${shOnto}]`);onto = shOnto;}if (subjMask) {@@ -2066,10 +2091,10 @@ export async function runExport(if (keyed) {const kpos = posePositionExprs(oc.kf!, oc.start);filters.push(- `[email protected]:s=${W}x${H}:r=${fps}:d=${num(spec.duration)},format=yuva420p[osb${k}]`+ `[email protected]:s=${W}x${H}:r=${fps}:d=${num(spec.duration)},format=${alphaFmt}[osb${k}]`);filters.push(- `[osb${k}][${seg}]overlay=x=${kpos.x}:y=${kpos.y}:eof_action=pass[osp${k}]`+ `[osb${k}][${seg}]overlay=x=${kpos.x}:y=${kpos.y}:eof_action=pass${ovl}[osp${k}]`);} else {// The box may reach past the frame; pad to the box holding both and@@ -2089,15 +2114,15 @@ export async function runExport(filters.push(`[osp${k}]format=rgba,split[os0${k}][os1${k}]`);filters.push(`[os1${k}]alphaextract[osa${k}]`);filters.push(`[osa${k}][${matte}]blend=all_mode=multiply[osm${k}]`);- filters.push(`[os0${k}][osm${k}]alphamerge,format=yuva420p[osc${k}]`);- filters.push(`[${onto}][osc${k}]overlay=0:0:${enable}:eof_action=pass[${next}]`);+ filters.push(`[os0${k}][osm${k}]alphamerge,format=${alphaFmt}[osc${k}]`);+ filters.push(`[${onto}][osc${k}]overlay=0:0:${enable}:eof_action=pass${ovl}[${next}]`);} else if (keyed) {const kpos = posePositionExprs(oc.kf!, oc.start);filters.push(- `[${onto}][${seg}]overlay=x=${kpos.x}:y=${kpos.y}:${enable}:eof_action=pass[${next}]`+ `[${onto}][${seg}]overlay=x=${kpos.x}:y=${kpos.y}:${enable}:eof_action=pass${ovl}[${next}]`);} else {- filters.push(`[${onto}][${seg}]overlay=${pos}:${enable}:eof_action=pass[${next}]`);+ filters.push(`[${onto}][${seg}]overlay=${pos}:${enable}:eof_action=pass${ovl}[${next}]`);}if (!oc.muted && audioPresence.get(oc.file)) {const vol = (oc.volume ?? 1) !== 1 ? `volume=${num(oc.volume ?? 1)},` : "";@@ -2162,18 +2187,18 @@ export async function runExport(filters.push(`[oe0${k}]fps=${fps}[oef${k}]`);filters.push(`[oe1${k}]alphaextract,fps=${fps}[oea${k}]`);filters.push(`[oea${k}][${matte}]blend=all_mode=multiply[oem${k}]`);- filters.push(`[oef${k}][oem${k}]alphamerge,format=yuva420p[oes${k}]`);- filters.push(`[${onto}][oes${k}]overlay=0:0:eof_action=pass[${next}]`);+ filters.push(`[oef${k}][oem${k}]alphamerge,format=${alphaFmt}[oes${k}]`);+ filters.push(`[${onto}][oes${k}]overlay=0:0:eof_action=pass${ovl}[${next}]`);return next;}- filters.push(`[${animIdx}:v]format=yuva420p,fps=${fps},setsar=1[oanim${k}]`);+ filters.push(`[${animIdx}:v]format=${alphaFmt},fps=${fps},setsar=1[oanim${k}]`);filters.push(- `[${onto}][oanim${k}]overlay=${num(o.x ?? 0)}:${num(o.y ?? 0)}:eof_action=pass[${next}]`+ `[${onto}][oanim${k}]overlay=${num(o.x ?? 0)}:${num(o.y ?? 0)}:eof_action=pass${ovl}[${next}]`);} else if (o.file) {const idx = inputIndex.get(o.file)!;filters.push(- `[${onto}][${idx}:v]overlay=0:0:enable='gte(t,${num(o.start)})*lt(t,${num(o.end)})'[${next}]`+ `[${onto}][${idx}:v]overlay=0:0:enable='gte(t,${num(o.start)})*lt(t,${num(o.end)})'${ovl}[${next}]`);} else {return onto;@@ -2215,7 +2240,8 @@ export async function runExport(spec.height,`fx${i}`,e.focus,- e.ramp+ e.ramp,+ chroma);if (!lines) continue;filters.push(...lines);@@ -2228,8 +2254,8 @@ export async function runExport(// costs one ffmpeg input per language instead of one per still. They sit// over every element and every effect.captionInputs.forEach((idx, k) => {- filters.push(`[${idx}:v]format=yuva420p,fps=${fps},setsar=1[caps${k}]`);- filters.push(`[${vLabel}][caps${k}]overlay=0:0:eof_action=pass[vcaps${k}]`);+ filters.push(`[${idx}:v]format=${alphaFmt},fps=${fps},setsar=1[caps${k}]`);+ filters.push(`[${vLabel}][caps${k}]overlay=0:0:eof_action=pass${ovl}[vcaps${k}]`);vLabel = `vcaps${k}`;});diff --git a/site/src/cut/server/turnMedia.test.ts b/site/src/cut/server/turnMedia.test.tsindex 35e6c311..3e111097 100644--- a/site/src/cut/server/turnMedia.test.ts+++ b/site/src/cut/server/turnMedia.test.ts@@ -30,6 +30,7 @@ const spec = (over: Partial<TurnSpec> = {}): TurnSpec => ({video: true,audio: true,colorFix: "",+ fmt: "yuv420p",...over,});diff --git a/site/src/cut/server/turnMedia.ts b/site/src/cut/server/turnMedia.tsindex 8e6b9db2..5ec442e8 100644--- a/site/src/cut/server/turnMedia.ts+++ b/site/src/cut/server/turnMedia.ts@@ -59,6 +59,13 @@ export interface TurnSpec {/** Bytes a second of the source's picture comes to decoded (see* `videoDecodeCost`), which sizes the chunks. */decodeCost?: number;+ /** The pixel format the pieces are written in, the one the graph reading+ * them works in. */+ fmt: string;+ /** Bake for a full-chroma delivery: ProRes 4444 pieces instead of the+ * H.264 ones, so a clip that plays backward is not the one place in a+ * master where the picture went through a lossy intermediate. */+ master?: boolean;}const vtQuality = (crf: number) => Math.round(Math.max(35, Math.min(80, 100 - crf * 1.8)));@@ -83,7 +90,7 @@ export async function bakeTurnedMedia(const chunk = spec.video ? turnChunkSeconds(spec.decodeCost) : TURN_CHUNK_S;const n = Math.max(1, Math.ceil(span / chunk));const pieces: string[] = [];- const enc = spec.video ? await io.h264Encoder() : null;+ const enc = spec.video && !spec.master ? await io.h264Encoder() : null;for (let i = 0; i < n; i++) {// Chunks run from the span's end back to its start, so the joined file// plays the span backward.@@ -93,11 +100,13 @@ export async function bakeTurnedMedia(const piece = `${outFile}.${i}${spec.video ? ".mov" : ".wav"}`;const args = ["-y", "-hide_banner", "-loglevel", "error", "-ss", num(from), "-t", num(to - from), "-i", src];if (spec.video) {- args.push("-vf", `${spec.colorFix}reverse,format=yuv420p`);+ args.push("-vf", `${spec.colorFix}reverse,format=${spec.fmt}`);args.push(- ...(enc === "libx264"- ? ["-c:v", "libx264", "-preset", "veryfast", "-crf", String(TURN_CRF)]- : ["-c:v", "h264_videotoolbox", "-q:v", String(vtQuality(TURN_CRF)), "-allow_sw", "1"])+ ...(spec.master+ ? ["-c:v", "prores_ks", "-profile:v", "4", "-vendor", "apl0", "-pix_fmt", "yuv444p10le"]+ : enc === "libx264"+ ? ["-c:v", "libx264", "-preset", "veryfast", "-crf", String(TURN_CRF)]+ : ["-c:v", "h264_videotoolbox", "-q:v", String(vtQuality(TURN_CRF)), "-allow_sw", "1"]));} else {args.push("-vn");