The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
feat(export): ProRes 4444 and VP9 alpha codecs that keep a transparent frame's alpha (#456)
Jareer23bc68998cfb
diff --git a/src/crates/concat-export/src/lib.rs b/src/crates/concat-export/src/lib.rsindex 111c6b8a..cc12b831 100644--- a/src/crates/concat-export/src/lib.rs+++ b/src/crates/concat-export/src/lib.rs@@ -433,7 +433,9 @@ pub struct ExportRequest {pub crf: u8,/// The x264 speed/size preset name, e.g. "medium".pub preset: String,- /// What to encode to, by name: "h264", "hevc" or "av1". H.264 when a+ /// What to encode to, by name: "h264", "hevc", "av1", or "prores-4444"+ /// or "vp9-alpha" to keep the frame's alpha, whose `output` must then+ /// be a .mov or a .webm (see [`VideoCodec::container`]). H.264 when a/// request does not say.#[serde(default, deserialize_with = "codec_by_name")]pub codec: VideoCodec,@@ -509,7 +511,9 @@ fn codec_by_name<'de, D: serde::Deserializer<'de>>() -> Result<VideoCodec, D::Error> {let name = String::deserialize(deserializer)?;VideoCodec::parse(&name).ok_or_else(|| {- serde::de::Error::custom(format!("unknown codec {name:?}: h264, hevc or av1"))+ serde::de::Error::custom(format!(+ "unknown codec {name:?}: h264, hevc, av1, prores-4444 or vp9-alpha"+ ))})}@@ -1168,6 +1172,24 @@ pub fn render_on());}let output = PathBuf::from(&request.output);+ // An alpha codec's file is only what it says in its own container: a+ // ProRes picture in a file named .mp4 opens nowhere, and a WebM's+ // alpha means nothing to a reader that takes the name for an MP4. The+ // three ordinary codecs keep writing an MP4 under whatever name they+ // are given, as they always have.+ if request.codec.keeps_alpha() {+ let wanted = request.codec.container();+ let named = output+ .extension()+ .is_some_and(|extension| extension.eq_ignore_ascii_case(wanted));+ if !named {+ return Err(format!(+ "{} is written to a .{wanted} file, and {} is not one",+ request.codec.label(),+ output.display()+ ));+ }+ }// Transitions become overlaps, ramps and fade filters before anything// else reads the clip list, so the picture and sound paths below never@@ -1231,9 +1253,15 @@ pub fn render_on(directory.display()));}- let silent = directory.join(format!(".{stem}.concat-video.mp4"));- let mixed = directory.join(format!(".{stem}.concat-audio.m4a"));- let muxed = directory.join(format!(".{stem}.concat-muxed.mp4"));+ // Each in the container the codec is written in, which the muxers pick+ // by the extension: a ProRes picture goes in a QuickTime file, and a+ // WebM's sound is Opus, the one a WebM takes besides Vorbis. A .mov+ // takes the AAC an MP4 does.+ let container = request.codec.container();+ let sound_container = if container == "webm" { "webm" } else { "m4a" };+ let silent = directory.join(format!(".{stem}.concat-video.{container}"));+ let mixed = directory.join(format!(".{stem}.concat-audio.{sound_container}"));+ let muxed = directory.join(format!(".{stem}.concat-muxed.{container}"));// The output path is touched once, by the rename at the end, whichever// way the file was made. The mux used to write it directly, and a mux@@ -1751,7 +1779,10 @@ fn render_picture(// An HDR file comes out of the compositor as its signal, sixteen bits a// channel; anything else, SDR - an HDR timeline's tone-mapped.let signal = output_of(request.color_space);- let hdr = request.hdr && signal != Signal::Sdr;+ // The alpha codecs are written SDR alone (see `Encoder::create_hdr`):+ // an HDR timeline is tone-mapped into one, as it is on a phone that+ // cannot write ten bits, rather than the export failing.+ let hdr = request.hdr && signal != Signal::Sdr && !request.codec.keeps_alpha();compositor.deliver_hdr(hdr);// How many devices have died under this export so far; see `recovered`.@@ -3668,6 +3699,133 @@ mod tests {result.expect("the export goes on on a fresh device");}+ /// An export in an alpha codec writes its own container with its sound+ /// in it - a .mov of ProRes and AAC, a .webm of VP9 and Opus - and one+ /// named for another container is refused before a frame is drawn.+ #[test]+ fn an_alpha_export_writes_its_own_container_with_its_sound() {+ use concat_core::frame::Frame;+ use concat_media::{EncodeOptions, Encoder, FrameSink};++ if WgpuCompositor::new().is_none() {+ eprintln!("nothing to draw on here; skipped");+ return;+ }+ let dir = std::env::temp_dir().join(format!("concat-export-alpha-{}", std::process::id()));+ let _ = std::fs::create_dir_all(&dir);+ let source = dir.join("source.mp4");+ let Ok(mut encoder) = Encoder::create(+ &source,+ 64,+ 64,+ FrameRate::THIRTY,+ &EncodeOptions::default(),+ ) else {+ return; // no ffmpeg here+ };+ for _ in 0..15 {+ encoder.write_frame(&Frame::black(64, 64)).expect("writes");+ }+ encoder.finish().expect("finishes");+ // Half a second of a 440 Hz tone, 16-bit stereo at 48 kHz.+ let tone = dir.join("tone.wav");+ {+ let samples: Vec<u8> = (0..24_000)+ .flat_map(|index| {+ let t = f64::from(index) / 48_000.0;+ let value = (0.5 * (std::f64::consts::TAU * 440.0 * t).sin() * 32767.0) as i16;+ let [low, high] = value.to_le_bytes();+ [low, high, low, high]+ })+ .collect();+ let mut bytes = Vec::new();+ bytes.extend_from_slice(b"RIFF");+ bytes.extend_from_slice(&(36 + samples.len() as u32).to_le_bytes());+ bytes.extend_from_slice(b"WAVEfmt ");+ for (value, width) in [+ (16u32, 4),+ (1, 2),+ (2, 2),+ (48_000, 4),+ (192_000, 4),+ (4, 2),+ (16, 2),+ ] {+ bytes.extend_from_slice(&value.to_le_bytes()[..width]);+ }+ bytes.extend_from_slice(b"data");+ bytes.extend_from_slice(&(samples.len() as u32).to_le_bytes());+ bytes.extend_from_slice(&samples);+ std::fs::write(&tone, bytes).expect("writes the tone");+ }++ let mut footage = clip("video", 0, 0.0, 0.5, 0.0);+ footage.path = source.to_string_lossy().into_owned();+ footage.media_width = Some(64);+ footage.media_height = Some(64);+ footage.has_audio = Some(false);+ let mut sound = clip("audio", 1, 0.0, 0.5, 0.0);+ sound.path = tone.to_string_lossy().into_owned();+ sound.has_audio = Some(true);+ let request = |codec: VideoCodec, output: &Path| ExportRequest {+ canvas: [0, 0, 0],+ output: output.to_string_lossy().into_owned(),+ width: 64,+ height: 64,+ rate_num: 30,+ rate_den: 1,+ crf: 30,+ preset: "ultrafast".into(),+ codec,+ ten_bit: false,+ rate_mode: RateMode::Vbr,+ bitrate_kbps: 0,+ color_range: concat_media::ColorRange::Limited,+ color_space: ColorSpace::Sdr,+ hdr: false,+ effect_time_offset: 0.0,+ clips: vec![footage.clone(), sound.clone()],+ };+ let render = |request: &ExportRequest| {+ let cancel = AtomicBool::new(false);+ let mut progress = |_: i64, _: i64, _: &'static str| {};+ let reporter = Reporter {+ progress: &mut progress,+ cancel: &cancel,+ };+ render_on(request, None, reporter)+ };++ for (codec, picture, heard) in [+ (VideoCodec::ProRes4444, "prores", "aac"),+ (VideoCodec::Vp9Alpha, "vp9", "opus"),+ ] {+ if !codec.available() {+ eprintln!("{} not in the linked FFmpeg; skipped", codec.label());+ continue;+ }+ let misnamed = dir.join("overlay.mp4");+ let error = render(&request(codec, &misnamed)).expect_err("an .mp4 is refused");+ assert!(error.contains(codec.container()), "{error}");+ assert!(!misnamed.exists());++ let output = dir.join(format!("overlay.{}", codec.container()));+ let written = render(&request(codec, &output))+ .unwrap_or_else(|error| panic!("{}: {error}", codec.label()));+ assert_eq!(Path::new(&written), output);+ let info = concat_media::probe(&output).expect("probes");+ assert_eq!(info.video.expect("a picture").codec, picture);+ assert_eq!(info.audio.expect("a sound").codec, heard);+ let leftovers: Vec<_> = std::fs::read_dir(&dir)+ .expect("lists")+ .filter_map(|entry| entry.ok())+ .filter(|entry| entry.file_name().to_string_lossy().contains(".concat-"))+ .collect();+ assert!(leftovers.is_empty(), "scratch files are removed");+ }+ let _ = std::fs::remove_dir_all(&dir);+ }+/// A device that dies under an export is replaced: first by a fresh/// one on the GPU, then by the software adapter; the third death ends/// the export with a message that says how many there were.diff --git a/src/crates/concat-media/src/audio.rs b/src/crates/concat-media/src/audio.rsindex 01641e4f..50245cca 100644--- a/src/crates/concat-media/src/audio.rs+++ b/src/crates/concat-media/src/audio.rs@@ -542,8 +542,27 @@ impl MixInput {}}+/// The Opus encoder to mix into and the samples it takes: libopus where it+/// is linked, which the desktops' builds and Homebrew carry, and FFmpeg's+/// own otherwise, which every build has - a phone's included. A WebM takes+/// Opus or Vorbis and nothing else, and Opus is the better of the two.+fn opus_encoder() -> Result<(&'static str, Sample)> {+ use ffmpeg::util::format::sample::Type;+ if encoder::find_by_name("libopus").is_some() {+ return Ok(("libopus", Sample::F32(Type::Packed)));+ }+ if encoder::find_by_name("opus").is_some() {+ return Ok(("opus", Sample::F32(Type::Planar)));+ }+ Err(Error::Missing {+ what: "encoder",+ name: "opus".to_owned(),+ })+}+/// Mixes `clips` into one audio file at `destination`: 16-bit PCM for a-/// `.wav` destination (case-insensitive), AAC otherwise, at 48 kHz stereo.+/// `.wav` destination (case-insensitive), Opus for a `.webm`, `.opus`,+/// `.ogg` or `.mka` one, AAC otherwise, at 48 kHz stereo.////// Every clip's file must contain an audio stream; see [`mix_graph`].pub fn mix_to_file(clips: &[AudioClip], duration: f64, destination: &Path) -> Result<()> {@@ -557,18 +576,22 @@ pub fn mix_to_file_with(duration: f64,destination: &Path,) -> Result<()> {- let wav = destination+ let extension = destination.extension()- .is_some_and(|extension| extension.eq_ignore_ascii_case("wav"));+ .map(|extension| extension.to_string_lossy().to_ascii_lowercase());+ let wav = extension.as_deref() == Some("wav");+ let opus = matches!(extension.as_deref(), Some("webm" | "opus" | "ogg" | "mka"));+ ffi::init();let (codec_name, mix_format) = if wav {("pcm_s16le",Sample::I16(ffmpeg::util::format::sample::Type::Packed),)+ } else if opus {+ opus_encoder()?} else {("aac", MIX_FORMAT)};- ffi::init();// A bad chain is refused before any file is opened.for chain in clips.iter()@@ -603,6 +626,11 @@ pub fn mix_to_file_with(if global_header {audio.set_flags(ffmpeg::codec::Flags::GLOBAL_HEADER);}+ // FFmpeg's own Opus encoder is still marked experimental, and opens+ // only when told that is all right; libopus needs no such word.+ if codec_name == "opus" {+ audio.compliance(ffmpeg::codec::Compliance::Experimental);+ }let mut encoder = audio.open().map_err(|error| ffi::fail("open encoder", destination, error))?;@@ -1000,6 +1028,12 @@ pub fn mux(video: &Path, audio: &Path, output: &Path) -> Result<()> {.map_err(|error| ffi::fail("add stream", output, error))?;stream.set_parameters(video_stream.parameters());stream.set_time_base(video_tb);+ // A WebM's alpha layer is in the packets, but the track only says+ // it has one in its header: without the word a reader takes the+ // picture for opaque and never opens the decoder that reads it.+ if let Some(mode) = video_stream.metadata().get("alpha_mode") {+ stream.metadata_mut().set("alpha_mode", mode);+ }}{let mut stream = out@@ -1008,7 +1042,15 @@ pub fn mux(video: &Path, audio: &Path, output: &Path) -> Result<()> {stream.set_parameters(audio_stream.parameters());stream.set_time_base(audio_tb);}- out.write_header_with(ffmpeg::dict! { "movflags" => "+faststart" })+ // The index up front, as the picture's own file had it - but not for+ // ProRes, where moving it writes gigabytes a minute again for a file+ // an editor reads whole.+ let header = if video_stream.parameters().id() == ffmpeg::codec::Id::PRORES {+ ffmpeg::Dictionary::new()+ } else {+ ffmpeg::dict! { "movflags" => "+faststart" }+ };+ out.write_header_with(header).map_err(|error| ffi::fail("write header", output, error))?;let out_video_tb = out.stream(0)@@ -1701,6 +1743,103 @@ mod tests {std::fs::remove_dir_all(dir).expect("removes test files");}+ /// An overlay with alpha and its sound, joined the way an export joins+ /// them: a ProRes picture with AAC in a .mov, a VP9 one with Opus in a+ /// .webm, the only sound a WebM takes besides Vorbis. The joined WebM+ /// still says it has an alpha layer, so the import reads it with the+ /// decoder that keeps it.+ #[test]+ fn an_overlay_with_alpha_takes_its_sound_in_its_own_container() {+ use crate::decode::{DecodeOptions, Decoder, FrameSource};+ use crate::{AudioDecoder, AudioOptions, EncodeOptions, Encoder, FrameSink, VideoCodec};+ use concat_core::frame::Frame;+ use concat_core::time::FrameRate;++ let dir = std::env::temp_dir().join(format!("concat-alpha-mux-{}", std::process::id()));+ std::fs::create_dir_all(&dir).expect("creates a folder");+ let tone = dir.join("tone.wav");+ tone_wav(&tone, 1.0);+ for codec in [VideoCodec::ProRes4444, VideoCodec::Vp9Alpha] {+ if !codec.available() {+ eprintln!("{} not in the linked FFmpeg; skipped", codec.label());+ continue;+ }+ let container = codec.container();+ let webm = container == "webm";+ let video = dir.join(format!("picture.{container}"));+ let sound = dir.join(if webm { "sound.webm" } else { "sound.m4a" });+ let output = dir.join(format!("joined.{container}"));++ // A second of a frame clear on its right half.+ let mut frame = Frame::transparent(64, 64);+ for y in 0..64 {+ for x in 0..32 {+ frame.set_pixel(x, y, [200, 40, 40, 255]);+ }+ }+ let options = EncodeOptions {+ codec,+ preset: "ultrafast".to_owned(),+ hardware: false,+ ..EncodeOptions::default()+ };+ let mut encoder =+ Encoder::create(&video, 64, 64, FrameRate::THIRTY, &options).expect("opens");+ for _ in 0..30 {+ encoder.write_frame(&frame).expect("writes");+ }+ encoder.finish().expect("finishes");+ let piece = AudioClip {+ path: tone.clone(),+ duration: 1.0,+ ..clip("")+ };+ mix_to_file(&[piece], 1.0, &sound).expect("mixes");+ mux(&video, &sound, &output).expect("muxes");++ let counts = packets_of(&output);+ assert_eq!(counts[0], 30, "{}: every picture packet", codec.label());+ {+ let input = ffmpeg::format::input(&output).expect("opens");+ let picture = input+ .streams()+ .best(ffmpeg::media::Type::Video)+ .expect("video");+ if webm {+ assert_eq!(+ picture.metadata().get("alpha_mode"),+ Some("1"),+ "the joined track still says it has alpha"+ );+ }+ let heard = input+ .streams()+ .best(ffmpeg::media::Type::Audio)+ .expect("audio");+ let wanted = if webm {+ ffmpeg::codec::Id::OPUS+ } else {+ ffmpeg::codec::Id::AAC+ };+ assert_eq!(heard.parameters().id(), wanted, "{}", codec.label());+ }+ let mut decoder =+ Decoder::open(&output, &DecodeOptions::default().in_software()).expect("opens");+ let back = decoder.next_frame().expect("decodes").expect("a frame");+ let [.., opaque] = back.pixel(16, 32).expect("inside");+ let [.., clear] = back.pixel(48, 32).expect("inside");+ assert!(opaque > 250, "{}: opaque came back {opaque}", codec.label());+ assert!(clear < 8, "{}: clear came back {clear}", codec.label());+ let mut sound = AudioDecoder::open(&output, &AudioOptions::default()).expect("opens");+ let mut audible = false;+ while let Some(samples) = sound.next_i16().expect("decodes") {+ audible |= samples.iter().any(|sample| i32::from(*sample).abs() > 1000);+ }+ assert!(audible, "{}: the tone is heard", codec.label());+ }+ std::fs::remove_dir_all(dir).expect("removes test files");+ }+#[test]fn saving_to_m4a_keeps_aac_encoding_and_audible_samples() {use crate::{AudioDecoder, AudioOptions};diff --git a/src/crates/concat-media/src/decode.rs b/src/crates/concat-media/src/decode.rsindex 61619e47..b08225b5 100644--- a/src/crates/concat-media/src/decode.rs+++ b/src/crates/concat-media/src/decode.rs@@ -1280,6 +1280,24 @@ impl Decoder {self.hardware = None;}}+ // FFmpeg's ProRes decoder reads the picture's matrix into the+ // stream and leaves every frame it makes untagged, and swscale+ // then reads BT.601 into what is a BT.709 picture: a ProRes+ // 4444 overlay's reds came back orange. Its frames are given+ // the stream's matrix where they say none. Only ProRes: the+ // other decoders tag their frames as their streams are tagged.+ if self.decoder.id() == ffmpeg::codec::Id::PRORES+ && self.color.matrix != ffmpeg::color::Space::Unspecified+ {+ // SAFETY: a frame this reader owns; the matrix is a plain+ // field the filtergraph reads with the picture.+ unsafe {+ let raw = frame.as_mut_ptr();+ if (*raw).colorspace == ffmpeg::sys::AVColorSpace::UNSPECIFIED {+ (*raw).colorspace = self.color.matrix.into();+ }+ }+ }self.discard_before = None;self.discard_through = None;self.last_pts = pts;diff --git a/src/crates/concat-media/src/encode.rs b/src/crates/concat-media/src/encode.rsindex db84e25f..6c8843ca 100644--- a/src/crates/concat-media/src/encode.rs+++ b/src/crates/concat-media/src/encode.rs@@ -47,11 +47,26 @@ pub enum VideoCodec {/// AV1: smaller again, royalty-free, the web's; slower to encode in/// software and not every old device plays it.Av1,+ /// Apple ProRes 4444 in a `.mov`, with its alpha: the file other+ /// editors - Premiere, Resolve, Final Cut, CapCut on a desktop - take+ /// a transparent overlay in. Ten bits, its quality set by the profile+ /// rather than a CRF, and very large: about forty times H.264.+ ProRes4444,+ /// VP9 with an alpha layer in a `.webm`: a transparent overlay for a+ /// web page or OBS, a small fraction of ProRes' size. Eight bits.+ Vp9Alpha,}impl VideoCodec {- /// Every codec, in the order a menu lists them.- pub const ALL: [VideoCodec; 3] = [VideoCodec::H264, VideoCodec::Hevc, VideoCodec::Av1];+ /// Every codec, in the order a menu lists them: the three that make an+ /// ordinary `.mp4`, then the two that keep a transparent frame's alpha.+ pub const ALL: [VideoCodec; 5] = [+ VideoCodec::H264,+ VideoCodec::Hevc,+ VideoCodec::Av1,+ VideoCodec::ProRes4444,+ VideoCodec::Vp9Alpha,+ ];/// The name a document or a request stores.pub fn name(self) -> &'static str {@@ -59,15 +74,23 @@ impl VideoCodec {VideoCodec::H264 => "h264",VideoCodec::Hevc => "hevc",VideoCodec::Av1 => "av1",+ VideoCodec::ProRes4444 => "prores-4444",+ VideoCodec::Vp9Alpha => "vp9-alpha",}}/// The codec a stored name means, or `None` for one nobody stores.+ /// Plain "vp9" is not taken for the alpha one: a VP9 file without alpha+ /// is a different request, and this one writes a second layer.pub fn parse(name: &str) -> Option<Self> {match name.trim().to_ascii_lowercase().as_str() {"h264" | "h.264" | "avc" => Some(VideoCodec::H264),"hevc" | "h265" | "h.265" => Some(VideoCodec::Hevc),"av1" => Some(VideoCodec::Av1),+ "prores-4444" | "prores_4444" | "prores4444" | "prores 4444" | "prores" => {+ Some(VideoCodec::ProRes4444)+ }+ "vp9-alpha" | "vp9_alpha" | "vp9alpha" | "vp9 alpha" => Some(VideoCodec::Vp9Alpha),_ => None,}}@@ -78,9 +101,37 @@ impl VideoCodec {VideoCodec::H264 => "H.264",VideoCodec::Hevc => "HEVC",VideoCodec::Av1 => "AV1",+ VideoCodec::ProRes4444 => "ProRes 4444",+ VideoCodec::Vp9Alpha => "VP9 alpha",+ }+ }++ /// Whether a file in it keeps the frame's alpha: a transparent+ /// background stays transparent in the file, where the other three+ /// write every pixel opaque.+ pub fn keeps_alpha(self) -> bool {+ matches!(self, VideoCodec::ProRes4444 | VideoCodec::Vp9Alpha)+ }++ /// The container a file in it is written in, as the file's extension:+ /// "mp4" for H.264, HEVC and AV1, "mov" for ProRes and "webm" for VP9+ /// with alpha. An MP4 has no place for either alpha layer: ProRes is+ /// QuickTime's, and VP9's alpha rides in a Matroska block addition.+ pub fn container(self) -> &'static str {+ match self {+ VideoCodec::H264 | VideoCodec::Hevc | VideoCodec::Av1 => "mp4",+ VideoCodec::ProRes4444 => "mov",+ VideoCodec::Vp9Alpha => "webm",}}+ /// Whether the codec is steered by a CRF or a bitrate at all. ProRes is+ /// not: its profile is the quality, and an export's rate mode, bitrate+ /// and CRF are left unused.+ pub fn rate_controlled(self) -> bool {+ self != VideoCodec::ProRes4444+ }+/// The FFmpeg encoders that make this codec, best first; an export/// takes the first that is linked and opens.///@@ -89,6 +140,12 @@ impl VideoCodec {/// encoders (scripts/ffmpeg-mobile.sh), and its cores are no match for/// its chip anyway.///+ /// The alpha codecs have one encoder each, on the CPU everywhere:+ /// FFmpeg's own ProRes encoder, which every build carries, a phone's+ /// too, and libvpx for VP9, which the desktops' GPL builds and Homebrew+ /// link and a phone's LGPL build does not. No hardware encoder here+ /// takes an alpha layer.+ ////// On a desktop the software encoders lead, and the machine's own/// comes after them for an FFmpeg built without them. With `hardware`,/// the GPU's leads for HEVC and AV1: VideoToolbox on macOS, and on@@ -101,6 +158,11 @@ impl VideoCodec {/// compatibility, not speed. Media Foundation, Windows' own, is last:/// it runs on whatever the machine has, and says little about how.pub fn encoders(self, hardware: bool) -> &'static [&'static str] {+ match self {+ VideoCodec::ProRes4444 => return &["prores_ks"],+ VideoCodec::Vp9Alpha => return &["libvpx-vp9"],+ VideoCodec::H264 | VideoCodec::Hevc | VideoCodec::Av1 => {}+ }let macos = cfg!(target_os = "macos");let windows = cfg!(target_os = "windows");if cfg!(target_os = "android") {@@ -108,13 +170,14 @@ impl VideoCodec {VideoCodec::H264 => &["h264_mediacodec"],VideoCodec::Hevc => &["hevc_mediacodec"],VideoCodec::Av1 => &["av1_mediacodec"],+ VideoCodec::ProRes4444 | VideoCodec::Vp9Alpha => &[],};}if cfg!(target_os = "ios") {return match self {VideoCodec::H264 => &["h264_videotoolbox"],VideoCodec::Hevc => &["hevc_videotoolbox"],- VideoCodec::Av1 => &[],+ VideoCodec::Av1 | VideoCodec::ProRes4444 | VideoCodec::Vp9Alpha => &[],};}match self {@@ -138,6 +201,7 @@ impl VideoCodec {&["av1_nvenc", "av1_qsv", "av1_amf", "libsvtav1", "libaom-av1"]}VideoCodec::Av1 => &["libsvtav1", "libaom-av1", "av1_nvenc", "av1_qsv", "av1_amf"],+ VideoCodec::ProRes4444 | VideoCodec::Vp9Alpha => &[],}}@@ -169,7 +233,8 @@ impl VideoCodec {/// [`encodable_among`] for the rest. A hardware encoder that is asked/// is opened once, at 720p, and the answer kept for the process.pub fn encodable(self) -> bool {- static KNOWN: [OnceLock<bool>; 3] = [const { OnceLock::new() }; 3];+ static KNOWN: [OnceLock<bool>; VideoCodec::ALL.len()] =+ [const { OnceLock::new() }; VideoCodec::ALL.len()];let slot = VideoCodec::ALL.iter().position(|codec| *codec == self)@@ -181,7 +246,8 @@ impl VideoCodec {/// Whether it can be written at ten bits here: some linked encoder for/// it takes ten-bit pictures. Not on a phone, whose encoders FFmpeg- /// feeds eight bits only.+ /// feeds eight bits only. ProRes is ten bits whether asked or not; VP9+ /// with alpha is eight, as libvpx takes its alpha layer at eight only.pub fn ten_bit_available(self) -> bool {self.linked(true, true).next().is_some()}@@ -203,12 +269,17 @@ impl VideoCodec {}/// Bytes per second relative to H.264 at the same quality, for a size- /// estimate: the rule of thumb the codecs are chosen by.+ /// estimate: the rule of thumb the codecs are chosen by. ProRes is not+ /// steered by a rate at all - its profile comes to about 330 Mb/s for+ /// a 1080p30 4444, some forty times H.264 - and VP9's alpha layer adds+ /// about a third to its picture.pub fn size_factor(self) -> f32 {match self {VideoCodec::H264 => 1.0,VideoCodec::Hevc => 0.6,VideoCodec::Av1 => 0.5,+ VideoCodec::ProRes4444 => 40.0,+ VideoCodec::Vp9Alpha => 0.8,}}}@@ -334,6 +405,28 @@ fn svt_preset(preset: &str) -> u8 {.map_or(6, |index| SVT[index])}+/// libvpx's `cpu-used` for an x264 preset, at its `good` deadline: 0 is+/// its slowest and 5 its fastest there, and x264's medium lands on 3.+fn vpx_speed(preset: &str) -> u8 {+ const VPX: [u8; 10] = [5, 5, 5, 4, 4, 3, 2, 1, 0, 0];+ X264_PRESETS+ .iter()+ .position(|name| *name == preset)+ .map_or(3, |index| VPX[index])+}++/// The picture an encoder of an alpha codec takes, when `encoder_name` is+/// one: ProRes 4444's ten-bit 4:4:4 with its alpha plane, and libvpx's+/// 4:2:0 with one, the only alpha layout it reads. swscale fills the alpha+/// plane from the RGBA frame's own alpha on the way.+fn alpha_pixel_format(encoder_name: &str) -> Option<Pixel> {+ match encoder_name {+ "prores_ks" => Some(Pixel::YUVA444P10LE),+ "libvpx-vp9" => Some(Pixel::YUVA420P),+ _ => None,+ }+}+/// VideoToolbox's quality, 1 to 100 with 100 the best, for an x264 CRF:/// 16 lands at 65 and 26 at 43, which is where the files come out about/// the size x264 makes them.@@ -345,7 +438,9 @@ fn videotoolbox_quality(crf: u8) -> u8 {/// in, and what it is steered by.#[derive(Clone, Copy, PartialEq, Eq, Debug)]enum Family {- /// x264, x265, SVT-AV1, libaom: planar 4:2:0, steered by a CRF.+ /// x264, x265, SVT-AV1, libaom: planar 4:2:0, steered by a CRF. ProRes+ /// and libvpx too, with pictures of their own that carry the alpha+ /// (see [`alpha_pixel_format`]).Software,/// Apple's: semi-planar 4:2:0, steered by a quality.VideoToolbox,@@ -565,6 +660,10 @@ fn opens_here(codec: VideoCodec, encoder_name: &'static str) -> bool {/// and AMF make ten-bit HEVC but not ten-bit H.264, and NVENC and Quick/// Sync make ten-bit HEVC and AV1.fn takes_ten_bits(encoder_name: &str) -> bool {+ // libvpx's alpha layer is eight bits only.+ if encoder_name == "libvpx-vp9" {+ return false;+ }match Family::of(encoder_name) {Family::Software => true,Family::VideoToolbox | Family::Amf => encoder_name.starts_with("hevc_"),@@ -690,9 +789,12 @@ fn open_video(let pq = hdr == Some(Signal::Pq);let family = Family::of(encoder_name);let plain = attempt != Attempt::AsAsked;- let pixel_format = family.pixel_format(options.ten_bit);- let cbr = options.rate_mode == RateMode::Cbr && options.bitrate_kbps > 0;- let average = options.rate_mode == RateMode::Average && options.bitrate_kbps > 0;+ let pixel_format =+ alpha_pixel_format(encoder_name).unwrap_or_else(|| family.pixel_format(options.ten_bit));+ // ProRes has no rate to steer: its profile is the quality.+ let rated = options.codec.rate_controlled();+ let cbr = rated && options.rate_mode == RateMode::Cbr && options.bitrate_kbps > 0;+ let average = rated && options.rate_mode == RateMode::Average && options.bitrate_kbps > 0;let mut video = ffmpeg::codec::Context::new_with_codec(codec).encoder().video()@@ -856,6 +958,49 @@ fn open_video("crf" => &options.crf.saturating_add(8).min(63).to_string(),"cpu-used" => "6",},+ // ProRes 4444 is the profile FFmpeg picks for a picture with alpha,+ // so the profile is not named: the encoder's own `profile` shares a+ // name with the context's, which would take the word first. The+ // alpha at sixteen bits, the most the format keeps, and Apple's+ // vendor code, which some of the editors that read it look for.+ "prores_ks" => ffmpeg::dict! {+ "alpha_bits" => "16",+ "vendor" => "apl0",+ },+ // libvpx's alt-ref frames are pictures that are never shown, and+ // its alpha layer is encoded beside the colour frame by frame, so+ // they are off. Row threading, as every core is wanted.+ "libvpx-vp9" if cbr => ffmpeg::dict! {+ "deadline" => "good",+ "cpu-used" => &vpx_speed(&options.preset).to_string(),+ "row-mt" => "1",+ "auto-alt-ref" => "0",+ "b" => bitrate.as_str(),+ "minrate" => bitrate.as_str(),+ "maxrate" => bitrate.as_str(),+ "bufsize" => bufsize.as_str(),+ },+ "libvpx-vp9" if average => ffmpeg::dict! {+ "deadline" => "good",+ "cpu-used" => &vpx_speed(&options.preset).to_string(),+ "row-mt" => "1",+ "auto-alt-ref" => "0",+ "b" => bitrate.as_str(),+ "maxrate" => peak.as_str(),+ "bufsize" => window.as_str(),+ },+ // Constant quality: a CRF with no bitrate beside it, which libvpx+ // otherwise reads as a ceiling. VP9's scale runs to 63 like AV1's,+ // and about twelve on from x264's is where the pictures match -+ // 18 lands at 30, Google's own suggestion for 1080p.+ "libvpx-vp9" => ffmpeg::dict! {+ "deadline" => "good",+ "cpu-used" => &vpx_speed(&options.preset).to_string(),+ "row-mt" => "1",+ "auto-alt-ref" => "0",+ "crf" => &options.crf.saturating_add(12).min(63).to_string(),+ "b" => "0",+ },// VideoToolbox steers by the context's bitrate when it is given no// quality."hevc_videotoolbox" | "h264_videotoolbox" if average => ffmpeg::dict! {@@ -997,6 +1142,25 @@ fn open_video(Ok((encoder, pixel_format))}+/// `options` as the codec can take them. ProRes is ten bits and video+/// range whatever is asked: its pictures are ten bits or more, and its+/// stream has no word for full range, so a full-range file would be read+/// back as video range by everything that opens it. VP9 with alpha is+/// eight bits, the only depth libvpx takes an alpha layer at. The other+/// codecs' options are left exactly as they were.+fn settled(options: &EncodeOptions) -> EncodeOptions {+ let mut options = options.clone();+ match options.codec {+ VideoCodec::ProRes4444 => {+ options.ten_bit = true;+ options.color_range = ColorRange::Limited;+ }+ VideoCodec::Vp9Alpha => options.ten_bit = false,+ VideoCodec::H264 | VideoCodec::Hevc | VideoCodec::Av1 => {}+ }+ options+}+/// A four-character code as the muxer stores it.const fn fourcc(tag: [u8; 4]) -> u32 {u32::from_le_bytes(tag)@@ -1253,6 +1417,19 @@ impl Encoder {detail: "H.264 does not carry HDR here: choose HEVC or AV1".to_owned(),});}+ // The alpha codecs are written SDR alone. ProRes can carry HDR, but+ // nothing here tags or checks it, and VP9's alpha layer is eight+ // bits.+ if options.codec.keeps_alpha() {+ return Err(Error::Ffi {+ operation: "open an HDR encoder",+ path: path.to_path_buf(),+ detail: format!(+ "{} is written SDR here: choose HEVC or AV1 for HDR",+ options.codec.label()+ ),+ });+ }let options = EncodeOptions {ten_bit: true,..options.clone()@@ -1285,6 +1462,7 @@ impl Encoder {hdr: Option<Signal>,) -> Result<Self> {ffi::init();+ let options = &settled(options);let pq = hdr == Some(Signal::Pq);let path = path.as_ref();let fps = frame_rate.fps();@@ -1402,8 +1580,16 @@ impl Encoder {}}}+ // The index up front, so a player starts before the file is in.+ // Not for ProRes: moving it there writes the whole file again, gigabytes+ // a minute, for an intermediate no one streams.+ let header = if options.codec == VideoCodec::ProRes4444 {+ ffmpeg::Dictionary::new()+ } else {+ ffmpeg::dict! { "movflags" => "+faststart" }+ };output- .write_header_with(ffmpeg::dict! { "movflags" => "+faststart" })+ .write_header_with(header).map_err(|error| ffi::fail("write header", path, error))?;let stream_time_base = output.stream(0)@@ -2362,7 +2548,15 @@ mod tests {for codec in VideoCodec::ALL {for hardware in [false, true] {let names = codec.encoders(hardware);- if cfg!(target_os = "android") {+ // No hardware anywhere takes an alpha layer: those are the+ // CPU's on a phone too.+ if codec.keeps_alpha() {+ assert!(+ names.iter().all(|name| !Family::of(name).hardware()),+ "{codec:?}"+ );+ assert!(!names.is_empty(), "{codec:?}");+ } else if cfg!(target_os = "android") {assert!(names.iter().all(|name| name.ends_with("_mediacodec")));} else if cfg!(target_os = "ios") {assert!(names.iter().all(|name| name.ends_with("_videotoolbox")));@@ -2556,7 +2750,12 @@ mod tests {/// the bit depth asked for, and `hvc1` on HEVC.#[test]fn every_available_codec_writes_a_tagged_file() {- for codec in VideoCodec::ALL {+ // The alpha codecs have their own test below, in their own+ // containers.+ for codec in VideoCodec::ALL+ .into_iter()+ .filter(|codec| !codec.keeps_alpha())+ {if !codec.available() {eprintln!("{} not in the linked FFmpeg; skipped", codec.label());continue;@@ -2810,4 +3009,193 @@ mod tests {assert!(std::fs::metadata(&path).is_ok_and(|meta| meta.len() > 0));let _ = std::fs::remove_file(&path);}++ /// The alpha codecs say what they are and where they go, and settle+ /// what they cannot do rather than refusing it: ProRes is ten bits and+ /// video range, VP9 with alpha eight bits.+ #[test]+ fn the_alpha_codecs_name_their_containers() {+ for codec in [VideoCodec::H264, VideoCodec::Hevc, VideoCodec::Av1] {+ assert!(!codec.keeps_alpha());+ assert_eq!(codec.container(), "mp4");+ assert!(codec.rate_controlled());+ }+ assert!(VideoCodec::ProRes4444.keeps_alpha());+ assert_eq!(VideoCodec::ProRes4444.container(), "mov");+ assert!(!VideoCodec::ProRes4444.rate_controlled());+ assert!(VideoCodec::Vp9Alpha.keeps_alpha());+ assert_eq!(VideoCodec::Vp9Alpha.container(), "webm");+ assert_eq!(+ VideoCodec::parse("ProRes 4444"),+ Some(VideoCodec::ProRes4444)+ );+ assert_eq!(VideoCodec::parse("vp9_alpha"), Some(VideoCodec::Vp9Alpha));+ assert_eq!(VideoCodec::parse("vp9"), None, "plain VP9 is not this");+ assert!(!VideoCodec::Vp9Alpha.ten_bit_available());++ let asked = EncodeOptions {+ ten_bit: true,+ color_range: ColorRange::Full,+ ..EncodeOptions::default()+ };+ let prores = settled(&EncodeOptions {+ codec: VideoCodec::ProRes4444,+ ten_bit: false,+ ..asked.clone()+ });+ assert!(prores.ten_bit);+ assert_eq!(prores.color_range, ColorRange::Limited);+ let vp9 = settled(&EncodeOptions {+ codec: VideoCodec::Vp9Alpha,+ ..asked.clone()+ });+ assert!(!vp9.ten_bit);+ assert_eq!(vp9.color_range, ColorRange::Full);+ let h264 = settled(&asked);+ assert!(h264.ten_bit, "the ordinary codecs are left as asked");+ assert_eq!(h264.color_range, ColorRange::Full);+ }++ /// Three bands, opaque red, half-transparent green and clear, written+ /// in each alpha codec and read back: the alpha survives, and so does+ /// the colour where there is some to see. The decoder's side is the+ /// import of a Concat overlay back into a project - ProRes' alpha comes+ /// through its own decoder, VP9's through libvpx (#429). A codec whose+ /// encoder this FFmpeg does not link is skipped.+ #[test]+ fn an_alpha_codec_keeps_the_frames_alpha() {+ use crate::decode::{DecodeOptions, Decoder, FrameSource};+ const RED: [u8; 4] = [200, 40, 40, 255];+ const GREEN: [u8; 4] = [40, 200, 40, 128];+ const CLEAR: [u8; 4] = [0, 0, 0, 0];+ let mut frame = Frame::transparent(64, 64);+ for y in 0..64 {+ for x in 0..64 {+ let colour = match x {+ 0..32 => RED,+ 32..48 => GREEN,+ _ => CLEAR,+ };+ frame.set_pixel(x, y, colour);+ }+ }+ for codec in [VideoCodec::ProRes4444, VideoCodec::Vp9Alpha] {+ if !codec.available() {+ eprintln!("{} not in the linked FFmpeg; skipped", codec.label());+ continue;+ }+ assert!(+ codec.encodable(),+ "{} is linked and on the CPU",+ codec.label()+ );+ let path = std::env::temp_dir().join(format!(+ "concat-encode-alpha-{}-{}.{}",+ codec.name(),+ std::process::id(),+ codec.container()+ ));+ let options = EncodeOptions {+ codec,+ preset: "ultrafast".to_owned(),+ crf: 18,+ // Asked for ten bits, which VP9 with alpha settles to eight+ // rather than refusing.+ ten_bit: true,+ hardware: false,+ ..EncodeOptions::default()+ };+ let mut encoder = Encoder::create(&path, 64, 64, FrameRate::THIRTY, &options)+ .unwrap_or_else(|error| panic!("{}: {error}", codec.label()));+ assert!(!encoder.on_hardware());+ for _ in 0..6 {+ encoder.write_frame(&frame).expect("writes");+ }+ encoder.finish().expect("finishes");++ {+ let input = ffmpeg::format::input(&path).expect("opens");+ let stream = input+ .streams()+ .best(ffmpeg::media::Type::Video)+ .expect("video");+ let parameters = stream.parameters();+ // SAFETY: the parameters are live for as long as `input`+ // is, and the profile and tag are plain fields.+ let (profile, tag) = unsafe {+ (+ (*parameters.as_ptr()).profile,+ (*parameters.as_ptr()).codec_tag,+ )+ };+ match codec {+ VideoCodec::ProRes4444 => {+ assert_eq!(parameters.id(), ffmpeg::codec::Id::PRORES);+ assert_eq!(profile, ffmpeg::sys::AV_PROFILE_PRORES_4444, "4444");+ assert_eq!(tag, fourcc(*b"ap4h"), "a QuickTime ProRes 4444");+ }+ _ => {+ assert_eq!(parameters.id(), ffmpeg::codec::Id::VP9);+ assert_eq!(+ stream.metadata().get("alpha_mode"),+ Some("1"),+ "the track says it has an alpha layer"+ );+ }+ }+ }++ let mut decoder =+ Decoder::open(&path, &DecodeOptions::default().in_software()).expect("opens");+ let mut frames = Vec::new();+ while let Some(frame) = decoder.next_frame().expect("decodes") {+ frames.push(frame);+ }+ let _ = std::fs::remove_file(&path);+ assert_eq!(frames.len(), 6, "{}: every frame", codec.label());+ let back = &frames[3];+ let near = |at: (u32, u32), want: [u8; 4], alpha: u8, colour: u8| {+ let got = back.pixel(at.0, at.1).expect("inside");+ assert!(+ got[3].abs_diff(want[3]) <= alpha,+ "{}: alpha {} at {at:?} came back as {}",+ codec.label(),+ want[3],+ got[3]+ );+ if want[3] > 0 {+ for channel in 0..3 {+ assert!(+ got[channel].abs_diff(want[channel]) <= colour,+ "{}: {want:?} at {at:?} came back as {got:?}",+ codec.label()+ );+ }+ }+ };+ near((12, 32), RED, 2, 10);+ near((40, 32), GREEN, 6, 12);+ near((58, 32), CLEAR, 4, 0);+ }+ }++ /// HDR is refused in an alpha codec, which is written SDR alone.+ #[test]+ fn an_alpha_codec_is_not_written_hdr() {+ for codec in [VideoCodec::ProRes4444, VideoCodec::Vp9Alpha] {+ let path = std::env::temp_dir().join(format!(+ "concat-encode-alpha-hdr-{}.{}",+ codec.name(),+ codec.container()+ ));+ let options = EncodeOptions {+ codec,+ ..EncodeOptions::default()+ };+ let result =+ Encoder::create_hdr(&path, 64, 64, FrameRate::THIRTY, &options, Signal::Pq);+ assert!(result.is_err(), "{}", codec.label());+ assert!(!path.exists());+ }+ }}