Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(export): ProRes 4444 and VP9 alpha codecs that keep a transparent frame's alpha (#456)

Jareer23bc68998cfb
diff --git a/src/crates/concat-export/src/lib.rs b/src/crates/concat-export/src/lib.rs
index 111c6b8a..cc12b831 100644
--- a/src/crates/concat-export/src/lib.rs
+++ b/src/crates/concat-export/src/lib.rs
@@ -433,7 +433,9 @@ pub struct ExportRequest {
pub crf: u8,
/// The x264 speed/size preset name, e.g. "medium".
pub preset: String,
- /// What to encode to, by name: "h264", "hevc" or "av1". H.264 when a
+ /// What to encode to, by name: "h264", "hevc", "av1", or "prores-4444"
+ /// or "vp9-alpha" to keep the frame's alpha, whose `output` must then
+ /// be a .mov or a .webm (see [`VideoCodec::container`]). H.264 when a
/// request does not say.
#[serde(default, deserialize_with = "codec_by_name")]
pub codec: VideoCodec,
@@ -509,7 +511,9 @@ fn codec_by_name<'de, D: serde::Deserializer<'de>>(
) -> Result<VideoCodec, D::Error> {
let name = String::deserialize(deserializer)?;
VideoCodec::parse(&name).ok_or_else(|| {
- serde::de::Error::custom(format!("unknown codec {name:?}: h264, hevc or av1"))
+ serde::de::Error::custom(format!(
+ "unknown codec {name:?}: h264, hevc, av1, prores-4444 or vp9-alpha"
+ ))
})
}
@@ -1168,6 +1172,24 @@ pub fn render_on(
));
}
let output = PathBuf::from(&request.output);
+ // An alpha codec's file is only what it says in its own container: a
+ // ProRes picture in a file named .mp4 opens nowhere, and a WebM's
+ // alpha means nothing to a reader that takes the name for an MP4. The
+ // three ordinary codecs keep writing an MP4 under whatever name they
+ // are given, as they always have.
+ if request.codec.keeps_alpha() {
+ let wanted = request.codec.container();
+ let named = output
+ .extension()
+ .is_some_and(|extension| extension.eq_ignore_ascii_case(wanted));
+ if !named {
+ return Err(format!(
+ "{} is written to a .{wanted} file, and {} is not one",
+ request.codec.label(),
+ output.display()
+ ));
+ }
+ }
// Transitions become overlaps, ramps and fade filters before anything
// else reads the clip list, so the picture and sound paths below never
@@ -1231,9 +1253,15 @@ pub fn render_on(
directory.display()
));
}
- let silent = directory.join(format!(".{stem}.concat-video.mp4"));
- let mixed = directory.join(format!(".{stem}.concat-audio.m4a"));
- let muxed = directory.join(format!(".{stem}.concat-muxed.mp4"));
+ // Each in the container the codec is written in, which the muxers pick
+ // by the extension: a ProRes picture goes in a QuickTime file, and a
+ // WebM's sound is Opus, the one a WebM takes besides Vorbis. A .mov
+ // takes the AAC an MP4 does.
+ let container = request.codec.container();
+ let sound_container = if container == "webm" { "webm" } else { "m4a" };
+ let silent = directory.join(format!(".{stem}.concat-video.{container}"));
+ let mixed = directory.join(format!(".{stem}.concat-audio.{sound_container}"));
+ let muxed = directory.join(format!(".{stem}.concat-muxed.{container}"));
// The output path is touched once, by the rename at the end, whichever
// way the file was made. The mux used to write it directly, and a mux
@@ -1751,7 +1779,10 @@ fn render_picture(
// An HDR file comes out of the compositor as its signal, sixteen bits a
// channel; anything else, SDR - an HDR timeline's tone-mapped.
let signal = output_of(request.color_space);
- let hdr = request.hdr && signal != Signal::Sdr;
+ // The alpha codecs are written SDR alone (see `Encoder::create_hdr`):
+ // an HDR timeline is tone-mapped into one, as it is on a phone that
+ // cannot write ten bits, rather than the export failing.
+ let hdr = request.hdr && signal != Signal::Sdr && !request.codec.keeps_alpha();
compositor.deliver_hdr(hdr);
// How many devices have died under this export so far; see `recovered`.
@@ -3668,6 +3699,133 @@ mod tests {
result.expect("the export goes on on a fresh device");
}
+ /// An export in an alpha codec writes its own container with its sound
+ /// in it - a .mov of ProRes and AAC, a .webm of VP9 and Opus - and one
+ /// named for another container is refused before a frame is drawn.
+ #[test]
+ fn an_alpha_export_writes_its_own_container_with_its_sound() {
+ use concat_core::frame::Frame;
+ use concat_media::{EncodeOptions, Encoder, FrameSink};
+
+ if WgpuCompositor::new().is_none() {
+ eprintln!("nothing to draw on here; skipped");
+ return;
+ }
+ let dir = std::env::temp_dir().join(format!("concat-export-alpha-{}", std::process::id()));
+ let _ = std::fs::create_dir_all(&dir);
+ let source = dir.join("source.mp4");
+ let Ok(mut encoder) = Encoder::create(
+ &source,
+ 64,
+ 64,
+ FrameRate::THIRTY,
+ &EncodeOptions::default(),
+ ) else {
+ return; // no ffmpeg here
+ };
+ for _ in 0..15 {
+ encoder.write_frame(&Frame::black(64, 64)).expect("writes");
+ }
+ encoder.finish().expect("finishes");
+ // Half a second of a 440 Hz tone, 16-bit stereo at 48 kHz.
+ let tone = dir.join("tone.wav");
+ {
+ let samples: Vec<u8> = (0..24_000)
+ .flat_map(|index| {
+ let t = f64::from(index) / 48_000.0;
+ let value = (0.5 * (std::f64::consts::TAU * 440.0 * t).sin() * 32767.0) as i16;
+ let [low, high] = value.to_le_bytes();
+ [low, high, low, high]
+ })
+ .collect();
+ let mut bytes = Vec::new();
+ bytes.extend_from_slice(b"RIFF");
+ bytes.extend_from_slice(&(36 + samples.len() as u32).to_le_bytes());
+ bytes.extend_from_slice(b"WAVEfmt ");
+ for (value, width) in [
+ (16u32, 4),
+ (1, 2),
+ (2, 2),
+ (48_000, 4),
+ (192_000, 4),
+ (4, 2),
+ (16, 2),
+ ] {
+ bytes.extend_from_slice(&value.to_le_bytes()[..width]);
+ }
+ bytes.extend_from_slice(b"data");
+ bytes.extend_from_slice(&(samples.len() as u32).to_le_bytes());
+ bytes.extend_from_slice(&samples);
+ std::fs::write(&tone, bytes).expect("writes the tone");
+ }
+
+ let mut footage = clip("video", 0, 0.0, 0.5, 0.0);
+ footage.path = source.to_string_lossy().into_owned();
+ footage.media_width = Some(64);
+ footage.media_height = Some(64);
+ footage.has_audio = Some(false);
+ let mut sound = clip("audio", 1, 0.0, 0.5, 0.0);
+ sound.path = tone.to_string_lossy().into_owned();
+ sound.has_audio = Some(true);
+ let request = |codec: VideoCodec, output: &Path| ExportRequest {
+ canvas: [0, 0, 0],
+ output: output.to_string_lossy().into_owned(),
+ width: 64,
+ height: 64,
+ rate_num: 30,
+ rate_den: 1,
+ crf: 30,
+ preset: "ultrafast".into(),
+ codec,
+ ten_bit: false,
+ rate_mode: RateMode::Vbr,
+ bitrate_kbps: 0,
+ color_range: concat_media::ColorRange::Limited,
+ color_space: ColorSpace::Sdr,
+ hdr: false,
+ effect_time_offset: 0.0,
+ clips: vec![footage.clone(), sound.clone()],
+ };
+ let render = |request: &ExportRequest| {
+ let cancel = AtomicBool::new(false);
+ let mut progress = |_: i64, _: i64, _: &'static str| {};
+ let reporter = Reporter {
+ progress: &mut progress,
+ cancel: &cancel,
+ };
+ render_on(request, None, reporter)
+ };
+
+ for (codec, picture, heard) in [
+ (VideoCodec::ProRes4444, "prores", "aac"),
+ (VideoCodec::Vp9Alpha, "vp9", "opus"),
+ ] {
+ if !codec.available() {
+ eprintln!("{} not in the linked FFmpeg; skipped", codec.label());
+ continue;
+ }
+ let misnamed = dir.join("overlay.mp4");
+ let error = render(&request(codec, &misnamed)).expect_err("an .mp4 is refused");
+ assert!(error.contains(codec.container()), "{error}");
+ assert!(!misnamed.exists());
+
+ let output = dir.join(format!("overlay.{}", codec.container()));
+ let written = render(&request(codec, &output))
+ .unwrap_or_else(|error| panic!("{}: {error}", codec.label()));
+ assert_eq!(Path::new(&written), output);
+ let info = concat_media::probe(&output).expect("probes");
+ assert_eq!(info.video.expect("a picture").codec, picture);
+ assert_eq!(info.audio.expect("a sound").codec, heard);
+ let leftovers: Vec<_> = std::fs::read_dir(&dir)
+ .expect("lists")
+ .filter_map(|entry| entry.ok())
+ .filter(|entry| entry.file_name().to_string_lossy().contains(".concat-"))
+ .collect();
+ assert!(leftovers.is_empty(), "scratch files are removed");
+ }
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
/// A device that dies under an export is replaced: first by a fresh
/// one on the GPU, then by the software adapter; the third death ends
/// the export with a message that says how many there were.
diff --git a/src/crates/concat-media/src/audio.rs b/src/crates/concat-media/src/audio.rs
index 01641e4f..50245cca 100644
--- a/src/crates/concat-media/src/audio.rs
+++ b/src/crates/concat-media/src/audio.rs
@@ -542,8 +542,27 @@ impl MixInput {
}
}
+/// The Opus encoder to mix into and the samples it takes: libopus where it
+/// is linked, which the desktops' builds and Homebrew carry, and FFmpeg's
+/// own otherwise, which every build has - a phone's included. A WebM takes
+/// Opus or Vorbis and nothing else, and Opus is the better of the two.
+fn opus_encoder() -> Result<(&'static str, Sample)> {
+ use ffmpeg::util::format::sample::Type;
+ if encoder::find_by_name("libopus").is_some() {
+ return Ok(("libopus", Sample::F32(Type::Packed)));
+ }
+ if encoder::find_by_name("opus").is_some() {
+ return Ok(("opus", Sample::F32(Type::Planar)));
+ }
+ Err(Error::Missing {
+ what: "encoder",
+ name: "opus".to_owned(),
+ })
+}
+
/// Mixes `clips` into one audio file at `destination`: 16-bit PCM for a
-/// `.wav` destination (case-insensitive), AAC otherwise, at 48 kHz stereo.
+/// `.wav` destination (case-insensitive), Opus for a `.webm`, `.opus`,
+/// `.ogg` or `.mka` one, AAC otherwise, at 48 kHz stereo.
///
/// Every clip's file must contain an audio stream; see [`mix_graph`].
pub fn mix_to_file(clips: &[AudioClip], duration: f64, destination: &Path) -> Result<()> {
@@ -557,18 +576,22 @@ pub fn mix_to_file_with(
duration: f64,
destination: &Path,
) -> Result<()> {
- let wav = destination
+ let extension = destination
.extension()
- .is_some_and(|extension| extension.eq_ignore_ascii_case("wav"));
+ .map(|extension| extension.to_string_lossy().to_ascii_lowercase());
+ let wav = extension.as_deref() == Some("wav");
+ let opus = matches!(extension.as_deref(), Some("webm" | "opus" | "ogg" | "mka"));
+ ffi::init();
let (codec_name, mix_format) = if wav {
(
"pcm_s16le",
Sample::I16(ffmpeg::util::format::sample::Type::Packed),
)
+ } else if opus {
+ opus_encoder()?
} else {
("aac", MIX_FORMAT)
};
- ffi::init();
// A bad chain is refused before any file is opened.
for chain in clips
.iter()
@@ -603,6 +626,11 @@ pub fn mix_to_file_with(
if global_header {
audio.set_flags(ffmpeg::codec::Flags::GLOBAL_HEADER);
}
+ // FFmpeg's own Opus encoder is still marked experimental, and opens
+ // only when told that is all right; libopus needs no such word.
+ if codec_name == "opus" {
+ audio.compliance(ffmpeg::codec::Compliance::Experimental);
+ }
let mut encoder = audio
.open()
.map_err(|error| ffi::fail("open encoder", destination, error))?;
@@ -1000,6 +1028,12 @@ pub fn mux(video: &Path, audio: &Path, output: &Path) -> Result<()> {
.map_err(|error| ffi::fail("add stream", output, error))?;
stream.set_parameters(video_stream.parameters());
stream.set_time_base(video_tb);
+ // A WebM's alpha layer is in the packets, but the track only says
+ // it has one in its header: without the word a reader takes the
+ // picture for opaque and never opens the decoder that reads it.
+ if let Some(mode) = video_stream.metadata().get("alpha_mode") {
+ stream.metadata_mut().set("alpha_mode", mode);
+ }
}
{
let mut stream = out
@@ -1008,7 +1042,15 @@ pub fn mux(video: &Path, audio: &Path, output: &Path) -> Result<()> {
stream.set_parameters(audio_stream.parameters());
stream.set_time_base(audio_tb);
}
- out.write_header_with(ffmpeg::dict! { "movflags" => "+faststart" })
+ // The index up front, as the picture's own file had it - but not for
+ // ProRes, where moving it writes gigabytes a minute again for a file
+ // an editor reads whole.
+ let header = if video_stream.parameters().id() == ffmpeg::codec::Id::PRORES {
+ ffmpeg::Dictionary::new()
+ } else {
+ ffmpeg::dict! { "movflags" => "+faststart" }
+ };
+ out.write_header_with(header)
.map_err(|error| ffi::fail("write header", output, error))?;
let out_video_tb = out
.stream(0)
@@ -1701,6 +1743,103 @@ mod tests {
std::fs::remove_dir_all(dir).expect("removes test files");
}
+ /// An overlay with alpha and its sound, joined the way an export joins
+ /// them: a ProRes picture with AAC in a .mov, a VP9 one with Opus in a
+ /// .webm, the only sound a WebM takes besides Vorbis. The joined WebM
+ /// still says it has an alpha layer, so the import reads it with the
+ /// decoder that keeps it.
+ #[test]
+ fn an_overlay_with_alpha_takes_its_sound_in_its_own_container() {
+ use crate::decode::{DecodeOptions, Decoder, FrameSource};
+ use crate::{AudioDecoder, AudioOptions, EncodeOptions, Encoder, FrameSink, VideoCodec};
+ use concat_core::frame::Frame;
+ use concat_core::time::FrameRate;
+
+ let dir = std::env::temp_dir().join(format!("concat-alpha-mux-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).expect("creates a folder");
+ let tone = dir.join("tone.wav");
+ tone_wav(&tone, 1.0);
+ for codec in [VideoCodec::ProRes4444, VideoCodec::Vp9Alpha] {
+ if !codec.available() {
+ eprintln!("{} not in the linked FFmpeg; skipped", codec.label());
+ continue;
+ }
+ let container = codec.container();
+ let webm = container == "webm";
+ let video = dir.join(format!("picture.{container}"));
+ let sound = dir.join(if webm { "sound.webm" } else { "sound.m4a" });
+ let output = dir.join(format!("joined.{container}"));
+
+ // A second of a frame clear on its right half.
+ let mut frame = Frame::transparent(64, 64);
+ for y in 0..64 {
+ for x in 0..32 {
+ frame.set_pixel(x, y, [200, 40, 40, 255]);
+ }
+ }
+ let options = EncodeOptions {
+ codec,
+ preset: "ultrafast".to_owned(),
+ hardware: false,
+ ..EncodeOptions::default()
+ };
+ let mut encoder =
+ Encoder::create(&video, 64, 64, FrameRate::THIRTY, &options).expect("opens");
+ for _ in 0..30 {
+ encoder.write_frame(&frame).expect("writes");
+ }
+ encoder.finish().expect("finishes");
+ let piece = AudioClip {
+ path: tone.clone(),
+ duration: 1.0,
+ ..clip("")
+ };
+ mix_to_file(&[piece], 1.0, &sound).expect("mixes");
+ mux(&video, &sound, &output).expect("muxes");
+
+ let counts = packets_of(&output);
+ assert_eq!(counts[0], 30, "{}: every picture packet", codec.label());
+ {
+ let input = ffmpeg::format::input(&output).expect("opens");
+ let picture = input
+ .streams()
+ .best(ffmpeg::media::Type::Video)
+ .expect("video");
+ if webm {
+ assert_eq!(
+ picture.metadata().get("alpha_mode"),
+ Some("1"),
+ "the joined track still says it has alpha"
+ );
+ }
+ let heard = input
+ .streams()
+ .best(ffmpeg::media::Type::Audio)
+ .expect("audio");
+ let wanted = if webm {
+ ffmpeg::codec::Id::OPUS
+ } else {
+ ffmpeg::codec::Id::AAC
+ };
+ assert_eq!(heard.parameters().id(), wanted, "{}", codec.label());
+ }
+ let mut decoder =
+ Decoder::open(&output, &DecodeOptions::default().in_software()).expect("opens");
+ let back = decoder.next_frame().expect("decodes").expect("a frame");
+ let [.., opaque] = back.pixel(16, 32).expect("inside");
+ let [.., clear] = back.pixel(48, 32).expect("inside");
+ assert!(opaque > 250, "{}: opaque came back {opaque}", codec.label());
+ assert!(clear < 8, "{}: clear came back {clear}", codec.label());
+ let mut sound = AudioDecoder::open(&output, &AudioOptions::default()).expect("opens");
+ let mut audible = false;
+ while let Some(samples) = sound.next_i16().expect("decodes") {
+ audible |= samples.iter().any(|sample| i32::from(*sample).abs() > 1000);
+ }
+ assert!(audible, "{}: the tone is heard", codec.label());
+ }
+ std::fs::remove_dir_all(dir).expect("removes test files");
+ }
+
#[test]
fn saving_to_m4a_keeps_aac_encoding_and_audible_samples() {
use crate::{AudioDecoder, AudioOptions};
diff --git a/src/crates/concat-media/src/decode.rs b/src/crates/concat-media/src/decode.rs
index 61619e47..b08225b5 100644
--- a/src/crates/concat-media/src/decode.rs
+++ b/src/crates/concat-media/src/decode.rs
@@ -1280,6 +1280,24 @@ impl Decoder {
self.hardware = None;
}
}
+ // FFmpeg's ProRes decoder reads the picture's matrix into the
+ // stream and leaves every frame it makes untagged, and swscale
+ // then reads BT.601 into what is a BT.709 picture: a ProRes
+ // 4444 overlay's reds came back orange. Its frames are given
+ // the stream's matrix where they say none. Only ProRes: the
+ // other decoders tag their frames as their streams are tagged.
+ if self.decoder.id() == ffmpeg::codec::Id::PRORES
+ && self.color.matrix != ffmpeg::color::Space::Unspecified
+ {
+ // SAFETY: a frame this reader owns; the matrix is a plain
+ // field the filtergraph reads with the picture.
+ unsafe {
+ let raw = frame.as_mut_ptr();
+ if (*raw).colorspace == ffmpeg::sys::AVColorSpace::UNSPECIFIED {
+ (*raw).colorspace = self.color.matrix.into();
+ }
+ }
+ }
self.discard_before = None;
self.discard_through = None;
self.last_pts = pts;
diff --git a/src/crates/concat-media/src/encode.rs b/src/crates/concat-media/src/encode.rs
index db84e25f..6c8843ca 100644
--- a/src/crates/concat-media/src/encode.rs
+++ b/src/crates/concat-media/src/encode.rs
@@ -47,11 +47,26 @@ pub enum VideoCodec {
/// AV1: smaller again, royalty-free, the web's; slower to encode in
/// software and not every old device plays it.
Av1,
+ /// Apple ProRes 4444 in a `.mov`, with its alpha: the file other
+ /// editors - Premiere, Resolve, Final Cut, CapCut on a desktop - take
+ /// a transparent overlay in. Ten bits, its quality set by the profile
+ /// rather than a CRF, and very large: about forty times H.264.
+ ProRes4444,
+ /// VP9 with an alpha layer in a `.webm`: a transparent overlay for a
+ /// web page or OBS, a small fraction of ProRes' size. Eight bits.
+ Vp9Alpha,
}
impl VideoCodec {
- /// Every codec, in the order a menu lists them.
- pub const ALL: [VideoCodec; 3] = [VideoCodec::H264, VideoCodec::Hevc, VideoCodec::Av1];
+ /// Every codec, in the order a menu lists them: the three that make an
+ /// ordinary `.mp4`, then the two that keep a transparent frame's alpha.
+ pub const ALL: [VideoCodec; 5] = [
+ VideoCodec::H264,
+ VideoCodec::Hevc,
+ VideoCodec::Av1,
+ VideoCodec::ProRes4444,
+ VideoCodec::Vp9Alpha,
+ ];
/// The name a document or a request stores.
pub fn name(self) -> &'static str {
@@ -59,15 +74,23 @@ impl VideoCodec {
VideoCodec::H264 => "h264",
VideoCodec::Hevc => "hevc",
VideoCodec::Av1 => "av1",
+ VideoCodec::ProRes4444 => "prores-4444",
+ VideoCodec::Vp9Alpha => "vp9-alpha",
}
}
/// The codec a stored name means, or `None` for one nobody stores.
+ /// Plain "vp9" is not taken for the alpha one: a VP9 file without alpha
+ /// is a different request, and this one writes a second layer.
pub fn parse(name: &str) -> Option<Self> {
match name.trim().to_ascii_lowercase().as_str() {
"h264" | "h.264" | "avc" => Some(VideoCodec::H264),
"hevc" | "h265" | "h.265" => Some(VideoCodec::Hevc),
"av1" => Some(VideoCodec::Av1),
+ "prores-4444" | "prores_4444" | "prores4444" | "prores 4444" | "prores" => {
+ Some(VideoCodec::ProRes4444)
+ }
+ "vp9-alpha" | "vp9_alpha" | "vp9alpha" | "vp9 alpha" => Some(VideoCodec::Vp9Alpha),
_ => None,
}
}
@@ -78,9 +101,37 @@ impl VideoCodec {
VideoCodec::H264 => "H.264",
VideoCodec::Hevc => "HEVC",
VideoCodec::Av1 => "AV1",
+ VideoCodec::ProRes4444 => "ProRes 4444",
+ VideoCodec::Vp9Alpha => "VP9 alpha",
+ }
+ }
+
+ /// Whether a file in it keeps the frame's alpha: a transparent
+ /// background stays transparent in the file, where the other three
+ /// write every pixel opaque.
+ pub fn keeps_alpha(self) -> bool {
+ matches!(self, VideoCodec::ProRes4444 | VideoCodec::Vp9Alpha)
+ }
+
+ /// The container a file in it is written in, as the file's extension:
+ /// "mp4" for H.264, HEVC and AV1, "mov" for ProRes and "webm" for VP9
+ /// with alpha. An MP4 has no place for either alpha layer: ProRes is
+ /// QuickTime's, and VP9's alpha rides in a Matroska block addition.
+ pub fn container(self) -> &'static str {
+ match self {
+ VideoCodec::H264 | VideoCodec::Hevc | VideoCodec::Av1 => "mp4",
+ VideoCodec::ProRes4444 => "mov",
+ VideoCodec::Vp9Alpha => "webm",
}
}
+ /// Whether the codec is steered by a CRF or a bitrate at all. ProRes is
+ /// not: its profile is the quality, and an export's rate mode, bitrate
+ /// and CRF are left unused.
+ pub fn rate_controlled(self) -> bool {
+ self != VideoCodec::ProRes4444
+ }
+
/// The FFmpeg encoders that make this codec, best first; an export
/// takes the first that is linked and opens.
///
@@ -89,6 +140,12 @@ impl VideoCodec {
/// encoders (scripts/ffmpeg-mobile.sh), and its cores are no match for
/// its chip anyway.
///
+ /// The alpha codecs have one encoder each, on the CPU everywhere:
+ /// FFmpeg's own ProRes encoder, which every build carries, a phone's
+ /// too, and libvpx for VP9, which the desktops' GPL builds and Homebrew
+ /// link and a phone's LGPL build does not. No hardware encoder here
+ /// takes an alpha layer.
+ ///
/// On a desktop the software encoders lead, and the machine's own
/// comes after them for an FFmpeg built without them. With `hardware`,
/// the GPU's leads for HEVC and AV1: VideoToolbox on macOS, and on
@@ -101,6 +158,11 @@ impl VideoCodec {
/// compatibility, not speed. Media Foundation, Windows' own, is last:
/// it runs on whatever the machine has, and says little about how.
pub fn encoders(self, hardware: bool) -> &'static [&'static str] {
+ match self {
+ VideoCodec::ProRes4444 => return &["prores_ks"],
+ VideoCodec::Vp9Alpha => return &["libvpx-vp9"],
+ VideoCodec::H264 | VideoCodec::Hevc | VideoCodec::Av1 => {}
+ }
let macos = cfg!(target_os = "macos");
let windows = cfg!(target_os = "windows");
if cfg!(target_os = "android") {
@@ -108,13 +170,14 @@ impl VideoCodec {
VideoCodec::H264 => &["h264_mediacodec"],
VideoCodec::Hevc => &["hevc_mediacodec"],
VideoCodec::Av1 => &["av1_mediacodec"],
+ VideoCodec::ProRes4444 | VideoCodec::Vp9Alpha => &[],
};
}
if cfg!(target_os = "ios") {
return match self {
VideoCodec::H264 => &["h264_videotoolbox"],
VideoCodec::Hevc => &["hevc_videotoolbox"],
- VideoCodec::Av1 => &[],
+ VideoCodec::Av1 | VideoCodec::ProRes4444 | VideoCodec::Vp9Alpha => &[],
};
}
match self {
@@ -138,6 +201,7 @@ impl VideoCodec {
&["av1_nvenc", "av1_qsv", "av1_amf", "libsvtav1", "libaom-av1"]
}
VideoCodec::Av1 => &["libsvtav1", "libaom-av1", "av1_nvenc", "av1_qsv", "av1_amf"],
+ VideoCodec::ProRes4444 | VideoCodec::Vp9Alpha => &[],
}
}
@@ -169,7 +233,8 @@ impl VideoCodec {
/// [`encodable_among`] for the rest. A hardware encoder that is asked
/// is opened once, at 720p, and the answer kept for the process.
pub fn encodable(self) -> bool {
- static KNOWN: [OnceLock<bool>; 3] = [const { OnceLock::new() }; 3];
+ static KNOWN: [OnceLock<bool>; VideoCodec::ALL.len()] =
+ [const { OnceLock::new() }; VideoCodec::ALL.len()];
let slot = VideoCodec::ALL
.iter()
.position(|codec| *codec == self)
@@ -181,7 +246,8 @@ impl VideoCodec {
/// Whether it can be written at ten bits here: some linked encoder for
/// it takes ten-bit pictures. Not on a phone, whose encoders FFmpeg
- /// feeds eight bits only.
+ /// feeds eight bits only. ProRes is ten bits whether asked or not; VP9
+ /// with alpha is eight, as libvpx takes its alpha layer at eight only.
pub fn ten_bit_available(self) -> bool {
self.linked(true, true).next().is_some()
}
@@ -203,12 +269,17 @@ impl VideoCodec {
}
/// Bytes per second relative to H.264 at the same quality, for a size
- /// estimate: the rule of thumb the codecs are chosen by.
+ /// estimate: the rule of thumb the codecs are chosen by. ProRes is not
+ /// steered by a rate at all - its profile comes to about 330 Mb/s for
+ /// a 1080p30 4444, some forty times H.264 - and VP9's alpha layer adds
+ /// about a third to its picture.
pub fn size_factor(self) -> f32 {
match self {
VideoCodec::H264 => 1.0,
VideoCodec::Hevc => 0.6,
VideoCodec::Av1 => 0.5,
+ VideoCodec::ProRes4444 => 40.0,
+ VideoCodec::Vp9Alpha => 0.8,
}
}
}
@@ -334,6 +405,28 @@ fn svt_preset(preset: &str) -> u8 {
.map_or(6, |index| SVT[index])
}
+/// libvpx's `cpu-used` for an x264 preset, at its `good` deadline: 0 is
+/// its slowest and 5 its fastest there, and x264's medium lands on 3.
+fn vpx_speed(preset: &str) -> u8 {
+ const VPX: [u8; 10] = [5, 5, 5, 4, 4, 3, 2, 1, 0, 0];
+ X264_PRESETS
+ .iter()
+ .position(|name| *name == preset)
+ .map_or(3, |index| VPX[index])
+}
+
+/// The picture an encoder of an alpha codec takes, when `encoder_name` is
+/// one: ProRes 4444's ten-bit 4:4:4 with its alpha plane, and libvpx's
+/// 4:2:0 with one, the only alpha layout it reads. swscale fills the alpha
+/// plane from the RGBA frame's own alpha on the way.
+fn alpha_pixel_format(encoder_name: &str) -> Option<Pixel> {
+ match encoder_name {
+ "prores_ks" => Some(Pixel::YUVA444P10LE),
+ "libvpx-vp9" => Some(Pixel::YUVA420P),
+ _ => None,
+ }
+}
+
/// VideoToolbox's quality, 1 to 100 with 100 the best, for an x264 CRF:
/// 16 lands at 65 and 26 at 43, which is where the files come out about
/// the size x264 makes them.
@@ -345,7 +438,9 @@ fn videotoolbox_quality(crf: u8) -> u8 {
/// in, and what it is steered by.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum Family {
- /// x264, x265, SVT-AV1, libaom: planar 4:2:0, steered by a CRF.
+ /// x264, x265, SVT-AV1, libaom: planar 4:2:0, steered by a CRF. ProRes
+ /// and libvpx too, with pictures of their own that carry the alpha
+ /// (see [`alpha_pixel_format`]).
Software,
/// Apple's: semi-planar 4:2:0, steered by a quality.
VideoToolbox,
@@ -565,6 +660,10 @@ fn opens_here(codec: VideoCodec, encoder_name: &'static str) -> bool {
/// and AMF make ten-bit HEVC but not ten-bit H.264, and NVENC and Quick
/// Sync make ten-bit HEVC and AV1.
fn takes_ten_bits(encoder_name: &str) -> bool {
+ // libvpx's alpha layer is eight bits only.
+ if encoder_name == "libvpx-vp9" {
+ return false;
+ }
match Family::of(encoder_name) {
Family::Software => true,
Family::VideoToolbox | Family::Amf => encoder_name.starts_with("hevc_"),
@@ -690,9 +789,12 @@ fn open_video(
let pq = hdr == Some(Signal::Pq);
let family = Family::of(encoder_name);
let plain = attempt != Attempt::AsAsked;
- let pixel_format = family.pixel_format(options.ten_bit);
- let cbr = options.rate_mode == RateMode::Cbr && options.bitrate_kbps > 0;
- let average = options.rate_mode == RateMode::Average && options.bitrate_kbps > 0;
+ let pixel_format =
+ alpha_pixel_format(encoder_name).unwrap_or_else(|| family.pixel_format(options.ten_bit));
+ // ProRes has no rate to steer: its profile is the quality.
+ let rated = options.codec.rate_controlled();
+ let cbr = rated && options.rate_mode == RateMode::Cbr && options.bitrate_kbps > 0;
+ let average = rated && options.rate_mode == RateMode::Average && options.bitrate_kbps > 0;
let mut video = ffmpeg::codec::Context::new_with_codec(codec)
.encoder()
.video()
@@ -856,6 +958,49 @@ fn open_video(
"crf" => &options.crf.saturating_add(8).min(63).to_string(),
"cpu-used" => "6",
},
+ // ProRes 4444 is the profile FFmpeg picks for a picture with alpha,
+ // so the profile is not named: the encoder's own `profile` shares a
+ // name with the context's, which would take the word first. The
+ // alpha at sixteen bits, the most the format keeps, and Apple's
+ // vendor code, which some of the editors that read it look for.
+ "prores_ks" => ffmpeg::dict! {
+ "alpha_bits" => "16",
+ "vendor" => "apl0",
+ },
+ // libvpx's alt-ref frames are pictures that are never shown, and
+ // its alpha layer is encoded beside the colour frame by frame, so
+ // they are off. Row threading, as every core is wanted.
+ "libvpx-vp9" if cbr => ffmpeg::dict! {
+ "deadline" => "good",
+ "cpu-used" => &vpx_speed(&options.preset).to_string(),
+ "row-mt" => "1",
+ "auto-alt-ref" => "0",
+ "b" => bitrate.as_str(),
+ "minrate" => bitrate.as_str(),
+ "maxrate" => bitrate.as_str(),
+ "bufsize" => bufsize.as_str(),
+ },
+ "libvpx-vp9" if average => ffmpeg::dict! {
+ "deadline" => "good",
+ "cpu-used" => &vpx_speed(&options.preset).to_string(),
+ "row-mt" => "1",
+ "auto-alt-ref" => "0",
+ "b" => bitrate.as_str(),
+ "maxrate" => peak.as_str(),
+ "bufsize" => window.as_str(),
+ },
+ // Constant quality: a CRF with no bitrate beside it, which libvpx
+ // otherwise reads as a ceiling. VP9's scale runs to 63 like AV1's,
+ // and about twelve on from x264's is where the pictures match -
+ // 18 lands at 30, Google's own suggestion for 1080p.
+ "libvpx-vp9" => ffmpeg::dict! {
+ "deadline" => "good",
+ "cpu-used" => &vpx_speed(&options.preset).to_string(),
+ "row-mt" => "1",
+ "auto-alt-ref" => "0",
+ "crf" => &options.crf.saturating_add(12).min(63).to_string(),
+ "b" => "0",
+ },
// VideoToolbox steers by the context's bitrate when it is given no
// quality.
"hevc_videotoolbox" | "h264_videotoolbox" if average => ffmpeg::dict! {
@@ -997,6 +1142,25 @@ fn open_video(
Ok((encoder, pixel_format))
}
+/// `options` as the codec can take them. ProRes is ten bits and video
+/// range whatever is asked: its pictures are ten bits or more, and its
+/// stream has no word for full range, so a full-range file would be read
+/// back as video range by everything that opens it. VP9 with alpha is
+/// eight bits, the only depth libvpx takes an alpha layer at. The other
+/// codecs' options are left exactly as they were.
+fn settled(options: &EncodeOptions) -> EncodeOptions {
+ let mut options = options.clone();
+ match options.codec {
+ VideoCodec::ProRes4444 => {
+ options.ten_bit = true;
+ options.color_range = ColorRange::Limited;
+ }
+ VideoCodec::Vp9Alpha => options.ten_bit = false,
+ VideoCodec::H264 | VideoCodec::Hevc | VideoCodec::Av1 => {}
+ }
+ options
+}
+
/// A four-character code as the muxer stores it.
const fn fourcc(tag: [u8; 4]) -> u32 {
u32::from_le_bytes(tag)
@@ -1253,6 +1417,19 @@ impl Encoder {
detail: "H.264 does not carry HDR here: choose HEVC or AV1".to_owned(),
});
}
+ // The alpha codecs are written SDR alone. ProRes can carry HDR, but
+ // nothing here tags or checks it, and VP9's alpha layer is eight
+ // bits.
+ if options.codec.keeps_alpha() {
+ return Err(Error::Ffi {
+ operation: "open an HDR encoder",
+ path: path.to_path_buf(),
+ detail: format!(
+ "{} is written SDR here: choose HEVC or AV1 for HDR",
+ options.codec.label()
+ ),
+ });
+ }
let options = EncodeOptions {
ten_bit: true,
..options.clone()
@@ -1285,6 +1462,7 @@ impl Encoder {
hdr: Option<Signal>,
) -> Result<Self> {
ffi::init();
+ let options = &settled(options);
let pq = hdr == Some(Signal::Pq);
let path = path.as_ref();
let fps = frame_rate.fps();
@@ -1402,8 +1580,16 @@ impl Encoder {
}
}
}
+ // The index up front, so a player starts before the file is in.
+ // Not for ProRes: moving it there writes the whole file again, gigabytes
+ // a minute, for an intermediate no one streams.
+ let header = if options.codec == VideoCodec::ProRes4444 {
+ ffmpeg::Dictionary::new()
+ } else {
+ ffmpeg::dict! { "movflags" => "+faststart" }
+ };
output
- .write_header_with(ffmpeg::dict! { "movflags" => "+faststart" })
+ .write_header_with(header)
.map_err(|error| ffi::fail("write header", path, error))?;
let stream_time_base = output
.stream(0)
@@ -2362,7 +2548,15 @@ mod tests {
for codec in VideoCodec::ALL {
for hardware in [false, true] {
let names = codec.encoders(hardware);
- if cfg!(target_os = "android") {
+ // No hardware anywhere takes an alpha layer: those are the
+ // CPU's on a phone too.
+ if codec.keeps_alpha() {
+ assert!(
+ names.iter().all(|name| !Family::of(name).hardware()),
+ "{codec:?}"
+ );
+ assert!(!names.is_empty(), "{codec:?}");
+ } else if cfg!(target_os = "android") {
assert!(names.iter().all(|name| name.ends_with("_mediacodec")));
} else if cfg!(target_os = "ios") {
assert!(names.iter().all(|name| name.ends_with("_videotoolbox")));
@@ -2556,7 +2750,12 @@ mod tests {
/// the bit depth asked for, and `hvc1` on HEVC.
#[test]
fn every_available_codec_writes_a_tagged_file() {
- for codec in VideoCodec::ALL {
+ // The alpha codecs have their own test below, in their own
+ // containers.
+ for codec in VideoCodec::ALL
+ .into_iter()
+ .filter(|codec| !codec.keeps_alpha())
+ {
if !codec.available() {
eprintln!("{} not in the linked FFmpeg; skipped", codec.label());
continue;
@@ -2810,4 +3009,193 @@ mod tests {
assert!(std::fs::metadata(&path).is_ok_and(|meta| meta.len() > 0));
let _ = std::fs::remove_file(&path);
}
+
+ /// The alpha codecs say what they are and where they go, and settle
+ /// what they cannot do rather than refusing it: ProRes is ten bits and
+ /// video range, VP9 with alpha eight bits.
+ #[test]
+ fn the_alpha_codecs_name_their_containers() {
+ for codec in [VideoCodec::H264, VideoCodec::Hevc, VideoCodec::Av1] {
+ assert!(!codec.keeps_alpha());
+ assert_eq!(codec.container(), "mp4");
+ assert!(codec.rate_controlled());
+ }
+ assert!(VideoCodec::ProRes4444.keeps_alpha());
+ assert_eq!(VideoCodec::ProRes4444.container(), "mov");
+ assert!(!VideoCodec::ProRes4444.rate_controlled());
+ assert!(VideoCodec::Vp9Alpha.keeps_alpha());
+ assert_eq!(VideoCodec::Vp9Alpha.container(), "webm");
+ assert_eq!(
+ VideoCodec::parse("ProRes 4444"),
+ Some(VideoCodec::ProRes4444)
+ );
+ assert_eq!(VideoCodec::parse("vp9_alpha"), Some(VideoCodec::Vp9Alpha));
+ assert_eq!(VideoCodec::parse("vp9"), None, "plain VP9 is not this");
+ assert!(!VideoCodec::Vp9Alpha.ten_bit_available());
+
+ let asked = EncodeOptions {
+ ten_bit: true,
+ color_range: ColorRange::Full,
+ ..EncodeOptions::default()
+ };
+ let prores = settled(&EncodeOptions {
+ codec: VideoCodec::ProRes4444,
+ ten_bit: false,
+ ..asked.clone()
+ });
+ assert!(prores.ten_bit);
+ assert_eq!(prores.color_range, ColorRange::Limited);
+ let vp9 = settled(&EncodeOptions {
+ codec: VideoCodec::Vp9Alpha,
+ ..asked.clone()
+ });
+ assert!(!vp9.ten_bit);
+ assert_eq!(vp9.color_range, ColorRange::Full);
+ let h264 = settled(&asked);
+ assert!(h264.ten_bit, "the ordinary codecs are left as asked");
+ assert_eq!(h264.color_range, ColorRange::Full);
+ }
+
+ /// Three bands, opaque red, half-transparent green and clear, written
+ /// in each alpha codec and read back: the alpha survives, and so does
+ /// the colour where there is some to see. The decoder's side is the
+ /// import of a Concat overlay back into a project - ProRes' alpha comes
+ /// through its own decoder, VP9's through libvpx (#429). A codec whose
+ /// encoder this FFmpeg does not link is skipped.
+ #[test]
+ fn an_alpha_codec_keeps_the_frames_alpha() {
+ use crate::decode::{DecodeOptions, Decoder, FrameSource};
+ const RED: [u8; 4] = [200, 40, 40, 255];
+ const GREEN: [u8; 4] = [40, 200, 40, 128];
+ const CLEAR: [u8; 4] = [0, 0, 0, 0];
+ let mut frame = Frame::transparent(64, 64);
+ for y in 0..64 {
+ for x in 0..64 {
+ let colour = match x {
+ 0..32 => RED,
+ 32..48 => GREEN,
+ _ => CLEAR,
+ };
+ frame.set_pixel(x, y, colour);
+ }
+ }
+ for codec in [VideoCodec::ProRes4444, VideoCodec::Vp9Alpha] {
+ if !codec.available() {
+ eprintln!("{} not in the linked FFmpeg; skipped", codec.label());
+ continue;
+ }
+ assert!(
+ codec.encodable(),
+ "{} is linked and on the CPU",
+ codec.label()
+ );
+ let path = std::env::temp_dir().join(format!(
+ "concat-encode-alpha-{}-{}.{}",
+ codec.name(),
+ std::process::id(),
+ codec.container()
+ ));
+ let options = EncodeOptions {
+ codec,
+ preset: "ultrafast".to_owned(),
+ crf: 18,
+ // Asked for ten bits, which VP9 with alpha settles to eight
+ // rather than refusing.
+ ten_bit: true,
+ hardware: false,
+ ..EncodeOptions::default()
+ };
+ let mut encoder = Encoder::create(&path, 64, 64, FrameRate::THIRTY, &options)
+ .unwrap_or_else(|error| panic!("{}: {error}", codec.label()));
+ assert!(!encoder.on_hardware());
+ for _ in 0..6 {
+ encoder.write_frame(&frame).expect("writes");
+ }
+ encoder.finish().expect("finishes");
+
+ {
+ let input = ffmpeg::format::input(&path).expect("opens");
+ let stream = input
+ .streams()
+ .best(ffmpeg::media::Type::Video)
+ .expect("video");
+ let parameters = stream.parameters();
+ // SAFETY: the parameters are live for as long as `input`
+ // is, and the profile and tag are plain fields.
+ let (profile, tag) = unsafe {
+ (
+ (*parameters.as_ptr()).profile,
+ (*parameters.as_ptr()).codec_tag,
+ )
+ };
+ match codec {
+ VideoCodec::ProRes4444 => {
+ assert_eq!(parameters.id(), ffmpeg::codec::Id::PRORES);
+ assert_eq!(profile, ffmpeg::sys::AV_PROFILE_PRORES_4444, "4444");
+ assert_eq!(tag, fourcc(*b"ap4h"), "a QuickTime ProRes 4444");
+ }
+ _ => {
+ assert_eq!(parameters.id(), ffmpeg::codec::Id::VP9);
+ assert_eq!(
+ stream.metadata().get("alpha_mode"),
+ Some("1"),
+ "the track says it has an alpha layer"
+ );
+ }
+ }
+ }
+
+ let mut decoder =
+ Decoder::open(&path, &DecodeOptions::default().in_software()).expect("opens");
+ let mut frames = Vec::new();
+ while let Some(frame) = decoder.next_frame().expect("decodes") {
+ frames.push(frame);
+ }
+ let _ = std::fs::remove_file(&path);
+ assert_eq!(frames.len(), 6, "{}: every frame", codec.label());
+ let back = &frames[3];
+ let near = |at: (u32, u32), want: [u8; 4], alpha: u8, colour: u8| {
+ let got = back.pixel(at.0, at.1).expect("inside");
+ assert!(
+ got[3].abs_diff(want[3]) <= alpha,
+ "{}: alpha {} at {at:?} came back as {}",
+ codec.label(),
+ want[3],
+ got[3]
+ );
+ if want[3] > 0 {
+ for channel in 0..3 {
+ assert!(
+ got[channel].abs_diff(want[channel]) <= colour,
+ "{}: {want:?} at {at:?} came back as {got:?}",
+ codec.label()
+ );
+ }
+ }
+ };
+ near((12, 32), RED, 2, 10);
+ near((40, 32), GREEN, 6, 12);
+ near((58, 32), CLEAR, 4, 0);
+ }
+ }
+
+ /// HDR is refused in an alpha codec, which is written SDR alone.
+ #[test]
+ fn an_alpha_codec_is_not_written_hdr() {
+ for codec in [VideoCodec::ProRes4444, VideoCodec::Vp9Alpha] {
+ let path = std::env::temp_dir().join(format!(
+ "concat-encode-alpha-hdr-{}.{}",
+ codec.name(),
+ codec.container()
+ ));
+ let options = EncodeOptions {
+ codec,
+ ..EncodeOptions::default()
+ };
+ let result =
+ Encoder::create_hdr(&path, 64, 64, FrameRate::THIRTY, &options, Signal::Pq);
+ assert!(result.is_err(), "{}", codec.label());
+ assert!(!path.exists());
+ }
+ }
}