Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(enc-ffmpeg): convert RGB to YUV with BT.709 on every software path
Richie McIlroy8cd4d98249ec
diff --git a/crates/enc-ffmpeg/src/video/colour.rs b/crates/enc-ffmpeg/src/video/colour.rsnew file mode 100644index 000000000..fc08dc20e--- /dev/null+++ b/crates/enc-ffmpeg/src/video/colour.rs@@ -0,0 +1,95 @@+use ffmpeg::format;++pub(crate) fn is_rgb(format: format::Pixel) -> bool {+ format.descriptor().is_some_and(|descriptor| unsafe {+ (*descriptor.as_ptr()).flags & ffmpeg::ffi::AV_PIX_FMT_FLAG_RGB as u64 != 0+ })+}++/// Makes an RGB to YUV swscale context produce what every encoder in this+/// crate tags its stream with: BT.709, limited range (swscale defaults to+/// BT.601). RGB input is full range. Other conversions are left untouched.+pub fn use_bt709_for_rgb_input(+ converter: &mut ffmpeg::software::scaling::Context,+ input: format::Pixel,+ output: format::Pixel,+) -> Result<(), ffmpeg::Error> {+ if !is_rgb(input) || is_rgb(output) {+ return Ok(());+ }++ let result = unsafe {+ let coefficients = ffmpeg::ffi::sws_getCoefficients(ffmpeg::ffi::SWS_CS_ITU709);+ ffmpeg::ffi::sws_setColorspaceDetails(+ converter.as_mut_ptr(),+ coefficients,+ 1,+ coefficients,+ 0,+ 0,+ 1 << 16,+ 1 << 16,+ )+ };+ if result < 0 {+ return Err(ffmpeg::Error::from(result));+ }+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;+ use ffmpeg::frame;++ fn convert_red(output: format::Pixel, bt709: bool) -> frame::Video {+ ffmpeg::init().unwrap();+ let mut converter = ffmpeg::software::scaling::Context::get(+ format::Pixel::RGBA,+ 16,+ 16,+ output,+ 16,+ 16,+ ffmpeg::software::scaling::flag::Flags::BICUBIC,+ )+ .unwrap();+ if bt709 {+ use_bt709_for_rgb_input(&mut converter, format::Pixel::RGBA, output).unwrap();+ }+ let mut rgba = frame::Video::new(format::Pixel::RGBA, 16, 16);+ let stride = rgba.stride(0);+ for row in 0..16 {+ for col in 0..16 {+ let i = row * stride + col * 4;+ rgba.data_mut(0)[i..i + 4].copy_from_slice(&[220, 40, 40, 255]);+ }+ }+ let mut out = frame::Video::new(output, 16, 16);+ converter.run(&rgba, &mut out).unwrap();+ out+ }++ #[test]+ fn rgb_input_converts_with_bt709_limited_range() {+ // (220,40,40) is Y 83.2, Cb 109.9, Cr 207.1 in BT.709 limited range+ // (BT.601 would give Y 96, Cb 101).+ let nv12 = convert_red(format::Pixel::NV12, true);+ let (y, cb, cr) = (nv12.data(0)[0], nv12.data(1)[0], nv12.data(1)[1]);+ assert!(y.abs_diff(83) <= 1 && cb.abs_diff(110) <= 1 && cr.abs_diff(207) <= 1);++ // ProRes 4444's 10-bit planes: the same colour at 4x the scale.+ let yuva = convert_red(format::Pixel::YUVA444P10LE, true);+ let sample = |plane: usize| u16::from_le_bytes([yuva.data(plane)[0], yuva.data(plane)[1]]);+ let [y, cb, cr] = [sample(0), sample(1), sample(2)].map(|v| f64::from(v) / 4.0);+ assert!((y - 83.2).abs() <= 1.0, "Y {y}");+ assert!((cb - 109.9).abs() <= 1.0, "Cb {cb}");+ assert!((cr - 207.1).abs() <= 1.0, "Cr {cr}");+ }++ #[test]+ fn swscale_defaults_to_bt601_without_it() {+ let nv12 = convert_red(format::Pixel::NV12, false);+ assert!(nv12.data(0)[0].abs_diff(96) <= 1);+ }+}diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rsindex 6ad7b80bf..eaeac3a98 100644--- a/crates/enc-ffmpeg/src/video/h264.rs+++ b/crates/enc-ffmpeg/src/video/h264.rs@@ -618,42 +618,14 @@ pub(crate) fn open_video_encoder_with_flags()}-fn is_rgb(format: format::Pixel) -> bool {- format.descriptor().is_some_and(|descriptor| unsafe {- (*descriptor.as_ptr()).flags & ffmpeg::ffi::AV_PIX_FMT_FLAG_RGB as u64 != 0- })-}-fn configure_rgb_converter(converter: &mut ffmpeg::software::scaling::Context,input: format::Pixel,output: format::Pixel,) -> Result<(), H264EncoderError> {- if !is_rgb(input) || is_rgb(output) {- return Ok(());- }-- // swscale defaults to BT.601 even though the encoder declares BT.709.- // RGB input is full range; the encoded YUV contract below is limited range.- let result = unsafe {- let coefficients = ffmpeg::ffi::sws_getCoefficients(ffmpeg::ffi::SWS_CS_ITU709);- ffmpeg::ffi::sws_setColorspaceDetails(- converter.as_mut_ptr(),- coefficients,- 1,- coefficients,- 0,- 0,- 1 << 16,- 1 << 16,- )- };- if result < 0 {- return Err(H264EncoderError::ColorConversion(ffmpeg::Error::from(- result,- )));- }- Ok(())+ // The encoded YUV contract below is limited range.+ super::colour::use_bt709_for_rgb_input(converter, input, output)+ .map_err(H264EncoderError::ColorConversion)}#[allow(clippy::too_many_arguments)]@@ -1798,6 +1770,7 @@ mod self_test_tests {#[cfg(test)]mod rgb_color_tests {+ use super::super::colour::is_rgb;use super::*;const COLORS: [[u8; 3]; 6] = [diff --git a/crates/enc-ffmpeg/src/video/hevc.rs b/crates/enc-ffmpeg/src/video/hevc.rsindex 37b555592..fc9d2495f 100644--- a/crates/enc-ffmpeg/src/video/hevc.rs+++ b/crates/enc-ffmpeg/src/video/hevc.rs@@ -214,7 +214,15 @@ impl HevcEncoderBuilder {output_height,flags,) {- Ok(context) => Some(context),+ Ok(mut context) => {+ super::colour::use_bt709_for_rgb_input(+ &mut context,+ input_config.pixel_format,+ output_format,+ )+ .map_err(HevcEncoderError::FFmpeg)?;+ Some(context)+ }Err(e) => {if needs_pixel_conversion {error!(diff --git a/crates/enc-ffmpeg/src/video/mod.rs b/crates/enc-ffmpeg/src/video/mod.rsindex fcb5be5f2..104261fc0 100644--- a/crates/enc-ffmpeg/src/video/mod.rs+++ b/crates/enc-ffmpeg/src/video/mod.rs@@ -1,3 +1,4 @@+pub mod colour;pub mod h264;pub mod h264_packet;pub mod hevc;diff --git a/crates/enc-ffmpeg/src/video/prores.rs b/crates/enc-ffmpeg/src/video/prores.rsindex f88ccb70c..6c1279608 100644--- a/crates/enc-ffmpeg/src/video/prores.rs+++ b/crates/enc-ffmpeg/src/video/prores.rs@@ -57,7 +57,7 @@ impl ProResEncoderBuilder {|| input_config.width != output_width|| input_config.height != output_height{- Some(ffmpeg::software::scaling::Context::get(+ let mut converter = ffmpeg::software::scaling::Context::get(input_config.pixel_format,input_config.width,input_config.height,@@ -65,7 +65,13 @@ impl ProResEncoderBuilder {output_width,output_height,ffmpeg::software::scaling::flag::Flags::BICUBIC,- )?)+ )?;+ super::colour::use_bt709_for_rgb_input(+ &mut converter,+ input_config.pixel_format,+ output_format,+ )?;+ Some(converter)} else {None};@@ -83,7 +89,9 @@ impl ProResEncoderBuilder {encoder.set_time_base(input_config.time_base);encoder.set_frame_rate(Some(input_config.frame_rate));encoder.set_colorspace(color::Space::BT709);- encoder.set_color_range(color::Range::JPEG);+ // Limited range: what the converter above produces (and the ProRes+ // convention).+ encoder.set_color_range(color::Range::MPEG);unsafe {(*encoder.as_mut_ptr()).color_primaries =ffmpeg::ffi::AVColorPrimaries::AVCOL_PRI_BT709;diff --git a/crates/render-farm/src/video.rs b/crates/render-farm/src/video.rsindex 90af08f74..ec24742ea 100644--- a/crates/render-farm/src/video.rs+++ b/crates/render-farm/src/video.rs@@ -592,15 +592,19 @@ fn encode(}let converter = match &mut rgba_converter {Some(converter) => converter,- None => rgba_converter.insert(ffmpeg::software::scaling::Context::get(- Pixel::RGBA,- width,- height,- Pixel::NV12,- width,- height,- ffmpeg::software::scaling::flag::Flags::BILINEAR,- )?),+ None => {+ let mut converter = ffmpeg::software::scaling::Context::get(+ Pixel::RGBA,+ width,+ height,+ Pixel::NV12,+ width,+ height,+ ffmpeg::software::scaling::flag::Flags::BILINEAR,+ )?;+ use_bt709(&mut converter)?;+ rgba_converter.insert(converter)+ }};converter.run(&rgba, &mut nv12)?;} else {@@ -626,6 +630,27 @@ fn encode(})}+/// swscale converts RGB with BT.601 unless told otherwise; the stream is+/// tagged BT.709 limited range (see the encoder setup above).+fn use_bt709(converter: &mut ffmpeg::software::scaling::Context) -> Result<(), ffmpeg::Error> {+ let result = unsafe {+ let coefficients = ffmpeg::ffi::sws_getCoefficients(ffmpeg::ffi::SWS_CS_ITU709);+ ffmpeg::ffi::sws_setColorspaceDetails(+ converter.as_mut_ptr(),+ coefficients,+ 1,+ coefficients,+ 0,+ 0,+ 1 << 16,+ 1 << 16,+ )+ };+ if result < 0 {+ return Err(ffmpeg::Error::from(result));+ }+ Ok(())+}fn cap_enc_ffmpeg_keyframe_secs() -> u32 {2}