Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(enc-ffmpeg): convert RGB to YUV with BT.709 on every software path

Richie McIlroy8cd4d98249ec
diff --git a/crates/enc-ffmpeg/src/video/colour.rs b/crates/enc-ffmpeg/src/video/colour.rs
new file mode 100644
index 000000000..fc08dc20e
--- /dev/null
+++ b/crates/enc-ffmpeg/src/video/colour.rs
@@ -0,0 +1,95 @@
+use ffmpeg::format;
+
+pub(crate) fn is_rgb(format: format::Pixel) -> bool {
+ format.descriptor().is_some_and(|descriptor| unsafe {
+ (*descriptor.as_ptr()).flags & ffmpeg::ffi::AV_PIX_FMT_FLAG_RGB as u64 != 0
+ })
+}
+
+/// Makes an RGB to YUV swscale context produce what every encoder in this
+/// crate tags its stream with: BT.709, limited range (swscale defaults to
+/// BT.601). RGB input is full range. Other conversions are left untouched.
+pub fn use_bt709_for_rgb_input(
+ converter: &mut ffmpeg::software::scaling::Context,
+ input: format::Pixel,
+ output: format::Pixel,
+) -> Result<(), ffmpeg::Error> {
+ if !is_rgb(input) || is_rgb(output) {
+ return Ok(());
+ }
+
+ let result = unsafe {
+ let coefficients = ffmpeg::ffi::sws_getCoefficients(ffmpeg::ffi::SWS_CS_ITU709);
+ ffmpeg::ffi::sws_setColorspaceDetails(
+ converter.as_mut_ptr(),
+ coefficients,
+ 1,
+ coefficients,
+ 0,
+ 0,
+ 1 << 16,
+ 1 << 16,
+ )
+ };
+ if result < 0 {
+ return Err(ffmpeg::Error::from(result));
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use ffmpeg::frame;
+
+ fn convert_red(output: format::Pixel, bt709: bool) -> frame::Video {
+ ffmpeg::init().unwrap();
+ let mut converter = ffmpeg::software::scaling::Context::get(
+ format::Pixel::RGBA,
+ 16,
+ 16,
+ output,
+ 16,
+ 16,
+ ffmpeg::software::scaling::flag::Flags::BICUBIC,
+ )
+ .unwrap();
+ if bt709 {
+ use_bt709_for_rgb_input(&mut converter, format::Pixel::RGBA, output).unwrap();
+ }
+ let mut rgba = frame::Video::new(format::Pixel::RGBA, 16, 16);
+ let stride = rgba.stride(0);
+ for row in 0..16 {
+ for col in 0..16 {
+ let i = row * stride + col * 4;
+ rgba.data_mut(0)[i..i + 4].copy_from_slice(&[220, 40, 40, 255]);
+ }
+ }
+ let mut out = frame::Video::new(output, 16, 16);
+ converter.run(&rgba, &mut out).unwrap();
+ out
+ }
+
+ #[test]
+ fn rgb_input_converts_with_bt709_limited_range() {
+ // (220,40,40) is Y 83.2, Cb 109.9, Cr 207.1 in BT.709 limited range
+ // (BT.601 would give Y 96, Cb 101).
+ let nv12 = convert_red(format::Pixel::NV12, true);
+ let (y, cb, cr) = (nv12.data(0)[0], nv12.data(1)[0], nv12.data(1)[1]);
+ assert!(y.abs_diff(83) <= 1 && cb.abs_diff(110) <= 1 && cr.abs_diff(207) <= 1);
+
+ // ProRes 4444's 10-bit planes: the same colour at 4x the scale.
+ let yuva = convert_red(format::Pixel::YUVA444P10LE, true);
+ let sample = |plane: usize| u16::from_le_bytes([yuva.data(plane)[0], yuva.data(plane)[1]]);
+ let [y, cb, cr] = [sample(0), sample(1), sample(2)].map(|v| f64::from(v) / 4.0);
+ assert!((y - 83.2).abs() <= 1.0, "Y {y}");
+ assert!((cb - 109.9).abs() <= 1.0, "Cb {cb}");
+ assert!((cr - 207.1).abs() <= 1.0, "Cr {cr}");
+ }
+
+ #[test]
+ fn swscale_defaults_to_bt601_without_it() {
+ let nv12 = convert_red(format::Pixel::NV12, false);
+ assert!(nv12.data(0)[0].abs_diff(96) <= 1);
+ }
+}
diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs
index 6ad7b80bf..eaeac3a98 100644
--- a/crates/enc-ffmpeg/src/video/h264.rs
+++ b/crates/enc-ffmpeg/src/video/h264.rs
@@ -618,42 +618,14 @@ pub(crate) fn open_video_encoder_with_flags(
)
}
-fn is_rgb(format: format::Pixel) -> bool {
- format.descriptor().is_some_and(|descriptor| unsafe {
- (*descriptor.as_ptr()).flags & ffmpeg::ffi::AV_PIX_FMT_FLAG_RGB as u64 != 0
- })
-}
-
fn configure_rgb_converter(
converter: &mut ffmpeg::software::scaling::Context,
input: format::Pixel,
output: format::Pixel,
) -> Result<(), H264EncoderError> {
- if !is_rgb(input) || is_rgb(output) {
- return Ok(());
- }
-
- // swscale defaults to BT.601 even though the encoder declares BT.709.
- // RGB input is full range; the encoded YUV contract below is limited range.
- let result = unsafe {
- let coefficients = ffmpeg::ffi::sws_getCoefficients(ffmpeg::ffi::SWS_CS_ITU709);
- ffmpeg::ffi::sws_setColorspaceDetails(
- converter.as_mut_ptr(),
- coefficients,
- 1,
- coefficients,
- 0,
- 0,
- 1 << 16,
- 1 << 16,
- )
- };
- if result < 0 {
- return Err(H264EncoderError::ColorConversion(ffmpeg::Error::from(
- result,
- )));
- }
- Ok(())
+ // The encoded YUV contract below is limited range.
+ super::colour::use_bt709_for_rgb_input(converter, input, output)
+ .map_err(H264EncoderError::ColorConversion)
}
#[allow(clippy::too_many_arguments)]
@@ -1798,6 +1770,7 @@ mod self_test_tests {
#[cfg(test)]
mod rgb_color_tests {
+ use super::super::colour::is_rgb;
use super::*;
const COLORS: [[u8; 3]; 6] = [
diff --git a/crates/enc-ffmpeg/src/video/hevc.rs b/crates/enc-ffmpeg/src/video/hevc.rs
index 37b555592..fc9d2495f 100644
--- a/crates/enc-ffmpeg/src/video/hevc.rs
+++ b/crates/enc-ffmpeg/src/video/hevc.rs
@@ -214,7 +214,15 @@ impl HevcEncoderBuilder {
output_height,
flags,
) {
- Ok(context) => Some(context),
+ Ok(mut context) => {
+ super::colour::use_bt709_for_rgb_input(
+ &mut context,
+ input_config.pixel_format,
+ output_format,
+ )
+ .map_err(HevcEncoderError::FFmpeg)?;
+ Some(context)
+ }
Err(e) => {
if needs_pixel_conversion {
error!(
diff --git a/crates/enc-ffmpeg/src/video/mod.rs b/crates/enc-ffmpeg/src/video/mod.rs
index fcb5be5f2..104261fc0 100644
--- a/crates/enc-ffmpeg/src/video/mod.rs
+++ b/crates/enc-ffmpeg/src/video/mod.rs
@@ -1,3 +1,4 @@
+pub mod colour;
pub mod h264;
pub mod h264_packet;
pub mod hevc;
diff --git a/crates/enc-ffmpeg/src/video/prores.rs b/crates/enc-ffmpeg/src/video/prores.rs
index f88ccb70c..6c1279608 100644
--- a/crates/enc-ffmpeg/src/video/prores.rs
+++ b/crates/enc-ffmpeg/src/video/prores.rs
@@ -57,7 +57,7 @@ impl ProResEncoderBuilder {
|| input_config.width != output_width
|| input_config.height != output_height
{
- Some(ffmpeg::software::scaling::Context::get(
+ let mut converter = ffmpeg::software::scaling::Context::get(
input_config.pixel_format,
input_config.width,
input_config.height,
@@ -65,7 +65,13 @@ impl ProResEncoderBuilder {
output_width,
output_height,
ffmpeg::software::scaling::flag::Flags::BICUBIC,
- )?)
+ )?;
+ super::colour::use_bt709_for_rgb_input(
+ &mut converter,
+ input_config.pixel_format,
+ output_format,
+ )?;
+ Some(converter)
} else {
None
};
@@ -83,7 +89,9 @@ impl ProResEncoderBuilder {
encoder.set_time_base(input_config.time_base);
encoder.set_frame_rate(Some(input_config.frame_rate));
encoder.set_colorspace(color::Space::BT709);
- encoder.set_color_range(color::Range::JPEG);
+ // Limited range: what the converter above produces (and the ProRes
+ // convention).
+ encoder.set_color_range(color::Range::MPEG);
unsafe {
(*encoder.as_mut_ptr()).color_primaries =
ffmpeg::ffi::AVColorPrimaries::AVCOL_PRI_BT709;
diff --git a/crates/render-farm/src/video.rs b/crates/render-farm/src/video.rs
index 90af08f74..ec24742ea 100644
--- a/crates/render-farm/src/video.rs
+++ b/crates/render-farm/src/video.rs
@@ -592,15 +592,19 @@ fn encode(
}
let converter = match &mut rgba_converter {
Some(converter) => converter,
- None => rgba_converter.insert(ffmpeg::software::scaling::Context::get(
- Pixel::RGBA,
- width,
- height,
- Pixel::NV12,
- width,
- height,
- ffmpeg::software::scaling::flag::Flags::BILINEAR,
- )?),
+ None => {
+ let mut converter = ffmpeg::software::scaling::Context::get(
+ Pixel::RGBA,
+ width,
+ height,
+ Pixel::NV12,
+ width,
+ height,
+ ffmpeg::software::scaling::flag::Flags::BILINEAR,
+ )?;
+ use_bt709(&mut converter)?;
+ rgba_converter.insert(converter)
+ }
};
converter.run(&rgba, &mut nv12)?;
} else {
@@ -626,6 +630,27 @@ fn encode(
})
}
+/// swscale converts RGB with BT.601 unless told otherwise; the stream is
+/// tagged BT.709 limited range (see the encoder setup above).
+fn use_bt709(converter: &mut ffmpeg::software::scaling::Context) -> Result<(), ffmpeg::Error> {
+ let result = unsafe {
+ let coefficients = ffmpeg::ffi::sws_getCoefficients(ffmpeg::ffi::SWS_CS_ITU709);
+ ffmpeg::ffi::sws_setColorspaceDetails(
+ converter.as_mut_ptr(),
+ coefficients,
+ 1,
+ coefficients,
+ 0,
+ 0,
+ 1 << 16,
+ 1 << 16,
+ )
+ };
+ if result < 0 {
+ return Err(ffmpeg::Error::from(result));
+ }
+ Ok(())
+}
fn cap_enc_ffmpeg_keyframe_secs() -> u32 {
2
}