Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(rendering): encode exports in BT.709 and decode sources by their tag
Richie McIlroy755bfe426544
diff --git a/apps/desktop-gpui/src/editor_crop.rs b/apps/desktop-gpui/src/editor_crop.rsindex 412b8f20c..e9bfd7b82 100644--- a/apps/desktop-gpui/src/editor_crop.rs+++ b/apps/desktop-gpui/src/editor_crop.rs@@ -2280,7 +2280,7 @@ async fn decode_display_frame(let y = screen.y_plane()?;let uv = screen.uv_plane()?;let mut out = vec![0u8; (width * height * 4) as usize];- cpu_yuv::nv12_to_rgba(+ cpu_yuv::nv12_to_rgba_with_matrix(y,uv,width,@@ -2288,6 +2288,7 @@ async fn decode_display_frame(screen.y_stride(),screen.uv_stride(),&mut out,+ screen.matrix(),);out}@@ -2296,7 +2297,7 @@ async fn decode_display_frame(let u = screen.u_plane()?;let v = screen.v_plane()?;let mut out = vec![0u8; (width * height * 4) as usize];- cpu_yuv::yuv420p_to_rgba(+ cpu_yuv::yuv420p_to_rgba_with_matrix(y,u,v,@@ -2305,6 +2306,7 @@ async fn decode_display_frame(screen.y_stride(),screen.uv_stride(),&mut out,+ screen.matrix(),);out}diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rsindex f120f6cff..627faa3dc 100644--- a/crates/export/src/mp4.rs+++ b/crates/export/src/mp4.rs@@ -837,7 +837,13 @@ fn nv12_from_rendered_frame(frame: Nv12RenderedFrame) -> ExportFrame {width,height,ffmpeg::software::scaling::flag::Flags::FAST_BILINEAR,- ) {+ ) && cap_enc_ffmpeg::colour::use_bt709_for_rgb_input(+ &mut converter,+ ffmpeg::format::Pixel::RGBA,+ ffmpeg::format::Pixel::NV12,+ )+ .is_ok()+ {let mut nv12_frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::NV12, width, height);if converter.run(&rgba_frame, &mut nv12_frame).is_ok() {let y_size = nv12_frame.stride(0) * height as usize;@@ -1383,8 +1389,16 @@ fn save_screenshot_from_nv12(let (y_data, uv_data) = nv12_data.split_at(y_plane_size);let mut rgba = vec![0u8; (width * height * 4) as usize];+ // The export's own NV12, so BT.709 like the stream it is encoded into.cap_rendering::cpu_yuv::nv12_to_rgba_simd(- y_data, uv_data, width, height, y_stride, width, &mut rgba,+ y_data,+ uv_data,+ width,+ height,+ y_stride,+ width,+ &mut rgba,+ cap_rendering::cpu_yuv::YuvMatrix::Bt709,);drop(nv12_data);diff --git a/crates/ffmpeg-hw-device/src/lib.rs b/crates/ffmpeg-hw-device/src/lib.rsindex 023182e6a..8691bebe3 100644--- a/crates/ffmpeg-hw-device/src/lib.rs+++ b/crates/ffmpeg-hw-device/src/lib.rs@@ -63,6 +63,15 @@ impl HwDevice {if av_hwframe_transfer_data(sw_frame.as_mut_ptr(), src.as_ptr(), 0) >= 0 {sw_frame.set_pts(src.pts());+ // The transfer copies pixels only. Keep the colour tags so+ // consumers convert hardware-decoded frames with the same+ // matrix as software-decoded ones.+ let (dst, src) = (&mut *sw_frame.as_mut_ptr(), &*src.as_ptr());+ dst.color_range = src.color_range;+ dst.color_primaries = src.color_primaries;+ dst.color_trc = src.color_trc;+ dst.colorspace = src.colorspace;+ dst.chroma_location = src.chroma_location;return Some(sw_frame);};}diff --git a/crates/rendering/src/cpu_yuv.rs b/crates/rendering/src/cpu_yuv.rsindex 0207e4602..cd242b493 100644--- a/crates/rendering/src/cpu_yuv.rs+++ b/crates/rendering/src/cpu_yuv.rs@@ -1,6 +1,53 @@use std::sync::Arc;use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};+/// The Y'CbCr matrix a limited-range picture was encoded with, which decides+/// how it converts back to RGB. Decoders take it from the source's colour tag:+/// BT.709 when the stream says so, BT.601 otherwise (untagged sources keep the+/// BT.601 decode they always had).+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]+pub enum YuvMatrix {+ #[default]+ Bt601,+ Bt709,+}++impl YuvMatrix {+ pub fn from_color_space(space: ffmpeg::util::color::Space) -> Self {+ match space {+ ffmpeg::util::color::Space::BT709 => Self::Bt709,+ _ => Self::Bt601,+ }+ }++ /// Limited-range Y'CbCr to RGB in 8.8 fixed point: `c = Y - 16`,+ /// `d = Cb - 128`, `e = Cr - 128`. Luma gain 255/219, chroma gains from+ /// the matrix's Kr and Kb (BT.601 0.299/0.114, BT.709 0.2126/0.0722).+ #[inline(always)]+ pub(crate) fn ycbcr_to_rgb(self, c: i32, d: i32, e: i32) -> [u8; 3] {+ let (rv, gu, gv, bu) = match self {+ Self::Bt601 => (409, 100, 208, 516),+ Self::Bt709 => (459, 55, 136, 541),+ };+ [+ clamp_u8((298 * c + rv * e + 128) >> 8),+ clamp_u8((298 * c - gu * d - gv * e + 128) >> 8),+ clamp_u8((298 * c + bu * d + 128) >> 8),+ ]+ }++ /// Floating-point coefficients for the GPU conversion shaders (pipeline+ /// override constants of nv12_to_rgba.wgsl and yuv420p_to_rgba.wgsl):+ /// R = Y + rv*V, G = Y - gu*U - gv*V, B = Y + bu*U.+ pub(crate) fn shader_constants(self) -> [(&'static str, f64); 4] {+ let (rv, gu, gv, bu) = match self {+ Self::Bt601 => (1.596, 0.391, 0.813, 2.018),+ Self::Bt709 => (1.792741, 0.213249, 0.532909, 2.112402),+ };+ [("RV", rv), ("GU", gu), ("GV", gv), ("BU", bu)]+ }+}+pub struct ConversionProgress {pub rows_completed: AtomicUsize,pub total_rows: usize,@@ -32,6 +79,36 @@ impl ConversionProgress {}}+// RGB to limited-range BT.709 Y'CbCr in 16.16 fixed point, rounded to the+// nearest code value: the matrix and range every export is tagged with. The+// luma row sums to 219/255 and each chroma row to 0, so the grey axis is+// exact (black 16/128/128, white 235/128/128).+const BT709_Y: [i32; 3] = [11966, 40254, 4064];+const BT709_CB: [i32; 3] = [-6596, -22189, 28785];+const BT709_CR: [i32; 3] = [28785, -26145, -2640];++#[inline(always)]+fn fixed_16(row: [i32; 3], r: i32, g: i32, b: i32) -> i32 {+ (row[0] * r + row[1] * g + row[2] * b + (1 << 15)) >> 16+}++/// BT.709 limited-range luma of an 8-bit RGB pixel.+#[inline(always)]+pub(crate) fn bt709_luma(r: i32, g: i32, b: i32) -> u8 {+ (16 + fixed_16(BT709_Y, r, g, b)).clamp(16, 235) as u8+}++/// BT.709 limited-range (Cb, Cr) of an 8-bit RGB pixel (for 4:2:0, the mean+/// of the 2x2 block).+#[inline(always)]+pub(crate) fn bt709_chroma(r: i32, g: i32, b: i32) -> [u8; 2] {+ [+ (128 + fixed_16(BT709_CB, r, g, b)).clamp(16, 240) as u8,+ (128 + fixed_16(BT709_CR, r, g, b)).clamp(16, 240) as u8,+ ]+}++/// BT.601 [`nv12_to_rgba_with_matrix`], the conversion this module always did.pub fn nv12_to_rgba(y_data: &[u8],uv_data: &[u8],@@ -40,6 +117,55 @@ pub fn nv12_to_rgba(y_stride: u32,uv_stride: u32,output: &mut [u8],+) {+ nv12_to_rgba_with_matrix(+ y_data,+ uv_data,+ width,+ height,+ y_stride,+ uv_stride,+ output,+ YuvMatrix::Bt601,+ );+}++/// BT.601 [`yuv420p_to_rgba_with_matrix`], the conversion this module always+/// did.+#[allow(clippy::too_many_arguments)]+pub fn yuv420p_to_rgba(+ y_data: &[u8],+ u_data: &[u8],+ v_data: &[u8],+ width: u32,+ height: u32,+ y_stride: u32,+ uv_stride: u32,+ output: &mut [u8],+) {+ yuv420p_to_rgba_with_matrix(+ y_data,+ u_data,+ v_data,+ width,+ height,+ y_stride,+ uv_stride,+ output,+ YuvMatrix::Bt601,+ );+}++#[allow(clippy::too_many_arguments)]+pub fn nv12_to_rgba_with_matrix(+ y_data: &[u8],+ uv_data: &[u8],+ width: u32,+ height: u32,+ y_stride: u32,+ uv_stride: u32,+ output: &mut [u8],+ matrix: YuvMatrix,) {let width = width as usize;let height = height as usize;@@ -63,9 +189,7 @@ pub fn nv12_to_rgba(let d = u - 128;let e = v - 128;- let r = clamp_u8((298 * c + 409 * e + 128) >> 8);- let g = clamp_u8((298 * c - 100 * d - 208 * e + 128) >> 8);- let b = clamp_u8((298 * c + 516 * d + 128) >> 8);+ let [r, g, b] = matrix.ycbcr_to_rgb(c, d, e);let out_idx = out_row_start + col * 4;if out_idx + 3 < output.len() {@@ -79,7 +203,7 @@ pub fn nv12_to_rgba(}#[allow(clippy::too_many_arguments)]-pub fn yuv420p_to_rgba(+pub fn yuv420p_to_rgba_with_matrix(y_data: &[u8],u_data: &[u8],v_data: &[u8],@@ -88,6 +212,7 @@ pub fn yuv420p_to_rgba(y_stride: u32,uv_stride: u32,output: &mut [u8],+ matrix: YuvMatrix,) {let width = width as usize;let height = height as usize;@@ -111,9 +236,7 @@ pub fn yuv420p_to_rgba(let d = u - 128;let e = v - 128;- let r = clamp_u8((298 * c + 409 * e + 128) >> 8);- let g = clamp_u8((298 * c - 100 * d - 208 * e + 128) >> 8);- let b = clamp_u8((298 * c + 516 * d + 128) >> 8);+ let [r, g, b] = matrix.ycbcr_to_rgb(c, d, e);let out_idx = out_row_start + col * 4;if out_idx + 3 < output.len() {@@ -173,9 +296,10 @@ pub fn nv12_to_rgba_simd(y_stride: u32,uv_stride: u32,output: &mut [u8],+ matrix: YuvMatrix,) {nv12_to_rgba_simd_with_progress(- y_data, uv_data, width, height, y_stride, uv_stride, output, None,+ y_data, uv_data, width, height, y_stride, uv_stride, output, None, matrix,);}@@ -190,6 +314,7 @@ pub fn nv12_to_rgba_simd_with_progress(uv_stride: u32,output: &mut [u8],progress: Option<Arc<ConversionProgress>>,+ matrix: YuvMatrix,) {let width_usize = width as usize;let height_usize = height as usize;@@ -220,7 +345,9 @@ pub fn nv12_to_rgba_simd_with_progress(|| uv_data.len() < uv_required|| output.len() < output_required{- return nv12_to_rgba(y_data, uv_data, width, height, y_stride, uv_stride, output);+ return nv12_to_rgba_with_matrix(+ y_data, uv_data, width, height, y_stride, uv_stride, output, matrix,+ );}let simd_level = SimdLevel::detect();@@ -238,6 +365,7 @@ pub fn nv12_to_rgba_simd_with_progress(output,simd_level,progress,+ matrix,);} else {nv12_convert_sequential(@@ -250,6 +378,7 @@ pub fn nv12_to_rgba_simd_with_progress(output,simd_level,progress,+ matrix,);}}@@ -266,6 +395,7 @@ fn nv12_convert_sequential(output: &mut [u8],simd_level: SimdLevel,progress: Option<Arc<ConversionProgress>>,+ matrix: YuvMatrix,) {for row in 0..height {if let Some(ref p) = progress@@ -275,7 +405,7 @@ fn nv12_convert_sequential(}nv12_convert_row(- y_data, uv_data, width, row, y_stride, uv_stride, output, simd_level,+ y_data, uv_data, width, row, y_stride, uv_stride, output, simd_level, matrix,);if let Some(ref p) = progress {@@ -296,6 +426,7 @@ fn nv12_convert_parallel(output: &mut [u8],simd_level: SimdLevel,progress: Option<Arc<ConversionProgress>>,+ matrix: YuvMatrix,) {use rayon::prelude::*;@@ -332,6 +463,7 @@ fn nv12_convert_parallel(band_output,local_row,simd_level,+ matrix,);if let Some(ref p) = progress {@@ -352,9 +484,10 @@ fn nv12_convert_row(uv_stride: usize,output: &mut [u8],simd_level: SimdLevel,+ matrix: YuvMatrix,) {nv12_convert_row_into(- y_data, uv_data, width, row, y_stride, uv_stride, output, row, simd_level,+ y_data, uv_data, width, row, y_stride, uv_stride, output, row, simd_level, matrix,);}@@ -370,6 +503,7 @@ fn nv12_convert_row_into(output: &mut [u8],dst_row: usize,simd_level: SimdLevel,+ matrix: YuvMatrix,) {let y_row_start = src_row * y_stride;let uv_row_start = (src_row / 2) * uv_stride;@@ -385,6 +519,7 @@ fn nv12_convert_row_into(uv_row_start,out_row_start,output,+ matrix,);},SimdLevel::Sse2 => unsafe {@@ -396,6 +531,7 @@ fn nv12_convert_row_into(uv_row_start,out_row_start,output,+ matrix,);},SimdLevel::Scalar => {@@ -407,6 +543,7 @@ fn nv12_convert_row_into(uv_row_start,out_row_start,output,+ matrix,);}}@@ -422,6 +559,7 @@ unsafe fn nv12_convert_row_avx2(uv_row_start: usize,out_row_start: usize,output: &mut [u8],+ matrix: YuvMatrix,) {unsafe {nv12_convert_row_sse2(@@ -432,6 +570,7 @@ unsafe fn nv12_convert_row_avx2(uv_row_start,out_row_start,output,+ matrix,);}}@@ -446,6 +585,7 @@ unsafe fn nv12_convert_row_sse2(uv_row_start: usize,out_row_start: usize,output: &mut [u8],+ matrix: YuvMatrix,) {nv12_convert_row_scalar(y_data,@@ -455,6 +595,7 @@ unsafe fn nv12_convert_row_sse2(uv_row_start,out_row_start,output,+ matrix,);}@@ -467,6 +608,7 @@ fn nv12_convert_row_scalar(uv_row_start: usize,out_row_start: usize,output: &mut [u8],+ matrix: YuvMatrix,) {for col in 0..width {let y_idx = y_row_start + col;@@ -480,9 +622,7 @@ fn nv12_convert_row_scalar(let d = u - 128;let e = v - 128;- let r = clamp_u8((298 * c + 409 * e + 128) >> 8);- let g = clamp_u8((298 * c - 100 * d - 208 * e + 128) >> 8);- let b = clamp_u8((298 * c + 516 * d + 128) >> 8);+ let [r, g, b] = matrix.ycbcr_to_rgb(c, d, e);let out_idx = out_row_start + col * 4;if out_idx + 3 < output.len() {@@ -495,6 +635,7 @@ fn nv12_convert_row_scalar(}#[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))]+#[allow(clippy::too_many_arguments)]pub fn nv12_to_rgba_simd(y_data: &[u8],uv_data: &[u8],@@ -503,8 +644,11 @@ pub fn nv12_to_rgba_simd(y_stride: u32,uv_stride: u32,output: &mut [u8],+ matrix: YuvMatrix,) {- nv12_to_rgba(y_data, uv_data, width, height, y_stride, uv_stride, output);+ nv12_to_rgba_with_matrix(+ y_data, uv_data, width, height, y_stride, uv_stride, output, matrix,+ );}#[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))]@@ -518,8 +662,11 @@ pub fn nv12_to_rgba_simd_with_progress(uv_stride: u32,output: &mut [u8],_progress: Option<Arc<ConversionProgress>>,+ matrix: YuvMatrix,) {- nv12_to_rgba(y_data, uv_data, width, height, y_stride, uv_stride, output);+ nv12_to_rgba_with_matrix(+ y_data, uv_data, width, height, y_stride, uv_stride, output, matrix,+ );}#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]@@ -533,9 +680,10 @@ pub fn yuv420p_to_rgba_simd(y_stride: u32,uv_stride: u32,output: &mut [u8],+ matrix: YuvMatrix,) {yuv420p_to_rgba_simd_with_progress(- y_data, u_data, v_data, width, height, y_stride, uv_stride, output, None,+ y_data, u_data, v_data, width, height, y_stride, uv_stride, output, None, matrix,);}@@ -551,6 +699,7 @@ pub fn yuv420p_to_rgba_simd_with_progress(uv_stride: u32,output: &mut [u8],progress: Option<Arc<ConversionProgress>>,+ matrix: YuvMatrix,) {let width_usize = width as usize;let height_usize = height as usize;@@ -582,8 +731,8 @@ pub fn yuv420p_to_rgba_simd_with_progress(|| v_data.len() < uv_required|| output.len() < output_required{- return yuv420p_to_rgba(- y_data, u_data, v_data, width, height, y_stride, uv_stride, output,+ return yuv420p_to_rgba_with_matrix(+ y_data, u_data, v_data, width, height, y_stride, uv_stride, output, matrix,);}@@ -603,6 +752,7 @@ pub fn yuv420p_to_rgba_simd_with_progress(output,simd_level,progress,+ matrix,);} else {yuv420p_convert_sequential(@@ -616,6 +766,7 @@ pub fn yuv420p_to_rgba_simd_with_progress(output,simd_level,progress,+ matrix,);}}@@ -633,6 +784,7 @@ fn yuv420p_convert_sequential(output: &mut [u8],simd_level: SimdLevel,progress: Option<Arc<ConversionProgress>>,+ matrix: YuvMatrix,) {for row in 0..height {if let Some(ref p) = progress@@ -642,7 +794,7 @@ fn yuv420p_convert_sequential(}yuv420p_convert_row(- y_data, u_data, v_data, width, row, y_stride, uv_stride, output, simd_level,+ y_data, u_data, v_data, width, row, y_stride, uv_stride, output, simd_level, matrix,);if let Some(ref p) = progress {@@ -664,6 +816,7 @@ fn yuv420p_convert_parallel(output: &mut [u8],simd_level: SimdLevel,progress: Option<Arc<ConversionProgress>>,+ matrix: YuvMatrix,) {use rayon::prelude::*;@@ -701,6 +854,7 @@ fn yuv420p_convert_parallel(band_output,local_row,simd_level,+ matrix,);if let Some(ref p) = progress {@@ -722,9 +876,10 @@ fn yuv420p_convert_row(uv_stride: usize,output: &mut [u8],simd_level: SimdLevel,+ matrix: YuvMatrix,) {yuv420p_convert_row_into(- y_data, u_data, v_data, width, row, y_stride, uv_stride, output, row, simd_level,+ y_data, u_data, v_data, width, row, y_stride, uv_stride, output, row, simd_level, matrix,);}@@ -741,6 +896,7 @@ fn yuv420p_convert_row_into(output: &mut [u8],dst_row: usize,simd_level: SimdLevel,+ matrix: YuvMatrix,) {let y_row_start = src_row * y_stride;let uv_row_start = (src_row / 2) * uv_stride;@@ -757,6 +913,7 @@ fn yuv420p_convert_row_into(uv_row_start,out_row_start,output,+ matrix,);},SimdLevel::Sse2 => unsafe {@@ -769,6 +926,7 @@ fn yuv420p_convert_row_into(uv_row_start,out_row_start,output,+ matrix,);},SimdLevel::Scalar => {@@ -781,6 +939,7 @@ fn yuv420p_convert_row_into(uv_row_start,out_row_start,output,+ matrix,);}}@@ -798,6 +957,7 @@ unsafe fn yuv420p_convert_row_avx2(uv_row_start: usize,out_row_start: usize,output: &mut [u8],+ matrix: YuvMatrix,) {unsafe {yuv420p_convert_row_sse2(@@ -809,6 +969,7 @@ unsafe fn yuv420p_convert_row_avx2(uv_row_start,out_row_start,output,+ matrix,);}}@@ -825,6 +986,7 @@ unsafe fn yuv420p_convert_row_sse2(uv_row_start: usize,out_row_start: usize,output: &mut [u8],+ matrix: YuvMatrix,) {yuv420p_convert_row_scalar(y_data,@@ -835,6 +997,7 @@ unsafe fn yuv420p_convert_row_sse2(uv_row_start,out_row_start,output,+ matrix,);}@@ -849,6 +1012,7 @@ fn yuv420p_convert_row_scalar(uv_row_start: usize,out_row_start: usize,output: &mut [u8],+ matrix: YuvMatrix,) {for col in 0..width {let y_idx = y_row_start + col;@@ -862,9 +1026,7 @@ fn yuv420p_convert_row_scalar(let d = u - 128;let e = v - 128;- let r = clamp_u8((298 * c + 409 * e + 128) >> 8);- let g = clamp_u8((298 * c - 100 * d - 208 * e + 128) >> 8);- let b = clamp_u8((298 * c + 516 * d + 128) >> 8);+ let [r, g, b] = matrix.ycbcr_to_rgb(c, d, e);let out_idx = out_row_start + col * 4;if out_idx + 3 < output.len() {@@ -887,9 +1049,10 @@ pub fn yuv420p_to_rgba_simd(y_stride: u32,uv_stride: u32,output: &mut [u8],+ matrix: YuvMatrix,) {- yuv420p_to_rgba(- y_data, u_data, v_data, width, height, y_stride, uv_stride, output,+ yuv420p_to_rgba_with_matrix(+ y_data, u_data, v_data, width, height, y_stride, uv_stride, output, matrix,);}@@ -905,9 +1068,10 @@ pub fn yuv420p_to_rgba_simd_with_progress(uv_stride: u32,output: &mut [u8],_progress: Option<Arc<ConversionProgress>>,+ matrix: YuvMatrix,) {- yuv420p_to_rgba(- y_data, u_data, v_data, width, height, y_stride, uv_stride, output,+ yuv420p_to_rgba_with_matrix(+ y_data, u_data, v_data, width, height, y_stride, uv_stride, output, matrix,);}@@ -933,7 +1097,7 @@ mod tests {let uv_data: Vec<u8> = vec![128, 128, 128, 128, 128, 128, 128, 128];let mut output = vec![0u8; (width * height * 4) as usize];- nv12_to_rgba(+ nv12_to_rgba_with_matrix(&y_data,&uv_data,width,@@ -941,6 +1105,7 @@ mod tests {y_stride,uv_stride,&mut output,+ YuvMatrix::Bt601,);for pixel in output.chunks(4) {@@ -966,7 +1131,7 @@ mod tests {let mut output_scalar = vec![0u8; (width * height * 4) as usize];let mut output_simd = vec![0u8; (width * height * 4) as usize];- nv12_to_rgba(+ nv12_to_rgba_with_matrix(&y_data,&uv_data,width,@@ -974,6 +1139,7 @@ mod tests {y_stride,uv_stride,&mut output_scalar,+ YuvMatrix::Bt601,);nv12_to_rgba_simd(@@ -984,6 +1150,7 @@ mod tests {y_stride,uv_stride,&mut output_simd,+ YuvMatrix::Bt601,);for (i, (s, d)) in output_scalar.iter().zip(output_simd.iter()).enumerate() {@@ -1036,7 +1203,7 @@ mod tests {let mut output1 = vec![0u8; (width * height * 4) as usize];let mut output2 = vec![0u8; (width * height * 4) as usize];- nv12_to_rgba(+ nv12_to_rgba_with_matrix(&y_data,&uv_data,width,@@ -1044,6 +1211,7 @@ mod tests {y_stride,uv_stride,&mut output1,+ YuvMatrix::Bt601,);nv12_to_rgba_simd(@@ -1054,6 +1222,7 @@ mod tests {y_stride,uv_stride,&mut output2,+ YuvMatrix::Bt601,);for (i, (a, b)) in output1.iter().zip(output2.iter()).enumerate() {@@ -1085,7 +1254,7 @@ mod tests {let mut output_scalar = vec![0u8; (width * height * 4) as usize];let mut output_simd = vec![0u8; (width * height * 4) as usize];- yuv420p_to_rgba(+ yuv420p_to_rgba_with_matrix(&y_data,&u_data,&v_data,@@ -1094,6 +1263,7 @@ mod tests {y_stride,uv_stride,&mut output_scalar,+ YuvMatrix::Bt601,);yuv420p_to_rgba_simd(@@ -1105,6 +1275,7 @@ mod tests {y_stride,uv_stride,&mut output_simd,+ YuvMatrix::Bt601,);for (i, (s, d)) in output_scalar.iter().zip(output_simd.iter()).enumerate() {@@ -1138,6 +1309,7 @@ mod tests {y_stride,uv_stride,&mut output,+ YuvMatrix::Bt601,);assert!(output.iter().any(|&x| x != 0));@@ -1168,9 +1340,171 @@ mod tests {uv_stride,&mut output,Some(progress.clone()),+ YuvMatrix::Bt601,);let rows_done = progress.rows_completed.load(Ordering::Relaxed);assert!(rows_done < height as usize);}++ /// Exact limited-range Y'CbCr of an 8-bit RGB colour for (Kr, Kb).+ fn reference_ycbcr(rgb: [u8; 3], kr: f64, kb: f64) -> [f64; 3] {+ let [r, g, b] = rgb.map(|v| f64::from(v) / 255.0);+ let y = kr * r + (1.0 - kr - kb) * g + kb * b;+ [+ 16.0 + 219.0 * y,+ 128.0 + 224.0 * (b - y) / (2.0 * (1.0 - kb)),+ 128.0 + 224.0 * (r - y) / (2.0 * (1.0 - kr)),+ ]+ }++ fn encode_bt709(rgb: [u8; 3]) -> [u8; 3] {+ let [r, g, b] = rgb.map(i32::from);+ let [cb, cr] = bt709_chroma(r, g, b);+ [bt709_luma(r, g, b), cb, cr]+ }++ const BT709: (f64, f64) = (0.2126, 0.0722);++ #[test]+ fn bt709_encode_matches_the_colour_chart() {+ // (RGB, BT.709 limited-range Y'CbCr): primaries, secondaries, the+ // ends of the grey axis and the suite's red background.+ let chart: [([u8; 3], [u8; 3]); 9] = [+ ([0, 0, 0], [16, 128, 128]),+ ([255, 255, 255], [235, 128, 128]),+ ([255, 0, 0], [63, 102, 240]),+ ([0, 255, 0], [173, 42, 26]),+ ([0, 0, 255], [32, 240, 118]),+ ([0, 255, 255], [188, 154, 16]),+ ([255, 0, 255], [78, 214, 230]),+ ([255, 255, 0], [219, 16, 138]),+ ([220, 40, 40], [83, 110, 207]),+ ];+ for (rgb, expected) in chart {+ assert_eq!(encode_bt709(rgb), expected, "{rgb:?}");+ }+ }++ #[test]+ fn bt709_encode_rounds_to_the_nearest_code_value() {+ for r in (0..=255u8).step_by(5) {+ for g in (0..=255u8).step_by(5) {+ for b in (0..=255u8).step_by(5) {+ let reference = reference_ycbcr([r, g, b], BT709.0, BT709.1);+ let actual = encode_bt709([r, g, b]);+ for k in 0..3 {+ let error = (f64::from(actual[k]) - reference[k]).abs();+ assert!(+ error <= 0.5 + 1.0 / 256.0,+ "{:?} component {k}: {} vs {:.3}",+ [r, g, b],+ actual[k],+ reference[k]+ );+ }+ }+ }+ }+ }++ #[test]+ fn bt709_encode_keeps_the_grey_axis_neutral_and_exact() {+ for v in 0..=255u8 {+ let expected_y = (16.0 + 219.0 * f64::from(v) / 255.0).round() as u8;+ assert_eq!(encode_bt709([v, v, v]), [expected_y, 128, 128], "grey {v}");+ }+ }++ #[test]+ fn bt709_decode_inverts_the_bt709_encode() {+ let mut worst = 0u8;+ for r in (0..=255u8).step_by(5) {+ for g in (0..=255u8).step_by(5) {+ for b in (0..=255u8).step_by(5) {+ let [y, cb, cr] = encode_bt709([r, g, b]).map(i32::from);+ let decoded = YuvMatrix::Bt709.ycbcr_to_rgb(y - 16, cb - 128, cr - 128);+ for (a, e) in decoded.iter().zip([r, g, b]) {+ worst = worst.max(a.abs_diff(e));+ }+ }+ }+ }+ assert!(worst <= 2, "BT.709 round trip off by {worst}");+ }++ #[test]+ fn decoding_with_the_other_matrix_shifts_saturated_colours() {+ // The bug this guards: BT.709 data read back as BT.601 moves the+ // suite's red background by more than the round-trip error.+ let [y, cb, cr] = encode_bt709([220, 40, 40]).map(i32::from);+ let wrong = YuvMatrix::Bt601.ycbcr_to_rgb(y - 16, cb - 128, cr - 128);+ let right = YuvMatrix::Bt709.ycbcr_to_rgb(y - 16, cb - 128, cr - 128);+ assert!(+ right+ .iter()+ .zip([220u8, 40, 40])+ .all(|(a, e)| a.abs_diff(e) <= 1)+ );+ assert!(+ wrong+ .iter()+ .zip([220u8, 40, 40])+ .any(|(a, e)| a.abs_diff(e) > 10)+ );+ }++ #[test]+ fn bt601_decode_is_unchanged() {+ // The integer BT.601 conversion this module always used.+ let legacy = |c: i32, d: i32, e: i32| {+ [+ clamp_u8((298 * c + 409 * e + 128) >> 8),+ clamp_u8((298 * c - 100 * d - 208 * e + 128) >> 8),+ clamp_u8((298 * c + 516 * d + 128) >> 8),+ ]+ };+ for y in (0..=255).step_by(3) {+ for u in (0..=255).step_by(3) {+ for v in (0..=255).step_by(3) {+ let (c, d, e) = (y - 16, u - 128, v - 128);+ assert_eq!(YuvMatrix::Bt601.ycbcr_to_rgb(c, d, e), legacy(c, d, e));+ }+ }+ }+ }++ #[test]+ fn matrix_follows_the_colour_tag() {+ use ffmpeg::util::color::Space;+ assert_eq!(YuvMatrix::from_color_space(Space::BT709), YuvMatrix::Bt709);+ for space in [+ Space::Unspecified,+ Space::BT470BG,+ Space::SMPTE170M,+ Space::RGB,+ ] {+ assert_eq!(YuvMatrix::from_color_space(space), YuvMatrix::Bt601);+ }+ }++ #[test]+ fn simd_conversion_uses_the_requested_matrix() {+ let (width, height) = (64u32, 4u32);+ let y = vec![81u8; (width * height) as usize];+ let uv: Vec<u8> = (0..(width * height / 2))+ .map(|i| if i % 2 == 0 { 90 } else { 240 })+ .collect();+ for matrix in [YuvMatrix::Bt601, YuvMatrix::Bt709] {+ let mut simd = vec![0u8; (width * height * 4) as usize];+ let mut scalar = simd.clone();+ nv12_to_rgba_simd(&y, &uv, width, height, width, width, &mut simd, matrix);+ nv12_to_rgba_with_matrix(&y, &uv, width, height, width, width, &mut scalar, matrix);+ assert_eq!(simd, scalar);+ assert_eq!(+ &simd[..3],+ &matrix.ycbcr_to_rgb(81 - 16, 90 - 128, 240 - 128)+ );+ }+ }}diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rsindex af79c9560..0703321ca 100644--- a/crates/rendering/src/decoder/avassetreader.rs+++ b/crates/rendering/src/decoder/avassetreader.rs@@ -21,7 +21,7 @@ use tokio::{sync::{OwnedSemaphorePermit, Semaphore, oneshot},};-use crate::{DecodedFrame, PixelFormat};+use crate::{DecodedFrame, PixelFormat, cpu_yuv::YuvMatrix};use super::frame_converter::{copy_bgra_to_rgba, copy_rgba_plane};use super::multi_position::{DecoderPoolManager, MultiPositionDecoderConfig, ScrubDetector};@@ -100,11 +100,13 @@ impl ProcessedFrame {#[derive(Clone)]struct CachedFrame(ProcessedFrame);-struct ImageBufProcessor;+struct ImageBufProcessor {+ matrix: YuvMatrix,+}impl ImageBufProcessor {- fn new() -> Self {- Self+ fn new(matrix: YuvMatrix) -> Self {+ Self { matrix }}fn extract_raw(&self, image_buf: &mut R<cv::ImageBuf>) -> (Vec<u8>, PixelFormat, u32, u32) {@@ -265,7 +267,10 @@ impl CachedFrame {}}};- Self(ProcessedFrame { number, frame })+ Self(ProcessedFrame {+ number,+ frame: frame.with_matrix(processor.matrix),+ })}_ => {let black_frame = vec![0u8; (width * height * 4) as usize];@@ -650,9 +655,13 @@ impl AVAssetReaderDecoder {if video.width() == 0 || video.height() == 0 {return Err("Invalid video dimensions".to_string());}- Ok((video.width(), video.height()))+ Ok((+ video.width(),+ video.height(),+ YuvMatrix::from_color_space(video.color_space()),+ ))})();- let (width, height) = match dimensions {+ let (width, height, matrix) = match dimensions {Ok(dimensions) => dimensions,Err(error) => {let _ = ready_tx.send(Err(error));@@ -688,7 +697,7 @@ impl AVAssetReaderDecoder {// pre-hole frame leaves the cache.let mut gap_hold: Option<ProcessedFrame> = None;- let processor = ImageBufProcessor::new();+ let processor = ImageBufProcessor::new(matrix);struct PendingRequest {frame: u32,diff --git a/crates/rendering/src/decoder/ffmpeg.rs b/crates/rendering/src/decoder/ffmpeg.rsindex b06811534..ec14a4641 100644--- a/crates/rendering/src/decoder/ffmpeg.rs+++ b/crates/rendering/src/decoder/ffmpeg.rs@@ -17,6 +17,8 @@ use crate::{DecodedFrame, PixelFormat};#[cfg(target_os = "windows")]use cap_video_decode::FrameTextures;+use crate::cpu_yuv::YuvMatrix;+use super::{DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, ManagedVideoControl, ManagedVideoError,ManagedVideoInput, VideoDecoderMessage, frame_converter::FrameConverter, pts_to_frame,@@ -30,6 +32,7 @@ struct ProcessedFrame {width: u32,height: u32,format: PixelFormat,+ matrix: YuvMatrix,y_stride: u32,uv_stride: u32,#[cfg(target_os = "linux")]@@ -40,9 +43,10 @@ impl ProcessedFrame {fn to_decoded_frame(&self) -> DecodedFrame {#[cfg(target_os = "linux")]if let Some(cuda) = &self.cuda {- return DecodedFrame::new_nv12_cuda(Arc::clone(cuda), Arc::clone(&self.data));+ return DecodedFrame::new_nv12_cuda(Arc::clone(cuda), Arc::clone(&self.data))+ .with_matrix(self.matrix);}- match self.format {+ let frame = match self.format {PixelFormat::Rgba => {DecodedFrame::new_with_arc(Arc::clone(&self.data), self.width, self.height)}@@ -60,7 +64,8 @@ impl ProcessedFrame {self.y_stride,self.uv_stride,),- }+ };+ frame.with_matrix(self.matrix)}}@@ -163,6 +168,7 @@ impl CachedFrame {width: frame.width(),height: frame.height(),format: PixelFormat::Nv12,+ matrix: YuvMatrix::from_color_space(frame.color_space()),y_stride: cuda.y_pitch as u32,uv_stride: cuda.uv_pitch as u32,cuda: Some(Arc::new(cuda)),@@ -180,6 +186,7 @@ impl CachedFrame {width: frame.width(),height: frame.height(),format: pixel_format,+ matrix: YuvMatrix::from_color_space(frame.color_space()),y_stride,uv_stride,#[cfg(target_os = "linux")]@@ -193,6 +200,7 @@ impl CachedFrame {width: frame.width(),height: frame.height(),format: PixelFormat::Rgba,+ matrix: YuvMatrix::from_color_space(frame.color_space()),y_stride: frame.width() * 4,uv_stride: 0,#[cfg(target_os = "linux")]@@ -1698,6 +1706,7 @@ mod cache_tests {width: 2,height: 2,format: PixelFormat::Nv12,+ matrix: YuvMatrix::Bt601,y_stride: 2,uv_stride: 2,#[cfg(target_os = "linux")]diff --git a/crates/rendering/src/decoder/frame_converter.rs b/crates/rendering/src/decoder/frame_converter.rsindex 7a449ffa7..1a156314a 100644--- a/crates/rendering/src/decoder/frame_converter.rs+++ b/crates/rendering/src/decoder/frame_converter.rs@@ -2,6 +2,8 @@use ffmpeg::{format, frame, software};+use crate::cpu_yuv::YuvMatrix;+pub struct FrameConverter {scaler: Option<ScalerState>,rgba_frame: frame::Video,@@ -10,6 +12,7 @@ pub struct FrameConverter {struct ScalerState {context: software::scaling::Context,input_format: format::Pixel,+ matrix: YuvMatrix,width: u32,height: u32,}@@ -34,7 +37,12 @@ impl FrameConverter {let width = frame.width();let height = frame.height();- self.ensure_scaler(frame.format(), width, height);+ self.ensure_scaler(+ frame.format(),+ YuvMatrix::from_color_space(frame.color_space()),+ width,+ height,+ );self.ensure_rgba_frame(width, height);{@@ -56,16 +64,31 @@ impl FrameConverter {)}- fn ensure_scaler(&mut self, input_format: format::Pixel, width: u32, height: u32) {+ fn ensure_scaler(+ &mut self,+ input_format: format::Pixel,+ matrix: YuvMatrix,+ width: u32,+ height: u32,+ ) {let needs_new = self.scaler.as_ref().is_none_or(|state| {- state.input_format != input_format || state.width != width || state.height != height+ state.input_format != input_format+ || state.matrix != matrix+ || state.width != width+ || state.height != height});if needs_new {+ let mut context =+ software::converter((width, height), input_format, format::Pixel::RGBA)+ .expect("failed to create frame scaler");+ if matrix == YuvMatrix::Bt709 {+ use_bt709_source(&mut context);+ }self.scaler = Some(ScalerState {- context: software::converter((width, height), input_format, format::Pixel::RGBA)- .expect("failed to create frame scaler"),+ context,input_format,+ matrix,width,height,});@@ -80,6 +103,41 @@ impl FrameConverter {}}+/// swscale converts YUV with BT.601 unless told otherwise: switch the source+/// matrix to BT.709 and keep the ranges and adjustments it chose.+fn use_bt709_source(context: &mut software::scaling::Context) {+ unsafe {+ let mut inv_table = std::ptr::null_mut();+ let mut table = std::ptr::null_mut();+ let (mut src_range, mut dst_range) = (0, 0);+ let (mut brightness, mut contrast, mut saturation) = (0, 0, 0);+ if ffmpeg::ffi::sws_getColorspaceDetails(+ context.as_mut_ptr(),+ &mut inv_table,+ &mut src_range,+ &mut table,+ &mut dst_range,+ &mut brightness,+ &mut contrast,+ &mut saturation,+ ) < 0+ {+ return;+ }+ let bt709 = ffmpeg::ffi::sws_getCoefficients(ffmpeg::ffi::SWS_CS_ITU709);+ ffmpeg::ffi::sws_setColorspaceDetails(+ context.as_mut_ptr(),+ bt709,+ src_range,+ table,+ dst_range,+ brightness,+ contrast,+ saturation,+ );+ }+}+pub fn copy_rgba_plane(data: &[u8], stride: usize, width: usize, height: usize) -> Vec<u8> {debug_assert!(stride >= width * 4, "stride too small for RGBA frame");diff --git a/crates/rendering/src/decoder/media_foundation.rs b/crates/rendering/src/decoder/media_foundation.rsindex 1fb01862c..894d37cbe 100644--- a/crates/rendering/src/decoder/media_foundation.rs+++ b/crates/rendering/src/decoder/media_foundation.rs@@ -229,6 +229,7 @@ impl MFDecoder {let video_width = decoder.width();let video_height = decoder.height();+ let matrix = super::probe_yuv_matrix(&path);let mut cache = BTreeMap::<u32, CachedFrame>::new();let mut cache_bytes = 0usize;@@ -281,7 +282,9 @@ impl MFDecoder {let mut unfulfilled = Vec::with_capacity(pending_requests.len());for req in pending_requests.drain(..) {if let Some(frame) = cached_frame_for(&cache, last_decoded_frame, req.frame) {- let _ = req.sender.send(frame.to_decoded_frame());+ let _ = req+ .sender+ .send(frame.to_decoded_frame().with_matrix(matrix));} else if !req.sender.is_closed() {unfulfilled.push(req);}@@ -464,7 +467,7 @@ impl MFDecoder {if let Some(frame) = frame_to_send&& let Some(s) = sender.take(){- let _ = s.send(frame.to_decoded_frame());+ let _ = s.send(frame.to_decoded_frame().with_matrix(matrix));}}@@ -496,7 +499,7 @@ impl MFDecoder {.map(|(_, f)| f.clone())});if let Some(frame) = fallback {- let _ = s.send(frame.to_decoded_frame());+ let _ = s.send(frame.to_decoded_frame().with_matrix(matrix));} else {let black_frame = DecodedFrame::new(vec![0u8; (video_width * video_height * 4) as usize],@@ -516,7 +519,9 @@ impl MFDecoder {.or_else(|| cache.range(..=req.frame).next_back().map(|(_, f)| f)).or(last_valid_frame.as_ref());if let Some(frame) = frame_to_send {- let _ = req.sender.send(frame.to_decoded_frame());+ let _ = req+ .sender+ .send(frame.to_decoded_frame().with_matrix(matrix));} else {let black_frame = DecodedFrame::new(vec![0u8; (video_width * video_height * 4) as usize],diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rsindex 170507f30..366a28826 100644--- a/crates/rendering/src/decoder/mod.rs+++ b/crates/rendering/src/decoder/mod.rs@@ -1,3 +1,4 @@+use crate::cpu_yuv::YuvMatrix;use ::ffmpeg::Rational;use std::{fmt,@@ -156,6 +157,7 @@ pub struct DecodedFrame {width: u32,height: u32,format: PixelFormat,+ matrix: YuvMatrix,y_stride: u32,uv_stride: u32,#[cfg(target_os = "macos")]@@ -281,6 +283,7 @@ impl DecodedFrame {width,height,format: PixelFormat::Rgba,+ matrix: YuvMatrix::Bt601,y_stride: width * 4,uv_stride: 0,#[cfg(target_os = "macos")]@@ -298,6 +301,7 @@ impl DecodedFrame {width,height,format: PixelFormat::Rgba,+ matrix: YuvMatrix::Bt601,y_stride: width * 4,uv_stride: 0,#[cfg(target_os = "macos")]@@ -322,6 +326,7 @@ impl DecodedFrame {width: frame.width,height: frame.height,format: PixelFormat::Nv12,+ matrix: YuvMatrix::Bt601,y_stride: frame.y_pitch as u32,uv_stride: frame.uv_pitch as u32,cuda_nv12: Some(frame),@@ -339,6 +344,7 @@ impl DecodedFrame {width,height,format: PixelFormat::Nv12,+ matrix: YuvMatrix::Bt601,y_stride,uv_stride,#[cfg(target_os = "macos")]@@ -362,6 +368,7 @@ impl DecodedFrame {width,height,format: PixelFormat::Nv12,+ matrix: YuvMatrix::Bt601,y_stride,uv_stride,#[cfg(target_os = "macos")]@@ -386,6 +393,7 @@ impl DecodedFrame {width,height,format: PixelFormat::Nv12,+ matrix: YuvMatrix::Bt601,y_stride,uv_stride,image_buf_backing: Some(Arc::new(SendableImageBuf::new(image_buf))),@@ -404,6 +412,7 @@ impl DecodedFrame {width,height,format: PixelFormat::Yuv420p,+ matrix: YuvMatrix::Bt601,y_stride,uv_stride,#[cfg(target_os = "macos")]@@ -427,6 +436,7 @@ impl DecodedFrame {width,height,format: PixelFormat::Yuv420p,+ matrix: YuvMatrix::Bt601,y_stride,uv_stride,#[cfg(target_os = "macos")]@@ -445,6 +455,7 @@ impl DecodedFrame {width,height,format: PixelFormat::Nv12,+ matrix: YuvMatrix::Bt601,y_stride: width,uv_stride: width,#[cfg(target_os = "macos")]@@ -465,6 +476,7 @@ impl DecodedFrame {width,height,format: PixelFormat::Nv12,+ matrix: YuvMatrix::Bt601,y_stride: width,uv_stride: width,#[cfg(target_os = "macos")]@@ -490,6 +502,7 @@ impl DecodedFrame {width,height,format: PixelFormat::Nv12,+ matrix: YuvMatrix::Bt601,y_stride: width,uv_stride: width,#[cfg(target_os = "macos")]@@ -560,6 +573,16 @@ impl DecodedFrame {self.format}+ /// The Y'CbCr matrix to convert this frame's planes to RGB with.+ pub fn matrix(&self) -> YuvMatrix {+ self.matrix+ }++ pub fn with_matrix(mut self, matrix: YuvMatrix) -> Self {+ self.matrix = matrix;+ self+ }+pub fn byte_len(&self) -> usize {if !self.data.is_empty() {return self.data.len();@@ -663,6 +686,22 @@ pub enum VideoDecoderMessage {GetFrame(f32, u32, tokio::sync::oneshot::Sender<DecodedFrame>),}+/// The source's Y'CbCr matrix from its colour tag, for decoders that do not+/// see FFmpeg frames (Media Foundation). BT.601 when the file cannot be read.+#[cfg(target_os = "windows")]+pub(crate) fn probe_yuv_matrix(path: &std::path::Path) -> YuvMatrix {+ ::ffmpeg::format::input(path)+ .ok()+ .and_then(|input| {+ let stream = input.streams().best(::ffmpeg::media::Type::Video)?;+ let context = ::ffmpeg::codec::Context::from_parameters(stream.parameters()).ok()?;+ Some(YuvMatrix::from_color_space(+ context.decoder().video().ok()?.color_space(),+ ))+ })+ .unwrap_or_default()+}+pub fn pts_to_frame(pts: i64, time_base: Rational, fps: u32) -> u32 {(fps as f64 * ((pts as f64 * time_base.numerator() as f64) / (time_base.denominator() as f64))).round() as u32diff --git a/crates/rendering/src/layers/camera.rs b/crates/rendering/src/layers/camera.rsindex 9f36daa88..9fca2068d 100644--- a/crates/rendering/src/layers/camera.rs+++ b/crates/rendering/src/layers/camera.rs@@ -117,6 +117,7 @@ impl CameraLayer {};let format = camera_frame.format();+ self.yuv_converter.set_matrix(camera_frame.matrix());let frame_storage = camera_frame.storage_identity();let is_same_frame = self@@ -402,6 +403,7 @@ impl CameraLayer {};let format = camera_frame.format();+ self.yuv_converter.set_matrix(camera_frame.matrix());let frame_storage = camera_frame.storage_identity();let is_same_frame = selfdiff --git a/crates/rendering/src/layers/display.rs b/crates/rendering/src/layers/display.rsindex ad7575b64..21f1f9786 100644--- a/crates/rendering/src/layers/display.rs+++ b/crates/rendering/src/layers/display.rs@@ -162,6 +162,7 @@ impl DisplayLayer {let actual_height = screen_frame.height();let source_size = XY::new(actual_width, actual_height);let format = screen_frame.format();+ self.yuv_converter.set_matrix(screen_frame.matrix());let current_recording_time = segment_frames.recording_time;let frame_storage = screen_frame.storage_identity();@@ -578,6 +579,7 @@ impl DisplayLayer {let actual_height = screen_frame.height();let source_size = XY::new(actual_width, actual_height);let format = screen_frame.format();+ self.yuv_converter.set_matrix(screen_frame.matrix());let current_recording_time = segment_frames.recording_time;let frame_storage = screen_frame.storage_identity();diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rsindex dcb858929..76b7f9490 100644--- a/crates/rendering/src/lib.rs+++ b/crates/rendering/src/lib.rs@@ -6911,7 +6911,7 @@ impl<'a> FrameRenderer<'a> {let r = src_row[col * 4] as i32;let g = src_row[col * 4 + 1] as i32;let b = src_row[col * 4 + 2] as i32;- y_row[col] = ((16 + ((65 * r + 129 * g + 25 * b + 128) >> 8)) as u8).clamp(16, 235);+ y_row[col] = cpu_yuv::bt709_luma(r, g, b);}}@@ -6942,10 +6942,7 @@ impl<'a> FrameRenderer<'a> {+ src_row1[col * 8 + 6] as i32+ 2)/ 4;- uv_row[col * 2] =- ((128 + ((-38 * r - 74 * g + 112 * b + 128) >> 8)) as u8).clamp(16, 240);- uv_row[col * 2 + 1] =- ((128 + ((112 * r - 94 * g - 18 * b + 128) >> 8)) as u8).clamp(16, 240);+ [uv_row[col * 2], uv_row[col * 2 + 1]] = cpu_yuv::bt709_chroma(r, g, b);}}diff --git a/crates/rendering/src/shaders/nv12_to_rgba.wgsl b/crates/rendering/src/shaders/nv12_to_rgba.wgslindex e6265357d..a6093790f 100644--- a/crates/rendering/src/shaders/nv12_to_rgba.wgsl+++ b/crates/rendering/src/shaders/nv12_to_rgba.wgsl@@ -1,3 +1,11 @@+// Limited-range Y'CbCr to RGB. The chroma coefficients are pipeline+// overrides (YuvMatrix::shader_constants): BT.601 by default, BT.709 for+// sources tagged BT.709.+override RV: f32 = 1.596;+override GU: f32 = 0.391;+override GV: f32 = 0.813;+override BU: f32 = 2.018;+@group(0) @binding(0) var y_plane: texture_2d<f32>;@group(0) @binding(1) var uv_plane: texture_2d<f32>;@group(0) @binding(2) var output: texture_storage_2d<rgba8unorm, write>;@@ -22,9 +30,9 @@ fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {let u = uv_raw.r - 0.5;let v = uv_raw.g - 0.5;- let r = y + 1.596 * v;- let g = y - 0.391 * u - 0.813 * v;- let b = y + 2.018 * u;+ let r = y + RV * v;+ let g = y - GU * u - GV * v;+ let b = y + BU * u;let color = vec4<f32>(clamp(r, 0.0, 1.0),diff --git a/crates/rendering/src/shaders/rgba_to_nv12.wgsl b/crates/rendering/src/shaders/rgba_to_nv12.wgslindex 444805da2..a4aa85e7b 100644--- a/crates/rendering/src/shaders/rgba_to_nv12.wgsl+++ b/crates/rendering/src/shaders/rgba_to_nv12.wgsl@@ -9,16 +9,18 @@ struct Params {@group(0) @binding(1) var<storage, read_write> nv12_output: array<u32>;@group(0) @binding(2) var<uniform> params: Params;+// BT.709 limited range, the matrix every export is tagged with (see+// crates/enc-ffmpeg/src/video/h264.rs). Rounded to the nearest code value.fn rgb_to_y(r: f32, g: f32, b: f32) -> u32 {- return u32(clamp(16.0 + 65.481 * r + 128.553 * g + 24.966 * b, 0.0, 255.0));+ return u32(clamp(16.5 + 46.5594 * r + 156.6288 * g + 15.8118 * b, 0.0, 255.0));}fn rgb_to_u(r: f32, g: f32, b: f32) -> u32 {- return u32(clamp(128.0 - 37.797 * r - 74.203 * g + 112.0 * b, 0.0, 255.0));+ return u32(clamp(128.5 - 25.6642 * r - 86.3358 * g + 112.0 * b, 0.0, 255.0));}fn rgb_to_v(r: f32, g: f32, b: f32) -> u32 {- return u32(clamp(128.0 + 112.0 * r - 93.786 * g - 18.214 * b, 0.0, 255.0));+ return u32(clamp(128.5 + 112.0 * r - 101.7303 * g - 10.2697 * b, 0.0, 255.0));}fn safe_load(coord: vec2<u32>, dims: vec2<u32>) -> vec4<f32> {diff --git a/crates/rendering/src/shaders/yuv420p_to_rgba.wgsl b/crates/rendering/src/shaders/yuv420p_to_rgba.wgslindex 0d04d30fe..8838df9bc 100644--- a/crates/rendering/src/shaders/yuv420p_to_rgba.wgsl+++ b/crates/rendering/src/shaders/yuv420p_to_rgba.wgsl@@ -1,3 +1,11 @@+// Limited-range Y'CbCr to RGB. The chroma coefficients are pipeline+// overrides (YuvMatrix::shader_constants): BT.601 by default, BT.709 for+// sources tagged BT.709.+override RV: f32 = 1.596;+override GU: f32 = 0.391;+override GV: f32 = 0.813;+override BU: f32 = 2.018;+@group(0) @binding(0) var y_plane: texture_2d<f32>;@group(0) @binding(1) var u_plane: texture_2d<f32>;@group(0) @binding(2) var v_plane: texture_2d<f32>;@@ -26,9 +34,9 @@ fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {let u = u_raw - 0.5;let v = v_raw - 0.5;- let r = y + 1.596 * v;- let g = y - 0.391 * u - 0.813 * v;- let b = y + 2.018 * u;+ let r = y + RV * v;+ let g = y - GU * u - GV * v;+ let b = y + BU * u;let color = vec4<f32>(clamp(r, 0.0, 1.0),diff --git a/crates/rendering/src/yuv_converter.rs b/crates/rendering/src/yuv_converter.rsindex 6791c1cc5..d9c7c9ffc 100644--- a/crates/rendering/src/yuv_converter.rs+++ b/crates/rendering/src/yuv_converter.rs@@ -1,4 +1,4 @@-use crate::cpu_yuv;+use crate::cpu_yuv::{self, YuvMatrix};#[cfg(target_os = "macos")]use crate::iosurface_texture::{@@ -259,9 +259,60 @@ impl BindGroupCache {use std::sync::Arc;+/// One compute pipeline per source matrix, compiled on first use, so a+/// project only ever builds the variants its sources need.+struct MatrixPipelines {+ device: wgpu::Device,+ label: &'static str,+ layout: wgpu::PipelineLayout,+ module: wgpu::ShaderModule,+ bt601: std::sync::OnceLock<wgpu::ComputePipeline>,+ bt709: std::sync::OnceLock<wgpu::ComputePipeline>,+}++impl MatrixPipelines {+ fn new(+ device: &wgpu::Device,+ label: &'static str,+ layout: wgpu::PipelineLayout,+ module: wgpu::ShaderModule,+ ) -> Self {+ Self {+ device: device.clone(),+ label,+ layout,+ module,+ bt601: std::sync::OnceLock::new(),+ bt709: std::sync::OnceLock::new(),+ }+ }++ fn get(&self, matrix: YuvMatrix) -> &wgpu::ComputePipeline {+ let cell = match matrix {+ YuvMatrix::Bt601 => &self.bt601,+ YuvMatrix::Bt709 => &self.bt709,+ };+ cell.get_or_init(|| {+ let constants = matrix.shader_constants();+ self.device+ .create_compute_pipeline(&wgpu::ComputePipelineDescriptor {+ label: Some(self.label),+ layout: Some(&self.layout),+ module: &self.module,+ entry_point: Some("main"),+ compilation_options: wgpu::PipelineCompilationOptions {+ constants: &constants,+ ..Default::default()+ },+ cache: None,+ })+ })+ }+}+pub struct YuvConverterPipelines {- pub nv12_pipeline: wgpu::ComputePipeline,- pub yuv420p_pipeline: wgpu::ComputePipeline,+ nv12: MatrixPipelines,+ yuv420p: MatrixPipelines,pub nv12_bind_group_layout: wgpu::BindGroupLayout,pub yuv420p_bind_group_layout: wgpu::BindGroupLayout,}@@ -381,29 +432,23 @@ impl YuvConverterPipelines {push_constant_ranges: &[],});- let nv12_pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {- label: Some("NV12 Converter Pipeline"),- layout: Some(&nv12_pipeline_layout),- module: &nv12_shader,- entry_point: Some("main"),- compilation_options: Default::default(),- cache: None,- });-- let yuv420p_pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {- label: Some("YUV420P Converter Pipeline"),- layout: Some(&yuv420p_pipeline_layout),- module: &yuv420p_shader,- entry_point: Some("main"),- compilation_options: Default::default(),- cache: None,- });-+ let nv12 = MatrixPipelines::new(+ device,+ "NV12 Converter Pipeline",+ nv12_pipeline_layout,+ nv12_shader,+ );+ let yuv420p = MatrixPipelines::new(+ device,+ "YUV420P Converter Pipeline",+ yuv420p_pipeline_layout,+ yuv420p_shader,+ );tracing::info!("Shared YUV converter pipelines created successfully");Self {- nv12_pipeline,- yuv420p_pipeline,+ nv12,+ yuv420p,nv12_bind_group_layout,yuv420p_bind_group_layout,}@@ -431,6 +476,7 @@ unsafe impl Sync for RetainedImageBuf {}pub struct YuvToRgbaConverter {pipelines: Arc<YuvConverterPipelines>,+ matrix: YuvMatrix,y_texture: wgpu::Texture,y_view: wgpu::TextureView,uv_texture: wgpu::Texture,@@ -467,6 +513,11 @@ pub struct YuvToRgbaConverter {}impl YuvToRgbaConverter {+ /// The matrix of the frames converted next (see [`YuvMatrix`]).+ pub fn set_matrix(&mut self, matrix: YuvMatrix) {+ self.matrix = matrix;+ }+pub fn new(device: &wgpu::Device) -> Self {let pipelines = Arc::new(YuvConverterPipelines::new(device));Self::new_with_shared_pipelines(device, pipelines)@@ -494,6 +545,7 @@ impl YuvToRgbaConverter {Self::create_output_textures(device, initial_width, initial_height);Self {+ matrix: YuvMatrix::default(),pipelines,y_texture,y_view,@@ -656,7 +708,7 @@ impl YuvToRgbaConverter {label: Some("NV12 CUDA Conversion Pass"),..Default::default()});- compute_pass.set_pipeline(&self.pipelines.nv12_pipeline);+ compute_pass.set_pipeline(self.pipelines.nv12.get(self.matrix));compute_pass.set_bind_group(0, bind_group, &[]);compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);}@@ -939,7 +991,7 @@ impl YuvToRgbaConverter {label: Some("NV12 Conversion Pass"),..Default::default()});- compute_pass.set_pipeline(&self.pipelines.nv12_pipeline);+ compute_pass.set_pipeline(self.pipelines.nv12.get(self.matrix));compute_pass.set_bind_group(0, bind_group, &[]);compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);}@@ -1024,7 +1076,7 @@ impl YuvToRgbaConverter {label: Some("NV12 Conversion Pass (Batched)"),..Default::default()});- compute_pass.set_pipeline(&self.pipelines.nv12_pipeline);+ compute_pass.set_pipeline(self.pipelines.nv12.get(self.matrix));compute_pass.set_bind_group(0, bind_group, &[]);compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);}@@ -1121,7 +1173,7 @@ impl YuvToRgbaConverter {label: Some("NV12 IOSurface Conversion Pass (Batched)"),..Default::default()});- compute_pass.set_pipeline(&self.pipelines.nv12_pipeline);+ compute_pass.set_pipeline(self.pipelines.nv12.get(self.matrix));compute_pass.set_bind_group(0, &bind_group, &[]);compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);}@@ -1202,7 +1254,7 @@ impl YuvToRgbaConverter {label: Some("YUV420P Conversion Pass"),..Default::default()});- compute_pass.set_pipeline(&self.pipelines.yuv420p_pipeline);+ compute_pass.set_pipeline(self.pipelines.yuv420p.get(self.matrix));compute_pass.set_bind_group(0, bind_group, &[]);compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);}@@ -1273,7 +1325,7 @@ impl YuvToRgbaConverter {label: Some("YUV420P Conversion Pass (Batched)"),..Default::default()});- compute_pass.set_pipeline(&self.pipelines.yuv420p_pipeline);+ compute_pass.set_pipeline(self.pipelines.yuv420p.get(self.matrix));compute_pass.set_bind_group(0, bind_group, &[]);compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);}@@ -1434,7 +1486,7 @@ impl YuvToRgbaConverter {label: Some("NV12 D3D11 Conversion Pass"),..Default::default()});- compute_pass.set_pipeline(&self.pipelines.nv12_pipeline);+ compute_pass.set_pipeline(self.pipelines.nv12.get(self.matrix));compute_pass.set_bind_group(0, &bind_group, &[]);compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);}@@ -1528,7 +1580,7 @@ impl YuvToRgbaConverter {label: Some("NV12 D3D11 Zero-Copy Conversion Pass"),..Default::default()});- compute_pass.set_pipeline(&self.pipelines.nv12_pipeline);+ compute_pass.set_pipeline(self.pipelines.nv12.get(self.matrix));compute_pass.set_bind_group(0, &bind_group, &[]);compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);}@@ -1639,6 +1691,7 @@ impl YuvToRgbaConverter {y_stride,uv_stride,&mut rgba_data,+ self.matrix,);queue.write_texture(@@ -1693,6 +1746,7 @@ impl YuvToRgbaConverter {y_stride,uv_stride,&mut rgba_data,+ self.matrix,);queue.write_texture(@@ -1722,3 +1776,222 @@ impl YuvToRgbaConverter {Some(self.current_output_texture())}}++/// The GPU conversions against the CPU reference in `cpu_yuv`: the export's+/// RGBA to NV12 shader must produce BT.709 limited range (the matrix the+/// encoders tag), and the decode shaders must apply the matrix they are asked+/// for. Skipped where no adapter exists.+#[cfg(test)]+mod colour_matrix_tests {+ use super::*;+ use crate::frame_pipeline::RgbaToNv12Converter;++ // Primaries, secondaries, grey axis and the suite's red background.+ const CHART: [[u8; 3]; 8] = [+ [0, 0, 0],+ [255, 255, 255],+ [128, 128, 128],+ [255, 0, 0],+ [0, 255, 0],+ [0, 0, 255],+ [255, 255, 0],+ [220, 40, 40],+ ];+ // Each chart colour fills a 4x2 block, so chroma averaging is exact.+ const WIDTH: u32 = 4 * CHART.len() as u32;+ const HEIGHT: u32 = 2;++ fn device() -> Option<(wgpu::Device, wgpu::Queue)> {+ let instance = crate::create_wgpu_instance_sync();+ let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {+ power_preference: wgpu::PowerPreference::LowPower,+ force_fallback_adapter: false,+ compatible_surface: None,+ }))+ .ok()?;+ pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()+ }++ fn reference_bt709(rgb: [u8; 3]) -> [f64; 3] {+ let (kr, kb) = (0.2126, 0.0722);+ let [r, g, b] = rgb.map(|v| f64::from(v) / 255.0);+ let y = kr * r + (1.0 - kr - kb) * g + kb * b;+ [+ 16.0 + 219.0 * y,+ 128.0 + 224.0 * (b - y) / (2.0 * (1.0 - kb)),+ 128.0 + 224.0 * (r - y) / (2.0 * (1.0 - kr)),+ ]+ }++ fn chart_texture(device: &wgpu::Device, queue: &wgpu::Queue) -> wgpu::Texture {+ let texture = device.create_texture(&wgpu::TextureDescriptor {+ label: Some("colour chart"),+ size: wgpu::Extent3d {+ width: WIDTH,+ height: HEIGHT,+ depth_or_array_layers: 1,+ },+ mip_level_count: 1,+ sample_count: 1,+ dimension: wgpu::TextureDimension::D2,+ format: wgpu::TextureFormat::Rgba8Unorm,+ usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,+ view_formats: &[],+ });+ let data: Vec<u8> = (0..HEIGHT)+ .flat_map(|_| {+ CHART+ .iter()+ .flat_map(|[r, g, b]| [*r, *g, *b, 255].repeat(4))+ })+ .collect();+ queue.write_texture(+ texture.as_image_copy(),+ &data,+ wgpu::TexelCopyBufferLayout {+ offset: 0,+ bytes_per_row: Some(WIDTH * 4),+ rows_per_image: None,+ },+ wgpu::Extent3d {+ width: WIDTH,+ height: HEIGHT,+ depth_or_array_layers: 1,+ },+ );+ texture+ }++ /// The chart as BT.709 NV12 from the CPU encoder (tight strides).+ fn chart_nv12() -> (Vec<u8>, Vec<u8>) {+ let mut y = Vec::new();+ let mut uv = Vec::new();+ for _ in 0..HEIGHT {+ for [r, g, b] in CHART.map(|c| c.map(i32::from)) {+ y.extend([cpu_yuv::bt709_luma(r, g, b); 4]);+ }+ }+ for [r, g, b] in CHART.map(|c| c.map(i32::from)) {+ let [cb, cr] = cpu_yuv::bt709_chroma(r, g, b);+ uv.extend([cb, cr, cb, cr]);+ }+ (y, uv)+ }++ fn read_rgba(device: &wgpu::Device, queue: &wgpu::Queue, texture: &wgpu::Texture) -> Vec<u8> {+ let padded = wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;+ let buffer = device.create_buffer(&wgpu::BufferDescriptor {+ label: Some("chart readback"),+ size: u64::from(padded * HEIGHT),+ usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,+ mapped_at_creation: false,+ });+ let mut encoder = device.create_command_encoder(&Default::default());+ encoder.copy_texture_to_buffer(+ texture.as_image_copy(),+ wgpu::TexelCopyBufferInfo {+ buffer: &buffer,+ layout: wgpu::TexelCopyBufferLayout {+ offset: 0,+ bytes_per_row: Some(padded),+ rows_per_image: None,+ },+ },+ wgpu::Extent3d {+ width: WIDTH,+ height: HEIGHT,+ depth_or_array_layers: 1,+ },+ );+ queue.submit(std::iter::once(encoder.finish()));+ buffer.slice(..).map_async(wgpu::MapMode::Read, |_| {});+ device.poll(wgpu::PollType::Wait).expect("poll");+ let mapped = buffer.slice(..).get_mapped_range();+ (0..HEIGHT as usize)+ .flat_map(|row| {+ let start = row * padded as usize;+ mapped[start..start + WIDTH as usize * 4].to_vec()+ })+ .collect()+ }++ #[tokio::test]+ async fn export_shader_encodes_bt709_limited_range() {+ let Some((device, queue)) = device() else {+ eprintln!("no GPU adapter available, skipping");+ return;+ };+ let source = chart_texture(&device, &queue);+ let mut converter = RgbaToNv12Converter::new(&device);+ let mut encoder = device.create_command_encoder(&Default::default());+ assert!(+ converter+ .submit_conversion(&device, &queue, &mut encoder, &source, WIDTH, HEIGHT, 0, 30)+ .await+ .expect("conversion submission")+ );+ queue.submit(std::iter::once(encoder.finish()));+ converter.after_submit(&queue);+ let frame = converter+ .take_pending()+ .expect("conversion pending")+ .wait_with_pool(&device, None)+ .await+ .expect("conversion completes");+ let stride = frame.y_stride as usize;+ let uv_base = stride * HEIGHT as usize;+ for (i, rgb) in CHART.iter().enumerate() {+ let [y, cb, cr] = reference_bt709(*rgb);+ let x = i * 4;+ let actual = [+ frame.data[x],+ frame.data[uv_base + x],+ frame.data[uv_base + x + 1],+ ];+ for (a, e) in actual.iter().zip([y, cb, cr]) {+ assert!(+ (f64::from(*a) - e).abs() <= 0.5 + 1e-3,+ "{rgb:?}: got {actual:?}, BT.709 reference {y:.2} {cb:.2} {cr:.2}"+ );+ }+ }+ }++ #[test]+ fn decode_shaders_apply_the_requested_matrix() {+ let Some((device, queue)) = device() else {+ eprintln!("no GPU adapter available, skipping");+ return;+ };+ let (y, uv) = chart_nv12();+ let mut converter = YuvToRgbaConverter::new(&device);+ for matrix in [YuvMatrix::Bt601, YuvMatrix::Bt709] {+ converter.set_matrix(matrix);+ converter+ .convert_nv12(&device, &queue, &y, &uv, WIDTH, HEIGHT, WIDTH, WIDTH)+ .expect("convert");+ let rgba = read_rgba(&device, &queue, converter.output_texture().unwrap());+ for (i, rgb) in CHART.iter().enumerate() {+ let x = i * 4;+ let expected = matrix.ycbcr_to_rgb(+ i32::from(y[x]) - 16,+ i32::from(uv[x]) - 128,+ i32::from(uv[x + 1]) - 128,+ );+ let actual = &rgba[x * 4..x * 4 + 3];+ // The shaders' float constants and 16/256 luma offset differ+ // from the 8.8 integer path by up to 2 levels (as before).+ assert!(+ actual.iter().zip(expected).all(|(a, e)| a.abs_diff(e) <= 2),+ "{matrix:?} {rgb:?}: GPU {actual:?}, CPU {expected:?}"+ );+ if matrix == YuvMatrix::Bt709 {+ assert!(+ actual.iter().zip(rgb).all(|(a, e)| a.abs_diff(*e) <= 2),+ "BT.709 round trip of {rgb:?} gave {actual:?}"+ );+ }+ }+ }+ }+}