Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(rendering): encode exports in BT.709 and decode sources by their tag

Richie McIlroy755bfe426544
diff --git a/apps/desktop-gpui/src/editor_crop.rs b/apps/desktop-gpui/src/editor_crop.rs
index 412b8f20c..e9bfd7b82 100644
--- a/apps/desktop-gpui/src/editor_crop.rs
+++ b/apps/desktop-gpui/src/editor_crop.rs
@@ -2280,7 +2280,7 @@ async fn decode_display_frame(
let y = screen.y_plane()?;
let uv = screen.uv_plane()?;
let mut out = vec![0u8; (width * height * 4) as usize];
- cpu_yuv::nv12_to_rgba(
+ cpu_yuv::nv12_to_rgba_with_matrix(
y,
uv,
width,
@@ -2288,6 +2288,7 @@ async fn decode_display_frame(
screen.y_stride(),
screen.uv_stride(),
&mut out,
+ screen.matrix(),
);
out
}
@@ -2296,7 +2297,7 @@ async fn decode_display_frame(
let u = screen.u_plane()?;
let v = screen.v_plane()?;
let mut out = vec![0u8; (width * height * 4) as usize];
- cpu_yuv::yuv420p_to_rgba(
+ cpu_yuv::yuv420p_to_rgba_with_matrix(
y,
u,
v,
@@ -2305,6 +2306,7 @@ async fn decode_display_frame(
screen.y_stride(),
screen.uv_stride(),
&mut out,
+ screen.matrix(),
);
out
}
diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs
index f120f6cff..627faa3dc 100644
--- a/crates/export/src/mp4.rs
+++ b/crates/export/src/mp4.rs
@@ -837,7 +837,13 @@ fn nv12_from_rendered_frame(frame: Nv12RenderedFrame) -> ExportFrame {
width,
height,
ffmpeg::software::scaling::flag::Flags::FAST_BILINEAR,
- ) {
+ ) && cap_enc_ffmpeg::colour::use_bt709_for_rgb_input(
+ &mut converter,
+ ffmpeg::format::Pixel::RGBA,
+ ffmpeg::format::Pixel::NV12,
+ )
+ .is_ok()
+ {
let mut nv12_frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::NV12, width, height);
if converter.run(&rgba_frame, &mut nv12_frame).is_ok() {
let y_size = nv12_frame.stride(0) * height as usize;
@@ -1383,8 +1389,16 @@ fn save_screenshot_from_nv12(
let (y_data, uv_data) = nv12_data.split_at(y_plane_size);
let mut rgba = vec![0u8; (width * height * 4) as usize];
+ // The export's own NV12, so BT.709 like the stream it is encoded into.
cap_rendering::cpu_yuv::nv12_to_rgba_simd(
- y_data, uv_data, width, height, y_stride, width, &mut rgba,
+ y_data,
+ uv_data,
+ width,
+ height,
+ y_stride,
+ width,
+ &mut rgba,
+ cap_rendering::cpu_yuv::YuvMatrix::Bt709,
);
drop(nv12_data);
diff --git a/crates/ffmpeg-hw-device/src/lib.rs b/crates/ffmpeg-hw-device/src/lib.rs
index 023182e6a..8691bebe3 100644
--- a/crates/ffmpeg-hw-device/src/lib.rs
+++ b/crates/ffmpeg-hw-device/src/lib.rs
@@ -63,6 +63,15 @@ impl HwDevice {
if av_hwframe_transfer_data(sw_frame.as_mut_ptr(), src.as_ptr(), 0) >= 0 {
sw_frame.set_pts(src.pts());
+ // The transfer copies pixels only. Keep the colour tags so
+ // consumers convert hardware-decoded frames with the same
+ // matrix as software-decoded ones.
+ let (dst, src) = (&mut *sw_frame.as_mut_ptr(), &*src.as_ptr());
+ dst.color_range = src.color_range;
+ dst.color_primaries = src.color_primaries;
+ dst.color_trc = src.color_trc;
+ dst.colorspace = src.colorspace;
+ dst.chroma_location = src.chroma_location;
return Some(sw_frame);
};
}
diff --git a/crates/rendering/src/cpu_yuv.rs b/crates/rendering/src/cpu_yuv.rs
index 0207e4602..cd242b493 100644
--- a/crates/rendering/src/cpu_yuv.rs
+++ b/crates/rendering/src/cpu_yuv.rs
@@ -1,6 +1,53 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+/// The Y'CbCr matrix a limited-range picture was encoded with, which decides
+/// how it converts back to RGB. Decoders take it from the source's colour tag:
+/// BT.709 when the stream says so, BT.601 otherwise (untagged sources keep the
+/// BT.601 decode they always had).
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
+pub enum YuvMatrix {
+ #[default]
+ Bt601,
+ Bt709,
+}
+
+impl YuvMatrix {
+ pub fn from_color_space(space: ffmpeg::util::color::Space) -> Self {
+ match space {
+ ffmpeg::util::color::Space::BT709 => Self::Bt709,
+ _ => Self::Bt601,
+ }
+ }
+
+ /// Limited-range Y'CbCr to RGB in 8.8 fixed point: `c = Y - 16`,
+ /// `d = Cb - 128`, `e = Cr - 128`. Luma gain 255/219, chroma gains from
+ /// the matrix's Kr and Kb (BT.601 0.299/0.114, BT.709 0.2126/0.0722).
+ #[inline(always)]
+ pub(crate) fn ycbcr_to_rgb(self, c: i32, d: i32, e: i32) -> [u8; 3] {
+ let (rv, gu, gv, bu) = match self {
+ Self::Bt601 => (409, 100, 208, 516),
+ Self::Bt709 => (459, 55, 136, 541),
+ };
+ [
+ clamp_u8((298 * c + rv * e + 128) >> 8),
+ clamp_u8((298 * c - gu * d - gv * e + 128) >> 8),
+ clamp_u8((298 * c + bu * d + 128) >> 8),
+ ]
+ }
+
+ /// Floating-point coefficients for the GPU conversion shaders (pipeline
+ /// override constants of nv12_to_rgba.wgsl and yuv420p_to_rgba.wgsl):
+ /// R = Y + rv*V, G = Y - gu*U - gv*V, B = Y + bu*U.
+ pub(crate) fn shader_constants(self) -> [(&'static str, f64); 4] {
+ let (rv, gu, gv, bu) = match self {
+ Self::Bt601 => (1.596, 0.391, 0.813, 2.018),
+ Self::Bt709 => (1.792741, 0.213249, 0.532909, 2.112402),
+ };
+ [("RV", rv), ("GU", gu), ("GV", gv), ("BU", bu)]
+ }
+}
+
pub struct ConversionProgress {
pub rows_completed: AtomicUsize,
pub total_rows: usize,
@@ -32,6 +79,36 @@ impl ConversionProgress {
}
}
+// RGB to limited-range BT.709 Y'CbCr in 16.16 fixed point, rounded to the
+// nearest code value: the matrix and range every export is tagged with. The
+// luma row sums to 219/255 and each chroma row to 0, so the grey axis is
+// exact (black 16/128/128, white 235/128/128).
+const BT709_Y: [i32; 3] = [11966, 40254, 4064];
+const BT709_CB: [i32; 3] = [-6596, -22189, 28785];
+const BT709_CR: [i32; 3] = [28785, -26145, -2640];
+
+#[inline(always)]
+fn fixed_16(row: [i32; 3], r: i32, g: i32, b: i32) -> i32 {
+ (row[0] * r + row[1] * g + row[2] * b + (1 << 15)) >> 16
+}
+
+/// BT.709 limited-range luma of an 8-bit RGB pixel.
+#[inline(always)]
+pub(crate) fn bt709_luma(r: i32, g: i32, b: i32) -> u8 {
+ (16 + fixed_16(BT709_Y, r, g, b)).clamp(16, 235) as u8
+}
+
+/// BT.709 limited-range (Cb, Cr) of an 8-bit RGB pixel (for 4:2:0, the mean
+/// of the 2x2 block).
+#[inline(always)]
+pub(crate) fn bt709_chroma(r: i32, g: i32, b: i32) -> [u8; 2] {
+ [
+ (128 + fixed_16(BT709_CB, r, g, b)).clamp(16, 240) as u8,
+ (128 + fixed_16(BT709_CR, r, g, b)).clamp(16, 240) as u8,
+ ]
+}
+
+/// BT.601 [`nv12_to_rgba_with_matrix`], the conversion this module always did.
pub fn nv12_to_rgba(
y_data: &[u8],
uv_data: &[u8],
@@ -40,6 +117,55 @@ pub fn nv12_to_rgba(
y_stride: u32,
uv_stride: u32,
output: &mut [u8],
+) {
+ nv12_to_rgba_with_matrix(
+ y_data,
+ uv_data,
+ width,
+ height,
+ y_stride,
+ uv_stride,
+ output,
+ YuvMatrix::Bt601,
+ );
+}
+
+/// BT.601 [`yuv420p_to_rgba_with_matrix`], the conversion this module always
+/// did.
+#[allow(clippy::too_many_arguments)]
+pub fn yuv420p_to_rgba(
+ y_data: &[u8],
+ u_data: &[u8],
+ v_data: &[u8],
+ width: u32,
+ height: u32,
+ y_stride: u32,
+ uv_stride: u32,
+ output: &mut [u8],
+) {
+ yuv420p_to_rgba_with_matrix(
+ y_data,
+ u_data,
+ v_data,
+ width,
+ height,
+ y_stride,
+ uv_stride,
+ output,
+ YuvMatrix::Bt601,
+ );
+}
+
+#[allow(clippy::too_many_arguments)]
+pub fn nv12_to_rgba_with_matrix(
+ y_data: &[u8],
+ uv_data: &[u8],
+ width: u32,
+ height: u32,
+ y_stride: u32,
+ uv_stride: u32,
+ output: &mut [u8],
+ matrix: YuvMatrix,
) {
let width = width as usize;
let height = height as usize;
@@ -63,9 +189,7 @@ pub fn nv12_to_rgba(
let d = u - 128;
let e = v - 128;
- let r = clamp_u8((298 * c + 409 * e + 128) >> 8);
- let g = clamp_u8((298 * c - 100 * d - 208 * e + 128) >> 8);
- let b = clamp_u8((298 * c + 516 * d + 128) >> 8);
+ let [r, g, b] = matrix.ycbcr_to_rgb(c, d, e);
let out_idx = out_row_start + col * 4;
if out_idx + 3 < output.len() {
@@ -79,7 +203,7 @@ pub fn nv12_to_rgba(
}
#[allow(clippy::too_many_arguments)]
-pub fn yuv420p_to_rgba(
+pub fn yuv420p_to_rgba_with_matrix(
y_data: &[u8],
u_data: &[u8],
v_data: &[u8],
@@ -88,6 +212,7 @@ pub fn yuv420p_to_rgba(
y_stride: u32,
uv_stride: u32,
output: &mut [u8],
+ matrix: YuvMatrix,
) {
let width = width as usize;
let height = height as usize;
@@ -111,9 +236,7 @@ pub fn yuv420p_to_rgba(
let d = u - 128;
let e = v - 128;
- let r = clamp_u8((298 * c + 409 * e + 128) >> 8);
- let g = clamp_u8((298 * c - 100 * d - 208 * e + 128) >> 8);
- let b = clamp_u8((298 * c + 516 * d + 128) >> 8);
+ let [r, g, b] = matrix.ycbcr_to_rgb(c, d, e);
let out_idx = out_row_start + col * 4;
if out_idx + 3 < output.len() {
@@ -173,9 +296,10 @@ pub fn nv12_to_rgba_simd(
y_stride: u32,
uv_stride: u32,
output: &mut [u8],
+ matrix: YuvMatrix,
) {
nv12_to_rgba_simd_with_progress(
- y_data, uv_data, width, height, y_stride, uv_stride, output, None,
+ y_data, uv_data, width, height, y_stride, uv_stride, output, None, matrix,
);
}
@@ -190,6 +314,7 @@ pub fn nv12_to_rgba_simd_with_progress(
uv_stride: u32,
output: &mut [u8],
progress: Option<Arc<ConversionProgress>>,
+ matrix: YuvMatrix,
) {
let width_usize = width as usize;
let height_usize = height as usize;
@@ -220,7 +345,9 @@ pub fn nv12_to_rgba_simd_with_progress(
|| uv_data.len() < uv_required
|| output.len() < output_required
{
- return nv12_to_rgba(y_data, uv_data, width, height, y_stride, uv_stride, output);
+ return nv12_to_rgba_with_matrix(
+ y_data, uv_data, width, height, y_stride, uv_stride, output, matrix,
+ );
}
let simd_level = SimdLevel::detect();
@@ -238,6 +365,7 @@ pub fn nv12_to_rgba_simd_with_progress(
output,
simd_level,
progress,
+ matrix,
);
} else {
nv12_convert_sequential(
@@ -250,6 +378,7 @@ pub fn nv12_to_rgba_simd_with_progress(
output,
simd_level,
progress,
+ matrix,
);
}
}
@@ -266,6 +395,7 @@ fn nv12_convert_sequential(
output: &mut [u8],
simd_level: SimdLevel,
progress: Option<Arc<ConversionProgress>>,
+ matrix: YuvMatrix,
) {
for row in 0..height {
if let Some(ref p) = progress
@@ -275,7 +405,7 @@ fn nv12_convert_sequential(
}
nv12_convert_row(
- y_data, uv_data, width, row, y_stride, uv_stride, output, simd_level,
+ y_data, uv_data, width, row, y_stride, uv_stride, output, simd_level, matrix,
);
if let Some(ref p) = progress {
@@ -296,6 +426,7 @@ fn nv12_convert_parallel(
output: &mut [u8],
simd_level: SimdLevel,
progress: Option<Arc<ConversionProgress>>,
+ matrix: YuvMatrix,
) {
use rayon::prelude::*;
@@ -332,6 +463,7 @@ fn nv12_convert_parallel(
band_output,
local_row,
simd_level,
+ matrix,
);
if let Some(ref p) = progress {
@@ -352,9 +484,10 @@ fn nv12_convert_row(
uv_stride: usize,
output: &mut [u8],
simd_level: SimdLevel,
+ matrix: YuvMatrix,
) {
nv12_convert_row_into(
- y_data, uv_data, width, row, y_stride, uv_stride, output, row, simd_level,
+ y_data, uv_data, width, row, y_stride, uv_stride, output, row, simd_level, matrix,
);
}
@@ -370,6 +503,7 @@ fn nv12_convert_row_into(
output: &mut [u8],
dst_row: usize,
simd_level: SimdLevel,
+ matrix: YuvMatrix,
) {
let y_row_start = src_row * y_stride;
let uv_row_start = (src_row / 2) * uv_stride;
@@ -385,6 +519,7 @@ fn nv12_convert_row_into(
uv_row_start,
out_row_start,
output,
+ matrix,
);
},
SimdLevel::Sse2 => unsafe {
@@ -396,6 +531,7 @@ fn nv12_convert_row_into(
uv_row_start,
out_row_start,
output,
+ matrix,
);
},
SimdLevel::Scalar => {
@@ -407,6 +543,7 @@ fn nv12_convert_row_into(
uv_row_start,
out_row_start,
output,
+ matrix,
);
}
}
@@ -422,6 +559,7 @@ unsafe fn nv12_convert_row_avx2(
uv_row_start: usize,
out_row_start: usize,
output: &mut [u8],
+ matrix: YuvMatrix,
) {
unsafe {
nv12_convert_row_sse2(
@@ -432,6 +570,7 @@ unsafe fn nv12_convert_row_avx2(
uv_row_start,
out_row_start,
output,
+ matrix,
);
}
}
@@ -446,6 +585,7 @@ unsafe fn nv12_convert_row_sse2(
uv_row_start: usize,
out_row_start: usize,
output: &mut [u8],
+ matrix: YuvMatrix,
) {
nv12_convert_row_scalar(
y_data,
@@ -455,6 +595,7 @@ unsafe fn nv12_convert_row_sse2(
uv_row_start,
out_row_start,
output,
+ matrix,
);
}
@@ -467,6 +608,7 @@ fn nv12_convert_row_scalar(
uv_row_start: usize,
out_row_start: usize,
output: &mut [u8],
+ matrix: YuvMatrix,
) {
for col in 0..width {
let y_idx = y_row_start + col;
@@ -480,9 +622,7 @@ fn nv12_convert_row_scalar(
let d = u - 128;
let e = v - 128;
- let r = clamp_u8((298 * c + 409 * e + 128) >> 8);
- let g = clamp_u8((298 * c - 100 * d - 208 * e + 128) >> 8);
- let b = clamp_u8((298 * c + 516 * d + 128) >> 8);
+ let [r, g, b] = matrix.ycbcr_to_rgb(c, d, e);
let out_idx = out_row_start + col * 4;
if out_idx + 3 < output.len() {
@@ -495,6 +635,7 @@ fn nv12_convert_row_scalar(
}
#[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))]
+#[allow(clippy::too_many_arguments)]
pub fn nv12_to_rgba_simd(
y_data: &[u8],
uv_data: &[u8],
@@ -503,8 +644,11 @@ pub fn nv12_to_rgba_simd(
y_stride: u32,
uv_stride: u32,
output: &mut [u8],
+ matrix: YuvMatrix,
) {
- nv12_to_rgba(y_data, uv_data, width, height, y_stride, uv_stride, output);
+ nv12_to_rgba_with_matrix(
+ y_data, uv_data, width, height, y_stride, uv_stride, output, matrix,
+ );
}
#[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))]
@@ -518,8 +662,11 @@ pub fn nv12_to_rgba_simd_with_progress(
uv_stride: u32,
output: &mut [u8],
_progress: Option<Arc<ConversionProgress>>,
+ matrix: YuvMatrix,
) {
- nv12_to_rgba(y_data, uv_data, width, height, y_stride, uv_stride, output);
+ nv12_to_rgba_with_matrix(
+ y_data, uv_data, width, height, y_stride, uv_stride, output, matrix,
+ );
}
#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
@@ -533,9 +680,10 @@ pub fn yuv420p_to_rgba_simd(
y_stride: u32,
uv_stride: u32,
output: &mut [u8],
+ matrix: YuvMatrix,
) {
yuv420p_to_rgba_simd_with_progress(
- y_data, u_data, v_data, width, height, y_stride, uv_stride, output, None,
+ y_data, u_data, v_data, width, height, y_stride, uv_stride, output, None, matrix,
);
}
@@ -551,6 +699,7 @@ pub fn yuv420p_to_rgba_simd_with_progress(
uv_stride: u32,
output: &mut [u8],
progress: Option<Arc<ConversionProgress>>,
+ matrix: YuvMatrix,
) {
let width_usize = width as usize;
let height_usize = height as usize;
@@ -582,8 +731,8 @@ pub fn yuv420p_to_rgba_simd_with_progress(
|| v_data.len() < uv_required
|| output.len() < output_required
{
- return yuv420p_to_rgba(
- y_data, u_data, v_data, width, height, y_stride, uv_stride, output,
+ return yuv420p_to_rgba_with_matrix(
+ y_data, u_data, v_data, width, height, y_stride, uv_stride, output, matrix,
);
}
@@ -603,6 +752,7 @@ pub fn yuv420p_to_rgba_simd_with_progress(
output,
simd_level,
progress,
+ matrix,
);
} else {
yuv420p_convert_sequential(
@@ -616,6 +766,7 @@ pub fn yuv420p_to_rgba_simd_with_progress(
output,
simd_level,
progress,
+ matrix,
);
}
}
@@ -633,6 +784,7 @@ fn yuv420p_convert_sequential(
output: &mut [u8],
simd_level: SimdLevel,
progress: Option<Arc<ConversionProgress>>,
+ matrix: YuvMatrix,
) {
for row in 0..height {
if let Some(ref p) = progress
@@ -642,7 +794,7 @@ fn yuv420p_convert_sequential(
}
yuv420p_convert_row(
- y_data, u_data, v_data, width, row, y_stride, uv_stride, output, simd_level,
+ y_data, u_data, v_data, width, row, y_stride, uv_stride, output, simd_level, matrix,
);
if let Some(ref p) = progress {
@@ -664,6 +816,7 @@ fn yuv420p_convert_parallel(
output: &mut [u8],
simd_level: SimdLevel,
progress: Option<Arc<ConversionProgress>>,
+ matrix: YuvMatrix,
) {
use rayon::prelude::*;
@@ -701,6 +854,7 @@ fn yuv420p_convert_parallel(
band_output,
local_row,
simd_level,
+ matrix,
);
if let Some(ref p) = progress {
@@ -722,9 +876,10 @@ fn yuv420p_convert_row(
uv_stride: usize,
output: &mut [u8],
simd_level: SimdLevel,
+ matrix: YuvMatrix,
) {
yuv420p_convert_row_into(
- y_data, u_data, v_data, width, row, y_stride, uv_stride, output, row, simd_level,
+ y_data, u_data, v_data, width, row, y_stride, uv_stride, output, row, simd_level, matrix,
);
}
@@ -741,6 +896,7 @@ fn yuv420p_convert_row_into(
output: &mut [u8],
dst_row: usize,
simd_level: SimdLevel,
+ matrix: YuvMatrix,
) {
let y_row_start = src_row * y_stride;
let uv_row_start = (src_row / 2) * uv_stride;
@@ -757,6 +913,7 @@ fn yuv420p_convert_row_into(
uv_row_start,
out_row_start,
output,
+ matrix,
);
},
SimdLevel::Sse2 => unsafe {
@@ -769,6 +926,7 @@ fn yuv420p_convert_row_into(
uv_row_start,
out_row_start,
output,
+ matrix,
);
},
SimdLevel::Scalar => {
@@ -781,6 +939,7 @@ fn yuv420p_convert_row_into(
uv_row_start,
out_row_start,
output,
+ matrix,
);
}
}
@@ -798,6 +957,7 @@ unsafe fn yuv420p_convert_row_avx2(
uv_row_start: usize,
out_row_start: usize,
output: &mut [u8],
+ matrix: YuvMatrix,
) {
unsafe {
yuv420p_convert_row_sse2(
@@ -809,6 +969,7 @@ unsafe fn yuv420p_convert_row_avx2(
uv_row_start,
out_row_start,
output,
+ matrix,
);
}
}
@@ -825,6 +986,7 @@ unsafe fn yuv420p_convert_row_sse2(
uv_row_start: usize,
out_row_start: usize,
output: &mut [u8],
+ matrix: YuvMatrix,
) {
yuv420p_convert_row_scalar(
y_data,
@@ -835,6 +997,7 @@ unsafe fn yuv420p_convert_row_sse2(
uv_row_start,
out_row_start,
output,
+ matrix,
);
}
@@ -849,6 +1012,7 @@ fn yuv420p_convert_row_scalar(
uv_row_start: usize,
out_row_start: usize,
output: &mut [u8],
+ matrix: YuvMatrix,
) {
for col in 0..width {
let y_idx = y_row_start + col;
@@ -862,9 +1026,7 @@ fn yuv420p_convert_row_scalar(
let d = u - 128;
let e = v - 128;
- let r = clamp_u8((298 * c + 409 * e + 128) >> 8);
- let g = clamp_u8((298 * c - 100 * d - 208 * e + 128) >> 8);
- let b = clamp_u8((298 * c + 516 * d + 128) >> 8);
+ let [r, g, b] = matrix.ycbcr_to_rgb(c, d, e);
let out_idx = out_row_start + col * 4;
if out_idx + 3 < output.len() {
@@ -887,9 +1049,10 @@ pub fn yuv420p_to_rgba_simd(
y_stride: u32,
uv_stride: u32,
output: &mut [u8],
+ matrix: YuvMatrix,
) {
- yuv420p_to_rgba(
- y_data, u_data, v_data, width, height, y_stride, uv_stride, output,
+ yuv420p_to_rgba_with_matrix(
+ y_data, u_data, v_data, width, height, y_stride, uv_stride, output, matrix,
);
}
@@ -905,9 +1068,10 @@ pub fn yuv420p_to_rgba_simd_with_progress(
uv_stride: u32,
output: &mut [u8],
_progress: Option<Arc<ConversionProgress>>,
+ matrix: YuvMatrix,
) {
- yuv420p_to_rgba(
- y_data, u_data, v_data, width, height, y_stride, uv_stride, output,
+ yuv420p_to_rgba_with_matrix(
+ y_data, u_data, v_data, width, height, y_stride, uv_stride, output, matrix,
);
}
@@ -933,7 +1097,7 @@ mod tests {
let uv_data: Vec<u8> = vec![128, 128, 128, 128, 128, 128, 128, 128];
let mut output = vec![0u8; (width * height * 4) as usize];
- nv12_to_rgba(
+ nv12_to_rgba_with_matrix(
&y_data,
&uv_data,
width,
@@ -941,6 +1105,7 @@ mod tests {
y_stride,
uv_stride,
&mut output,
+ YuvMatrix::Bt601,
);
for pixel in output.chunks(4) {
@@ -966,7 +1131,7 @@ mod tests {
let mut output_scalar = vec![0u8; (width * height * 4) as usize];
let mut output_simd = vec![0u8; (width * height * 4) as usize];
- nv12_to_rgba(
+ nv12_to_rgba_with_matrix(
&y_data,
&uv_data,
width,
@@ -974,6 +1139,7 @@ mod tests {
y_stride,
uv_stride,
&mut output_scalar,
+ YuvMatrix::Bt601,
);
nv12_to_rgba_simd(
@@ -984,6 +1150,7 @@ mod tests {
y_stride,
uv_stride,
&mut output_simd,
+ YuvMatrix::Bt601,
);
for (i, (s, d)) in output_scalar.iter().zip(output_simd.iter()).enumerate() {
@@ -1036,7 +1203,7 @@ mod tests {
let mut output1 = vec![0u8; (width * height * 4) as usize];
let mut output2 = vec![0u8; (width * height * 4) as usize];
- nv12_to_rgba(
+ nv12_to_rgba_with_matrix(
&y_data,
&uv_data,
width,
@@ -1044,6 +1211,7 @@ mod tests {
y_stride,
uv_stride,
&mut output1,
+ YuvMatrix::Bt601,
);
nv12_to_rgba_simd(
@@ -1054,6 +1222,7 @@ mod tests {
y_stride,
uv_stride,
&mut output2,
+ YuvMatrix::Bt601,
);
for (i, (a, b)) in output1.iter().zip(output2.iter()).enumerate() {
@@ -1085,7 +1254,7 @@ mod tests {
let mut output_scalar = vec![0u8; (width * height * 4) as usize];
let mut output_simd = vec![0u8; (width * height * 4) as usize];
- yuv420p_to_rgba(
+ yuv420p_to_rgba_with_matrix(
&y_data,
&u_data,
&v_data,
@@ -1094,6 +1263,7 @@ mod tests {
y_stride,
uv_stride,
&mut output_scalar,
+ YuvMatrix::Bt601,
);
yuv420p_to_rgba_simd(
@@ -1105,6 +1275,7 @@ mod tests {
y_stride,
uv_stride,
&mut output_simd,
+ YuvMatrix::Bt601,
);
for (i, (s, d)) in output_scalar.iter().zip(output_simd.iter()).enumerate() {
@@ -1138,6 +1309,7 @@ mod tests {
y_stride,
uv_stride,
&mut output,
+ YuvMatrix::Bt601,
);
assert!(output.iter().any(|&x| x != 0));
@@ -1168,9 +1340,171 @@ mod tests {
uv_stride,
&mut output,
Some(progress.clone()),
+ YuvMatrix::Bt601,
);
let rows_done = progress.rows_completed.load(Ordering::Relaxed);
assert!(rows_done < height as usize);
}
+
+ /// Exact limited-range Y'CbCr of an 8-bit RGB colour for (Kr, Kb).
+ fn reference_ycbcr(rgb: [u8; 3], kr: f64, kb: f64) -> [f64; 3] {
+ let [r, g, b] = rgb.map(|v| f64::from(v) / 255.0);
+ let y = kr * r + (1.0 - kr - kb) * g + kb * b;
+ [
+ 16.0 + 219.0 * y,
+ 128.0 + 224.0 * (b - y) / (2.0 * (1.0 - kb)),
+ 128.0 + 224.0 * (r - y) / (2.0 * (1.0 - kr)),
+ ]
+ }
+
+ fn encode_bt709(rgb: [u8; 3]) -> [u8; 3] {
+ let [r, g, b] = rgb.map(i32::from);
+ let [cb, cr] = bt709_chroma(r, g, b);
+ [bt709_luma(r, g, b), cb, cr]
+ }
+
+ const BT709: (f64, f64) = (0.2126, 0.0722);
+
+ #[test]
+ fn bt709_encode_matches_the_colour_chart() {
+ // (RGB, BT.709 limited-range Y'CbCr): primaries, secondaries, the
+ // ends of the grey axis and the suite's red background.
+ let chart: [([u8; 3], [u8; 3]); 9] = [
+ ([0, 0, 0], [16, 128, 128]),
+ ([255, 255, 255], [235, 128, 128]),
+ ([255, 0, 0], [63, 102, 240]),
+ ([0, 255, 0], [173, 42, 26]),
+ ([0, 0, 255], [32, 240, 118]),
+ ([0, 255, 255], [188, 154, 16]),
+ ([255, 0, 255], [78, 214, 230]),
+ ([255, 255, 0], [219, 16, 138]),
+ ([220, 40, 40], [83, 110, 207]),
+ ];
+ for (rgb, expected) in chart {
+ assert_eq!(encode_bt709(rgb), expected, "{rgb:?}");
+ }
+ }
+
+ #[test]
+ fn bt709_encode_rounds_to_the_nearest_code_value() {
+ for r in (0..=255u8).step_by(5) {
+ for g in (0..=255u8).step_by(5) {
+ for b in (0..=255u8).step_by(5) {
+ let reference = reference_ycbcr([r, g, b], BT709.0, BT709.1);
+ let actual = encode_bt709([r, g, b]);
+ for k in 0..3 {
+ let error = (f64::from(actual[k]) - reference[k]).abs();
+ assert!(
+ error <= 0.5 + 1.0 / 256.0,
+ "{:?} component {k}: {} vs {:.3}",
+ [r, g, b],
+ actual[k],
+ reference[k]
+ );
+ }
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn bt709_encode_keeps_the_grey_axis_neutral_and_exact() {
+ for v in 0..=255u8 {
+ let expected_y = (16.0 + 219.0 * f64::from(v) / 255.0).round() as u8;
+ assert_eq!(encode_bt709([v, v, v]), [expected_y, 128, 128], "grey {v}");
+ }
+ }
+
+ #[test]
+ fn bt709_decode_inverts_the_bt709_encode() {
+ let mut worst = 0u8;
+ for r in (0..=255u8).step_by(5) {
+ for g in (0..=255u8).step_by(5) {
+ for b in (0..=255u8).step_by(5) {
+ let [y, cb, cr] = encode_bt709([r, g, b]).map(i32::from);
+ let decoded = YuvMatrix::Bt709.ycbcr_to_rgb(y - 16, cb - 128, cr - 128);
+ for (a, e) in decoded.iter().zip([r, g, b]) {
+ worst = worst.max(a.abs_diff(e));
+ }
+ }
+ }
+ }
+ assert!(worst <= 2, "BT.709 round trip off by {worst}");
+ }
+
+ #[test]
+ fn decoding_with_the_other_matrix_shifts_saturated_colours() {
+ // The bug this guards: BT.709 data read back as BT.601 moves the
+ // suite's red background by more than the round-trip error.
+ let [y, cb, cr] = encode_bt709([220, 40, 40]).map(i32::from);
+ let wrong = YuvMatrix::Bt601.ycbcr_to_rgb(y - 16, cb - 128, cr - 128);
+ let right = YuvMatrix::Bt709.ycbcr_to_rgb(y - 16, cb - 128, cr - 128);
+ assert!(
+ right
+ .iter()
+ .zip([220u8, 40, 40])
+ .all(|(a, e)| a.abs_diff(e) <= 1)
+ );
+ assert!(
+ wrong
+ .iter()
+ .zip([220u8, 40, 40])
+ .any(|(a, e)| a.abs_diff(e) > 10)
+ );
+ }
+
+ #[test]
+ fn bt601_decode_is_unchanged() {
+ // The integer BT.601 conversion this module always used.
+ let legacy = |c: i32, d: i32, e: i32| {
+ [
+ clamp_u8((298 * c + 409 * e + 128) >> 8),
+ clamp_u8((298 * c - 100 * d - 208 * e + 128) >> 8),
+ clamp_u8((298 * c + 516 * d + 128) >> 8),
+ ]
+ };
+ for y in (0..=255).step_by(3) {
+ for u in (0..=255).step_by(3) {
+ for v in (0..=255).step_by(3) {
+ let (c, d, e) = (y - 16, u - 128, v - 128);
+ assert_eq!(YuvMatrix::Bt601.ycbcr_to_rgb(c, d, e), legacy(c, d, e));
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn matrix_follows_the_colour_tag() {
+ use ffmpeg::util::color::Space;
+ assert_eq!(YuvMatrix::from_color_space(Space::BT709), YuvMatrix::Bt709);
+ for space in [
+ Space::Unspecified,
+ Space::BT470BG,
+ Space::SMPTE170M,
+ Space::RGB,
+ ] {
+ assert_eq!(YuvMatrix::from_color_space(space), YuvMatrix::Bt601);
+ }
+ }
+
+ #[test]
+ fn simd_conversion_uses_the_requested_matrix() {
+ let (width, height) = (64u32, 4u32);
+ let y = vec![81u8; (width * height) as usize];
+ let uv: Vec<u8> = (0..(width * height / 2))
+ .map(|i| if i % 2 == 0 { 90 } else { 240 })
+ .collect();
+ for matrix in [YuvMatrix::Bt601, YuvMatrix::Bt709] {
+ let mut simd = vec![0u8; (width * height * 4) as usize];
+ let mut scalar = simd.clone();
+ nv12_to_rgba_simd(&y, &uv, width, height, width, width, &mut simd, matrix);
+ nv12_to_rgba_with_matrix(&y, &uv, width, height, width, width, &mut scalar, matrix);
+ assert_eq!(simd, scalar);
+ assert_eq!(
+ &simd[..3],
+ &matrix.ycbcr_to_rgb(81 - 16, 90 - 128, 240 - 128)
+ );
+ }
+ }
}
diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs
index af79c9560..0703321ca 100644
--- a/crates/rendering/src/decoder/avassetreader.rs
+++ b/crates/rendering/src/decoder/avassetreader.rs
@@ -21,7 +21,7 @@ use tokio::{
sync::{OwnedSemaphorePermit, Semaphore, oneshot},
};
-use crate::{DecodedFrame, PixelFormat};
+use crate::{DecodedFrame, PixelFormat, cpu_yuv::YuvMatrix};
use super::frame_converter::{copy_bgra_to_rgba, copy_rgba_plane};
use super::multi_position::{DecoderPoolManager, MultiPositionDecoderConfig, ScrubDetector};
@@ -100,11 +100,13 @@ impl ProcessedFrame {
#[derive(Clone)]
struct CachedFrame(ProcessedFrame);
-struct ImageBufProcessor;
+struct ImageBufProcessor {
+ matrix: YuvMatrix,
+}
impl ImageBufProcessor {
- fn new() -> Self {
- Self
+ fn new(matrix: YuvMatrix) -> Self {
+ Self { matrix }
}
fn extract_raw(&self, image_buf: &mut R<cv::ImageBuf>) -> (Vec<u8>, PixelFormat, u32, u32) {
@@ -265,7 +267,10 @@ impl CachedFrame {
}
}
};
- Self(ProcessedFrame { number, frame })
+ Self(ProcessedFrame {
+ number,
+ frame: frame.with_matrix(processor.matrix),
+ })
}
_ => {
let black_frame = vec![0u8; (width * height * 4) as usize];
@@ -650,9 +655,13 @@ impl AVAssetReaderDecoder {
if video.width() == 0 || video.height() == 0 {
return Err("Invalid video dimensions".to_string());
}
- Ok((video.width(), video.height()))
+ Ok((
+ video.width(),
+ video.height(),
+ YuvMatrix::from_color_space(video.color_space()),
+ ))
})();
- let (width, height) = match dimensions {
+ let (width, height, matrix) = match dimensions {
Ok(dimensions) => dimensions,
Err(error) => {
let _ = ready_tx.send(Err(error));
@@ -688,7 +697,7 @@ impl AVAssetReaderDecoder {
// pre-hole frame leaves the cache.
let mut gap_hold: Option<ProcessedFrame> = None;
- let processor = ImageBufProcessor::new();
+ let processor = ImageBufProcessor::new(matrix);
struct PendingRequest {
frame: u32,
diff --git a/crates/rendering/src/decoder/ffmpeg.rs b/crates/rendering/src/decoder/ffmpeg.rs
index b06811534..ec14a4641 100644
--- a/crates/rendering/src/decoder/ffmpeg.rs
+++ b/crates/rendering/src/decoder/ffmpeg.rs
@@ -17,6 +17,8 @@ use crate::{DecodedFrame, PixelFormat};
#[cfg(target_os = "windows")]
use cap_video_decode::FrameTextures;
+use crate::cpu_yuv::YuvMatrix;
+
use super::{
DecoderInitResult, DecoderType, FRAME_CACHE_SIZE, ManagedVideoControl, ManagedVideoError,
ManagedVideoInput, VideoDecoderMessage, frame_converter::FrameConverter, pts_to_frame,
@@ -30,6 +32,7 @@ struct ProcessedFrame {
width: u32,
height: u32,
format: PixelFormat,
+ matrix: YuvMatrix,
y_stride: u32,
uv_stride: u32,
#[cfg(target_os = "linux")]
@@ -40,9 +43,10 @@ impl ProcessedFrame {
fn to_decoded_frame(&self) -> DecodedFrame {
#[cfg(target_os = "linux")]
if let Some(cuda) = &self.cuda {
- return DecodedFrame::new_nv12_cuda(Arc::clone(cuda), Arc::clone(&self.data));
+ return DecodedFrame::new_nv12_cuda(Arc::clone(cuda), Arc::clone(&self.data))
+ .with_matrix(self.matrix);
}
- match self.format {
+ let frame = match self.format {
PixelFormat::Rgba => {
DecodedFrame::new_with_arc(Arc::clone(&self.data), self.width, self.height)
}
@@ -60,7 +64,8 @@ impl ProcessedFrame {
self.y_stride,
self.uv_stride,
),
- }
+ };
+ frame.with_matrix(self.matrix)
}
}
@@ -163,6 +168,7 @@ impl CachedFrame {
width: frame.width(),
height: frame.height(),
format: PixelFormat::Nv12,
+ matrix: YuvMatrix::from_color_space(frame.color_space()),
y_stride: cuda.y_pitch as u32,
uv_stride: cuda.uv_pitch as u32,
cuda: Some(Arc::new(cuda)),
@@ -180,6 +186,7 @@ impl CachedFrame {
width: frame.width(),
height: frame.height(),
format: pixel_format,
+ matrix: YuvMatrix::from_color_space(frame.color_space()),
y_stride,
uv_stride,
#[cfg(target_os = "linux")]
@@ -193,6 +200,7 @@ impl CachedFrame {
width: frame.width(),
height: frame.height(),
format: PixelFormat::Rgba,
+ matrix: YuvMatrix::from_color_space(frame.color_space()),
y_stride: frame.width() * 4,
uv_stride: 0,
#[cfg(target_os = "linux")]
@@ -1698,6 +1706,7 @@ mod cache_tests {
width: 2,
height: 2,
format: PixelFormat::Nv12,
+ matrix: YuvMatrix::Bt601,
y_stride: 2,
uv_stride: 2,
#[cfg(target_os = "linux")]
diff --git a/crates/rendering/src/decoder/frame_converter.rs b/crates/rendering/src/decoder/frame_converter.rs
index 7a449ffa7..1a156314a 100644
--- a/crates/rendering/src/decoder/frame_converter.rs
+++ b/crates/rendering/src/decoder/frame_converter.rs
@@ -2,6 +2,8 @@
use ffmpeg::{format, frame, software};
+use crate::cpu_yuv::YuvMatrix;
+
pub struct FrameConverter {
scaler: Option<ScalerState>,
rgba_frame: frame::Video,
@@ -10,6 +12,7 @@ pub struct FrameConverter {
struct ScalerState {
context: software::scaling::Context,
input_format: format::Pixel,
+ matrix: YuvMatrix,
width: u32,
height: u32,
}
@@ -34,7 +37,12 @@ impl FrameConverter {
let width = frame.width();
let height = frame.height();
- self.ensure_scaler(frame.format(), width, height);
+ self.ensure_scaler(
+ frame.format(),
+ YuvMatrix::from_color_space(frame.color_space()),
+ width,
+ height,
+ );
self.ensure_rgba_frame(width, height);
{
@@ -56,16 +64,31 @@ impl FrameConverter {
)
}
- fn ensure_scaler(&mut self, input_format: format::Pixel, width: u32, height: u32) {
+ fn ensure_scaler(
+ &mut self,
+ input_format: format::Pixel,
+ matrix: YuvMatrix,
+ width: u32,
+ height: u32,
+ ) {
let needs_new = self.scaler.as_ref().is_none_or(|state| {
- state.input_format != input_format || state.width != width || state.height != height
+ state.input_format != input_format
+ || state.matrix != matrix
+ || state.width != width
+ || state.height != height
});
if needs_new {
+ let mut context =
+ software::converter((width, height), input_format, format::Pixel::RGBA)
+ .expect("failed to create frame scaler");
+ if matrix == YuvMatrix::Bt709 {
+ use_bt709_source(&mut context);
+ }
self.scaler = Some(ScalerState {
- context: software::converter((width, height), input_format, format::Pixel::RGBA)
- .expect("failed to create frame scaler"),
+ context,
input_format,
+ matrix,
width,
height,
});
@@ -80,6 +103,41 @@ impl FrameConverter {
}
}
+/// swscale converts YUV with BT.601 unless told otherwise: switch the source
+/// matrix to BT.709 and keep the ranges and adjustments it chose.
+fn use_bt709_source(context: &mut software::scaling::Context) {
+ unsafe {
+ let mut inv_table = std::ptr::null_mut();
+ let mut table = std::ptr::null_mut();
+ let (mut src_range, mut dst_range) = (0, 0);
+ let (mut brightness, mut contrast, mut saturation) = (0, 0, 0);
+ if ffmpeg::ffi::sws_getColorspaceDetails(
+ context.as_mut_ptr(),
+ &mut inv_table,
+ &mut src_range,
+ &mut table,
+ &mut dst_range,
+ &mut brightness,
+ &mut contrast,
+ &mut saturation,
+ ) < 0
+ {
+ return;
+ }
+ let bt709 = ffmpeg::ffi::sws_getCoefficients(ffmpeg::ffi::SWS_CS_ITU709);
+ ffmpeg::ffi::sws_setColorspaceDetails(
+ context.as_mut_ptr(),
+ bt709,
+ src_range,
+ table,
+ dst_range,
+ brightness,
+ contrast,
+ saturation,
+ );
+ }
+}
+
pub fn copy_rgba_plane(data: &[u8], stride: usize, width: usize, height: usize) -> Vec<u8> {
debug_assert!(stride >= width * 4, "stride too small for RGBA frame");
diff --git a/crates/rendering/src/decoder/media_foundation.rs b/crates/rendering/src/decoder/media_foundation.rs
index 1fb01862c..894d37cbe 100644
--- a/crates/rendering/src/decoder/media_foundation.rs
+++ b/crates/rendering/src/decoder/media_foundation.rs
@@ -229,6 +229,7 @@ impl MFDecoder {
let video_width = decoder.width();
let video_height = decoder.height();
+ let matrix = super::probe_yuv_matrix(&path);
let mut cache = BTreeMap::<u32, CachedFrame>::new();
let mut cache_bytes = 0usize;
@@ -281,7 +282,9 @@ impl MFDecoder {
let mut unfulfilled = Vec::with_capacity(pending_requests.len());
for req in pending_requests.drain(..) {
if let Some(frame) = cached_frame_for(&cache, last_decoded_frame, req.frame) {
- let _ = req.sender.send(frame.to_decoded_frame());
+ let _ = req
+ .sender
+ .send(frame.to_decoded_frame().with_matrix(matrix));
} else if !req.sender.is_closed() {
unfulfilled.push(req);
}
@@ -464,7 +467,7 @@ impl MFDecoder {
if let Some(frame) = frame_to_send
&& let Some(s) = sender.take()
{
- let _ = s.send(frame.to_decoded_frame());
+ let _ = s.send(frame.to_decoded_frame().with_matrix(matrix));
}
}
@@ -496,7 +499,7 @@ impl MFDecoder {
.map(|(_, f)| f.clone())
});
if let Some(frame) = fallback {
- let _ = s.send(frame.to_decoded_frame());
+ let _ = s.send(frame.to_decoded_frame().with_matrix(matrix));
} else {
let black_frame = DecodedFrame::new(
vec![0u8; (video_width * video_height * 4) as usize],
@@ -516,7 +519,9 @@ impl MFDecoder {
.or_else(|| cache.range(..=req.frame).next_back().map(|(_, f)| f))
.or(last_valid_frame.as_ref());
if let Some(frame) = frame_to_send {
- let _ = req.sender.send(frame.to_decoded_frame());
+ let _ = req
+ .sender
+ .send(frame.to_decoded_frame().with_matrix(matrix));
} else {
let black_frame = DecodedFrame::new(
vec![0u8; (video_width * video_height * 4) as usize],
diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs
index 170507f30..366a28826 100644
--- a/crates/rendering/src/decoder/mod.rs
+++ b/crates/rendering/src/decoder/mod.rs
@@ -1,3 +1,4 @@
+use crate::cpu_yuv::YuvMatrix;
use ::ffmpeg::Rational;
use std::{
fmt,
@@ -156,6 +157,7 @@ pub struct DecodedFrame {
width: u32,
height: u32,
format: PixelFormat,
+ matrix: YuvMatrix,
y_stride: u32,
uv_stride: u32,
#[cfg(target_os = "macos")]
@@ -281,6 +283,7 @@ impl DecodedFrame {
width,
height,
format: PixelFormat::Rgba,
+ matrix: YuvMatrix::Bt601,
y_stride: width * 4,
uv_stride: 0,
#[cfg(target_os = "macos")]
@@ -298,6 +301,7 @@ impl DecodedFrame {
width,
height,
format: PixelFormat::Rgba,
+ matrix: YuvMatrix::Bt601,
y_stride: width * 4,
uv_stride: 0,
#[cfg(target_os = "macos")]
@@ -322,6 +326,7 @@ impl DecodedFrame {
width: frame.width,
height: frame.height,
format: PixelFormat::Nv12,
+ matrix: YuvMatrix::Bt601,
y_stride: frame.y_pitch as u32,
uv_stride: frame.uv_pitch as u32,
cuda_nv12: Some(frame),
@@ -339,6 +344,7 @@ impl DecodedFrame {
width,
height,
format: PixelFormat::Nv12,
+ matrix: YuvMatrix::Bt601,
y_stride,
uv_stride,
#[cfg(target_os = "macos")]
@@ -362,6 +368,7 @@ impl DecodedFrame {
width,
height,
format: PixelFormat::Nv12,
+ matrix: YuvMatrix::Bt601,
y_stride,
uv_stride,
#[cfg(target_os = "macos")]
@@ -386,6 +393,7 @@ impl DecodedFrame {
width,
height,
format: PixelFormat::Nv12,
+ matrix: YuvMatrix::Bt601,
y_stride,
uv_stride,
image_buf_backing: Some(Arc::new(SendableImageBuf::new(image_buf))),
@@ -404,6 +412,7 @@ impl DecodedFrame {
width,
height,
format: PixelFormat::Yuv420p,
+ matrix: YuvMatrix::Bt601,
y_stride,
uv_stride,
#[cfg(target_os = "macos")]
@@ -427,6 +436,7 @@ impl DecodedFrame {
width,
height,
format: PixelFormat::Yuv420p,
+ matrix: YuvMatrix::Bt601,
y_stride,
uv_stride,
#[cfg(target_os = "macos")]
@@ -445,6 +455,7 @@ impl DecodedFrame {
width,
height,
format: PixelFormat::Nv12,
+ matrix: YuvMatrix::Bt601,
y_stride: width,
uv_stride: width,
#[cfg(target_os = "macos")]
@@ -465,6 +476,7 @@ impl DecodedFrame {
width,
height,
format: PixelFormat::Nv12,
+ matrix: YuvMatrix::Bt601,
y_stride: width,
uv_stride: width,
#[cfg(target_os = "macos")]
@@ -490,6 +502,7 @@ impl DecodedFrame {
width,
height,
format: PixelFormat::Nv12,
+ matrix: YuvMatrix::Bt601,
y_stride: width,
uv_stride: width,
#[cfg(target_os = "macos")]
@@ -560,6 +573,16 @@ impl DecodedFrame {
self.format
}
+ /// The Y'CbCr matrix to convert this frame's planes to RGB with.
+ pub fn matrix(&self) -> YuvMatrix {
+ self.matrix
+ }
+
+ pub fn with_matrix(mut self, matrix: YuvMatrix) -> Self {
+ self.matrix = matrix;
+ self
+ }
+
pub fn byte_len(&self) -> usize {
if !self.data.is_empty() {
return self.data.len();
@@ -663,6 +686,22 @@ pub enum VideoDecoderMessage {
GetFrame(f32, u32, tokio::sync::oneshot::Sender<DecodedFrame>),
}
+/// The source's Y'CbCr matrix from its colour tag, for decoders that do not
+/// see FFmpeg frames (Media Foundation). BT.601 when the file cannot be read.
+#[cfg(target_os = "windows")]
+pub(crate) fn probe_yuv_matrix(path: &std::path::Path) -> YuvMatrix {
+ ::ffmpeg::format::input(path)
+ .ok()
+ .and_then(|input| {
+ let stream = input.streams().best(::ffmpeg::media::Type::Video)?;
+ let context = ::ffmpeg::codec::Context::from_parameters(stream.parameters()).ok()?;
+ Some(YuvMatrix::from_color_space(
+ context.decoder().video().ok()?.color_space(),
+ ))
+ })
+ .unwrap_or_default()
+}
+
pub fn pts_to_frame(pts: i64, time_base: Rational, fps: u32) -> u32 {
(fps as f64 * ((pts as f64 * time_base.numerator() as f64) / (time_base.denominator() as f64)))
.round() as u32
diff --git a/crates/rendering/src/layers/camera.rs b/crates/rendering/src/layers/camera.rs
index 9f36daa88..9fca2068d 100644
--- a/crates/rendering/src/layers/camera.rs
+++ b/crates/rendering/src/layers/camera.rs
@@ -117,6 +117,7 @@ impl CameraLayer {
};
let format = camera_frame.format();
+ self.yuv_converter.set_matrix(camera_frame.matrix());
let frame_storage = camera_frame.storage_identity();
let is_same_frame = self
@@ -402,6 +403,7 @@ impl CameraLayer {
};
let format = camera_frame.format();
+ self.yuv_converter.set_matrix(camera_frame.matrix());
let frame_storage = camera_frame.storage_identity();
let is_same_frame = self
diff --git a/crates/rendering/src/layers/display.rs b/crates/rendering/src/layers/display.rs
index ad7575b64..21f1f9786 100644
--- a/crates/rendering/src/layers/display.rs
+++ b/crates/rendering/src/layers/display.rs
@@ -162,6 +162,7 @@ impl DisplayLayer {
let actual_height = screen_frame.height();
let source_size = XY::new(actual_width, actual_height);
let format = screen_frame.format();
+ self.yuv_converter.set_matrix(screen_frame.matrix());
let current_recording_time = segment_frames.recording_time;
let frame_storage = screen_frame.storage_identity();
@@ -578,6 +579,7 @@ impl DisplayLayer {
let actual_height = screen_frame.height();
let source_size = XY::new(actual_width, actual_height);
let format = screen_frame.format();
+ self.yuv_converter.set_matrix(screen_frame.matrix());
let current_recording_time = segment_frames.recording_time;
let frame_storage = screen_frame.storage_identity();
diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs
index dcb858929..76b7f9490 100644
--- a/crates/rendering/src/lib.rs
+++ b/crates/rendering/src/lib.rs
@@ -6911,7 +6911,7 @@ impl<'a> FrameRenderer<'a> {
let r = src_row[col * 4] as i32;
let g = src_row[col * 4 + 1] as i32;
let b = src_row[col * 4 + 2] as i32;
- y_row[col] = ((16 + ((65 * r + 129 * g + 25 * b + 128) >> 8)) as u8).clamp(16, 235);
+ y_row[col] = cpu_yuv::bt709_luma(r, g, b);
}
}
@@ -6942,10 +6942,7 @@ impl<'a> FrameRenderer<'a> {
+ src_row1[col * 8 + 6] as i32
+ 2)
/ 4;
- uv_row[col * 2] =
- ((128 + ((-38 * r - 74 * g + 112 * b + 128) >> 8)) as u8).clamp(16, 240);
- uv_row[col * 2 + 1] =
- ((128 + ((112 * r - 94 * g - 18 * b + 128) >> 8)) as u8).clamp(16, 240);
+ [uv_row[col * 2], uv_row[col * 2 + 1]] = cpu_yuv::bt709_chroma(r, g, b);
}
}
diff --git a/crates/rendering/src/shaders/nv12_to_rgba.wgsl b/crates/rendering/src/shaders/nv12_to_rgba.wgsl
index e6265357d..a6093790f 100644
--- a/crates/rendering/src/shaders/nv12_to_rgba.wgsl
+++ b/crates/rendering/src/shaders/nv12_to_rgba.wgsl
@@ -1,3 +1,11 @@
+// Limited-range Y'CbCr to RGB. The chroma coefficients are pipeline
+// overrides (YuvMatrix::shader_constants): BT.601 by default, BT.709 for
+// sources tagged BT.709.
+override RV: f32 = 1.596;
+override GU: f32 = 0.391;
+override GV: f32 = 0.813;
+override BU: f32 = 2.018;
+
@group(0) @binding(0) var y_plane: texture_2d<f32>;
@group(0) @binding(1) var uv_plane: texture_2d<f32>;
@group(0) @binding(2) var output: texture_storage_2d<rgba8unorm, write>;
@@ -22,9 +30,9 @@ fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {
let u = uv_raw.r - 0.5;
let v = uv_raw.g - 0.5;
- let r = y + 1.596 * v;
- let g = y - 0.391 * u - 0.813 * v;
- let b = y + 2.018 * u;
+ let r = y + RV * v;
+ let g = y - GU * u - GV * v;
+ let b = y + BU * u;
let color = vec4<f32>(
clamp(r, 0.0, 1.0),
diff --git a/crates/rendering/src/shaders/rgba_to_nv12.wgsl b/crates/rendering/src/shaders/rgba_to_nv12.wgsl
index 444805da2..a4aa85e7b 100644
--- a/crates/rendering/src/shaders/rgba_to_nv12.wgsl
+++ b/crates/rendering/src/shaders/rgba_to_nv12.wgsl
@@ -9,16 +9,18 @@ struct Params {
@group(0) @binding(1) var<storage, read_write> nv12_output: array<u32>;
@group(0) @binding(2) var<uniform> params: Params;
+// BT.709 limited range, the matrix every export is tagged with (see
+// crates/enc-ffmpeg/src/video/h264.rs). Rounded to the nearest code value.
fn rgb_to_y(r: f32, g: f32, b: f32) -> u32 {
- return u32(clamp(16.0 + 65.481 * r + 128.553 * g + 24.966 * b, 0.0, 255.0));
+ return u32(clamp(16.5 + 46.5594 * r + 156.6288 * g + 15.8118 * b, 0.0, 255.0));
}
fn rgb_to_u(r: f32, g: f32, b: f32) -> u32 {
- return u32(clamp(128.0 - 37.797 * r - 74.203 * g + 112.0 * b, 0.0, 255.0));
+ return u32(clamp(128.5 - 25.6642 * r - 86.3358 * g + 112.0 * b, 0.0, 255.0));
}
fn rgb_to_v(r: f32, g: f32, b: f32) -> u32 {
- return u32(clamp(128.0 + 112.0 * r - 93.786 * g - 18.214 * b, 0.0, 255.0));
+ return u32(clamp(128.5 + 112.0 * r - 101.7303 * g - 10.2697 * b, 0.0, 255.0));
}
fn safe_load(coord: vec2<u32>, dims: vec2<u32>) -> vec4<f32> {
diff --git a/crates/rendering/src/shaders/yuv420p_to_rgba.wgsl b/crates/rendering/src/shaders/yuv420p_to_rgba.wgsl
index 0d04d30fe..8838df9bc 100644
--- a/crates/rendering/src/shaders/yuv420p_to_rgba.wgsl
+++ b/crates/rendering/src/shaders/yuv420p_to_rgba.wgsl
@@ -1,3 +1,11 @@
+// Limited-range Y'CbCr to RGB. The chroma coefficients are pipeline
+// overrides (YuvMatrix::shader_constants): BT.601 by default, BT.709 for
+// sources tagged BT.709.
+override RV: f32 = 1.596;
+override GU: f32 = 0.391;
+override GV: f32 = 0.813;
+override BU: f32 = 2.018;
+
@group(0) @binding(0) var y_plane: texture_2d<f32>;
@group(0) @binding(1) var u_plane: texture_2d<f32>;
@group(0) @binding(2) var v_plane: texture_2d<f32>;
@@ -26,9 +34,9 @@ fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {
let u = u_raw - 0.5;
let v = v_raw - 0.5;
- let r = y + 1.596 * v;
- let g = y - 0.391 * u - 0.813 * v;
- let b = y + 2.018 * u;
+ let r = y + RV * v;
+ let g = y - GU * u - GV * v;
+ let b = y + BU * u;
let color = vec4<f32>(
clamp(r, 0.0, 1.0),
diff --git a/crates/rendering/src/yuv_converter.rs b/crates/rendering/src/yuv_converter.rs
index 6791c1cc5..d9c7c9ffc 100644
--- a/crates/rendering/src/yuv_converter.rs
+++ b/crates/rendering/src/yuv_converter.rs
@@ -1,4 +1,4 @@
-use crate::cpu_yuv;
+use crate::cpu_yuv::{self, YuvMatrix};
#[cfg(target_os = "macos")]
use crate::iosurface_texture::{
@@ -259,9 +259,60 @@ impl BindGroupCache {
use std::sync::Arc;
+/// One compute pipeline per source matrix, compiled on first use, so a
+/// project only ever builds the variants its sources need.
+struct MatrixPipelines {
+ device: wgpu::Device,
+ label: &'static str,
+ layout: wgpu::PipelineLayout,
+ module: wgpu::ShaderModule,
+ bt601: std::sync::OnceLock<wgpu::ComputePipeline>,
+ bt709: std::sync::OnceLock<wgpu::ComputePipeline>,
+}
+
+impl MatrixPipelines {
+ fn new(
+ device: &wgpu::Device,
+ label: &'static str,
+ layout: wgpu::PipelineLayout,
+ module: wgpu::ShaderModule,
+ ) -> Self {
+ Self {
+ device: device.clone(),
+ label,
+ layout,
+ module,
+ bt601: std::sync::OnceLock::new(),
+ bt709: std::sync::OnceLock::new(),
+ }
+ }
+
+ fn get(&self, matrix: YuvMatrix) -> &wgpu::ComputePipeline {
+ let cell = match matrix {
+ YuvMatrix::Bt601 => &self.bt601,
+ YuvMatrix::Bt709 => &self.bt709,
+ };
+ cell.get_or_init(|| {
+ let constants = matrix.shader_constants();
+ self.device
+ .create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
+ label: Some(self.label),
+ layout: Some(&self.layout),
+ module: &self.module,
+ entry_point: Some("main"),
+ compilation_options: wgpu::PipelineCompilationOptions {
+ constants: &constants,
+ ..Default::default()
+ },
+ cache: None,
+ })
+ })
+ }
+}
+
pub struct YuvConverterPipelines {
- pub nv12_pipeline: wgpu::ComputePipeline,
- pub yuv420p_pipeline: wgpu::ComputePipeline,
+ nv12: MatrixPipelines,
+ yuv420p: MatrixPipelines,
pub nv12_bind_group_layout: wgpu::BindGroupLayout,
pub yuv420p_bind_group_layout: wgpu::BindGroupLayout,
}
@@ -381,29 +432,23 @@ impl YuvConverterPipelines {
push_constant_ranges: &[],
});
- let nv12_pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
- label: Some("NV12 Converter Pipeline"),
- layout: Some(&nv12_pipeline_layout),
- module: &nv12_shader,
- entry_point: Some("main"),
- compilation_options: Default::default(),
- cache: None,
- });
-
- let yuv420p_pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
- label: Some("YUV420P Converter Pipeline"),
- layout: Some(&yuv420p_pipeline_layout),
- module: &yuv420p_shader,
- entry_point: Some("main"),
- compilation_options: Default::default(),
- cache: None,
- });
-
+ let nv12 = MatrixPipelines::new(
+ device,
+ "NV12 Converter Pipeline",
+ nv12_pipeline_layout,
+ nv12_shader,
+ );
+ let yuv420p = MatrixPipelines::new(
+ device,
+ "YUV420P Converter Pipeline",
+ yuv420p_pipeline_layout,
+ yuv420p_shader,
+ );
tracing::info!("Shared YUV converter pipelines created successfully");
Self {
- nv12_pipeline,
- yuv420p_pipeline,
+ nv12,
+ yuv420p,
nv12_bind_group_layout,
yuv420p_bind_group_layout,
}
@@ -431,6 +476,7 @@ unsafe impl Sync for RetainedImageBuf {}
pub struct YuvToRgbaConverter {
pipelines: Arc<YuvConverterPipelines>,
+ matrix: YuvMatrix,
y_texture: wgpu::Texture,
y_view: wgpu::TextureView,
uv_texture: wgpu::Texture,
@@ -467,6 +513,11 @@ pub struct YuvToRgbaConverter {
}
impl YuvToRgbaConverter {
+ /// The matrix of the frames converted next (see [`YuvMatrix`]).
+ pub fn set_matrix(&mut self, matrix: YuvMatrix) {
+ self.matrix = matrix;
+ }
+
pub fn new(device: &wgpu::Device) -> Self {
let pipelines = Arc::new(YuvConverterPipelines::new(device));
Self::new_with_shared_pipelines(device, pipelines)
@@ -494,6 +545,7 @@ impl YuvToRgbaConverter {
Self::create_output_textures(device, initial_width, initial_height);
Self {
+ matrix: YuvMatrix::default(),
pipelines,
y_texture,
y_view,
@@ -656,7 +708,7 @@ impl YuvToRgbaConverter {
label: Some("NV12 CUDA Conversion Pass"),
..Default::default()
});
- compute_pass.set_pipeline(&self.pipelines.nv12_pipeline);
+ compute_pass.set_pipeline(self.pipelines.nv12.get(self.matrix));
compute_pass.set_bind_group(0, bind_group, &[]);
compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);
}
@@ -939,7 +991,7 @@ impl YuvToRgbaConverter {
label: Some("NV12 Conversion Pass"),
..Default::default()
});
- compute_pass.set_pipeline(&self.pipelines.nv12_pipeline);
+ compute_pass.set_pipeline(self.pipelines.nv12.get(self.matrix));
compute_pass.set_bind_group(0, bind_group, &[]);
compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);
}
@@ -1024,7 +1076,7 @@ impl YuvToRgbaConverter {
label: Some("NV12 Conversion Pass (Batched)"),
..Default::default()
});
- compute_pass.set_pipeline(&self.pipelines.nv12_pipeline);
+ compute_pass.set_pipeline(self.pipelines.nv12.get(self.matrix));
compute_pass.set_bind_group(0, bind_group, &[]);
compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);
}
@@ -1121,7 +1173,7 @@ impl YuvToRgbaConverter {
label: Some("NV12 IOSurface Conversion Pass (Batched)"),
..Default::default()
});
- compute_pass.set_pipeline(&self.pipelines.nv12_pipeline);
+ compute_pass.set_pipeline(self.pipelines.nv12.get(self.matrix));
compute_pass.set_bind_group(0, &bind_group, &[]);
compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);
}
@@ -1202,7 +1254,7 @@ impl YuvToRgbaConverter {
label: Some("YUV420P Conversion Pass"),
..Default::default()
});
- compute_pass.set_pipeline(&self.pipelines.yuv420p_pipeline);
+ compute_pass.set_pipeline(self.pipelines.yuv420p.get(self.matrix));
compute_pass.set_bind_group(0, bind_group, &[]);
compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);
}
@@ -1273,7 +1325,7 @@ impl YuvToRgbaConverter {
label: Some("YUV420P Conversion Pass (Batched)"),
..Default::default()
});
- compute_pass.set_pipeline(&self.pipelines.yuv420p_pipeline);
+ compute_pass.set_pipeline(self.pipelines.yuv420p.get(self.matrix));
compute_pass.set_bind_group(0, bind_group, &[]);
compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);
}
@@ -1434,7 +1486,7 @@ impl YuvToRgbaConverter {
label: Some("NV12 D3D11 Conversion Pass"),
..Default::default()
});
- compute_pass.set_pipeline(&self.pipelines.nv12_pipeline);
+ compute_pass.set_pipeline(self.pipelines.nv12.get(self.matrix));
compute_pass.set_bind_group(0, &bind_group, &[]);
compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);
}
@@ -1528,7 +1580,7 @@ impl YuvToRgbaConverter {
label: Some("NV12 D3D11 Zero-Copy Conversion Pass"),
..Default::default()
});
- compute_pass.set_pipeline(&self.pipelines.nv12_pipeline);
+ compute_pass.set_pipeline(self.pipelines.nv12.get(self.matrix));
compute_pass.set_bind_group(0, &bind_group, &[]);
compute_pass.dispatch_workgroups(width.div_ceil(8), height.div_ceil(8), 1);
}
@@ -1639,6 +1691,7 @@ impl YuvToRgbaConverter {
y_stride,
uv_stride,
&mut rgba_data,
+ self.matrix,
);
queue.write_texture(
@@ -1693,6 +1746,7 @@ impl YuvToRgbaConverter {
y_stride,
uv_stride,
&mut rgba_data,
+ self.matrix,
);
queue.write_texture(
@@ -1722,3 +1776,222 @@ impl YuvToRgbaConverter {
Some(self.current_output_texture())
}
}
+
+/// The GPU conversions against the CPU reference in `cpu_yuv`: the export's
+/// RGBA to NV12 shader must produce BT.709 limited range (the matrix the
+/// encoders tag), and the decode shaders must apply the matrix they are asked
+/// for. Skipped where no adapter exists.
+#[cfg(test)]
+mod colour_matrix_tests {
+ use super::*;
+ use crate::frame_pipeline::RgbaToNv12Converter;
+
+ // Primaries, secondaries, grey axis and the suite's red background.
+ const CHART: [[u8; 3]; 8] = [
+ [0, 0, 0],
+ [255, 255, 255],
+ [128, 128, 128],
+ [255, 0, 0],
+ [0, 255, 0],
+ [0, 0, 255],
+ [255, 255, 0],
+ [220, 40, 40],
+ ];
+ // Each chart colour fills a 4x2 block, so chroma averaging is exact.
+ const WIDTH: u32 = 4 * CHART.len() as u32;
+ const HEIGHT: u32 = 2;
+
+ fn device() -> Option<(wgpu::Device, wgpu::Queue)> {
+ let instance = crate::create_wgpu_instance_sync();
+ let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {
+ power_preference: wgpu::PowerPreference::LowPower,
+ force_fallback_adapter: false,
+ compatible_surface: None,
+ }))
+ .ok()?;
+ pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()
+ }
+
+ fn reference_bt709(rgb: [u8; 3]) -> [f64; 3] {
+ let (kr, kb) = (0.2126, 0.0722);
+ let [r, g, b] = rgb.map(|v| f64::from(v) / 255.0);
+ let y = kr * r + (1.0 - kr - kb) * g + kb * b;
+ [
+ 16.0 + 219.0 * y,
+ 128.0 + 224.0 * (b - y) / (2.0 * (1.0 - kb)),
+ 128.0 + 224.0 * (r - y) / (2.0 * (1.0 - kr)),
+ ]
+ }
+
+ fn chart_texture(device: &wgpu::Device, queue: &wgpu::Queue) -> wgpu::Texture {
+ let texture = device.create_texture(&wgpu::TextureDescriptor {
+ label: Some("colour chart"),
+ size: wgpu::Extent3d {
+ width: WIDTH,
+ height: HEIGHT,
+ depth_or_array_layers: 1,
+ },
+ mip_level_count: 1,
+ sample_count: 1,
+ dimension: wgpu::TextureDimension::D2,
+ format: wgpu::TextureFormat::Rgba8Unorm,
+ usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+ view_formats: &[],
+ });
+ let data: Vec<u8> = (0..HEIGHT)
+ .flat_map(|_| {
+ CHART
+ .iter()
+ .flat_map(|[r, g, b]| [*r, *g, *b, 255].repeat(4))
+ })
+ .collect();
+ queue.write_texture(
+ texture.as_image_copy(),
+ &data,
+ wgpu::TexelCopyBufferLayout {
+ offset: 0,
+ bytes_per_row: Some(WIDTH * 4),
+ rows_per_image: None,
+ },
+ wgpu::Extent3d {
+ width: WIDTH,
+ height: HEIGHT,
+ depth_or_array_layers: 1,
+ },
+ );
+ texture
+ }
+
+ /// The chart as BT.709 NV12 from the CPU encoder (tight strides).
+ fn chart_nv12() -> (Vec<u8>, Vec<u8>) {
+ let mut y = Vec::new();
+ let mut uv = Vec::new();
+ for _ in 0..HEIGHT {
+ for [r, g, b] in CHART.map(|c| c.map(i32::from)) {
+ y.extend([cpu_yuv::bt709_luma(r, g, b); 4]);
+ }
+ }
+ for [r, g, b] in CHART.map(|c| c.map(i32::from)) {
+ let [cb, cr] = cpu_yuv::bt709_chroma(r, g, b);
+ uv.extend([cb, cr, cb, cr]);
+ }
+ (y, uv)
+ }
+
+ fn read_rgba(device: &wgpu::Device, queue: &wgpu::Queue, texture: &wgpu::Texture) -> Vec<u8> {
+ let padded = wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;
+ let buffer = device.create_buffer(&wgpu::BufferDescriptor {
+ label: Some("chart readback"),
+ size: u64::from(padded * HEIGHT),
+ usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
+ mapped_at_creation: false,
+ });
+ let mut encoder = device.create_command_encoder(&Default::default());
+ encoder.copy_texture_to_buffer(
+ texture.as_image_copy(),
+ wgpu::TexelCopyBufferInfo {
+ buffer: &buffer,
+ layout: wgpu::TexelCopyBufferLayout {
+ offset: 0,
+ bytes_per_row: Some(padded),
+ rows_per_image: None,
+ },
+ },
+ wgpu::Extent3d {
+ width: WIDTH,
+ height: HEIGHT,
+ depth_or_array_layers: 1,
+ },
+ );
+ queue.submit(std::iter::once(encoder.finish()));
+ buffer.slice(..).map_async(wgpu::MapMode::Read, |_| {});
+ device.poll(wgpu::PollType::Wait).expect("poll");
+ let mapped = buffer.slice(..).get_mapped_range();
+ (0..HEIGHT as usize)
+ .flat_map(|row| {
+ let start = row * padded as usize;
+ mapped[start..start + WIDTH as usize * 4].to_vec()
+ })
+ .collect()
+ }
+
+ #[tokio::test]
+ async fn export_shader_encodes_bt709_limited_range() {
+ let Some((device, queue)) = device() else {
+ eprintln!("no GPU adapter available, skipping");
+ return;
+ };
+ let source = chart_texture(&device, &queue);
+ let mut converter = RgbaToNv12Converter::new(&device);
+ let mut encoder = device.create_command_encoder(&Default::default());
+ assert!(
+ converter
+ .submit_conversion(&device, &queue, &mut encoder, &source, WIDTH, HEIGHT, 0, 30)
+ .await
+ .expect("conversion submission")
+ );
+ queue.submit(std::iter::once(encoder.finish()));
+ converter.after_submit(&queue);
+ let frame = converter
+ .take_pending()
+ .expect("conversion pending")
+ .wait_with_pool(&device, None)
+ .await
+ .expect("conversion completes");
+ let stride = frame.y_stride as usize;
+ let uv_base = stride * HEIGHT as usize;
+ for (i, rgb) in CHART.iter().enumerate() {
+ let [y, cb, cr] = reference_bt709(*rgb);
+ let x = i * 4;
+ let actual = [
+ frame.data[x],
+ frame.data[uv_base + x],
+ frame.data[uv_base + x + 1],
+ ];
+ for (a, e) in actual.iter().zip([y, cb, cr]) {
+ assert!(
+ (f64::from(*a) - e).abs() <= 0.5 + 1e-3,
+ "{rgb:?}: got {actual:?}, BT.709 reference {y:.2} {cb:.2} {cr:.2}"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn decode_shaders_apply_the_requested_matrix() {
+ let Some((device, queue)) = device() else {
+ eprintln!("no GPU adapter available, skipping");
+ return;
+ };
+ let (y, uv) = chart_nv12();
+ let mut converter = YuvToRgbaConverter::new(&device);
+ for matrix in [YuvMatrix::Bt601, YuvMatrix::Bt709] {
+ converter.set_matrix(matrix);
+ converter
+ .convert_nv12(&device, &queue, &y, &uv, WIDTH, HEIGHT, WIDTH, WIDTH)
+ .expect("convert");
+ let rgba = read_rgba(&device, &queue, converter.output_texture().unwrap());
+ for (i, rgb) in CHART.iter().enumerate() {
+ let x = i * 4;
+ let expected = matrix.ycbcr_to_rgb(
+ i32::from(y[x]) - 16,
+ i32::from(uv[x]) - 128,
+ i32::from(uv[x + 1]) - 128,
+ );
+ let actual = &rgba[x * 4..x * 4 + 3];
+ // The shaders' float constants and 16/256 luma offset differ
+ // from the 8.8 integer path by up to 2 levels (as before).
+ assert!(
+ actual.iter().zip(expected).all(|(a, e)| a.abs_diff(e) <= 2),
+ "{matrix:?} {rgb:?}: GPU {actual:?}, CPU {expected:?}"
+ );
+ if matrix == YuvMatrix::Bt709 {
+ assert!(
+ actual.iter().zip(rgb).all(|(a, e)| a.abs_diff(*e) <= 2),
+ "BT.709 round trip of {rgb:?} gave {actual:?}"
+ );
+ }
+ }
+ }
+ }
+}