Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(rendering): AVX2 the software YUV to RGBA path

Kyzered673e5f2ca7
diff --git a/crates/rendering/src/cpu_yuv.rs b/crates/rendering/src/cpu_yuv.rs
index e32b53cd7..b33e6f3f9 100644
--- a/crates/rendering/src/cpu_yuv.rs
+++ b/crates/rendering/src/cpu_yuv.rs
@@ -34,12 +34,19 @@ impl YuvMatrix {
]
}
+ /// `(rv, gu, gv, bu)` in 8.8 fixed point. Shared with the AVX2 kernel so
+ /// the two paths cannot drift.
#[inline(always)]
- fn chroma_contributions(self, d: i32, e: i32) -> [i32; 3] {
- let (rv, gu, gv, bu) = match self {
+ fn chroma_gains(self) -> (i32, i32, i32, i32) {
+ match self {
Self::Bt601 => (409, 100, 208, 516),
Self::Bt709 => (459, 55, 136, 541),
- };
+ }
+ }
+
+ #[inline(always)]
+ fn chroma_contributions(self, d: i32, e: i32) -> [i32; 3] {
+ let (rv, gu, gv, bu) = self.chroma_gains();
[rv * e + 128, -gu * d - gv * e + 128, bu * d + 128]
}
@@ -173,6 +180,28 @@ pub fn nv12_to_rgba_with_matrix(
uv_stride: u32,
output: &mut [u8],
matrix: YuvMatrix,
+) {
+ #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
+ if nv12_simd_inputs_fit(y_data, uv_data, output, width, height, y_stride, uv_stride) {
+ return nv12_to_rgba_simd(
+ y_data, uv_data, width, height, y_stride, uv_stride, output, matrix,
+ );
+ }
+ nv12_to_rgba_scalar(
+ y_data, uv_data, width, height, y_stride, uv_stride, output, matrix,
+ );
+}
+
+#[allow(clippy::too_many_arguments)]
+fn nv12_to_rgba_scalar(
+ y_data: &[u8],
+ uv_data: &[u8],
+ width: u32,
+ height: u32,
+ y_stride: u32,
+ uv_stride: u32,
+ output: &mut [u8],
+ matrix: YuvMatrix,
) {
let width = width as usize;
let height = height as usize;
@@ -220,6 +249,31 @@ pub fn yuv420p_to_rgba_with_matrix(
uv_stride: u32,
output: &mut [u8],
matrix: YuvMatrix,
+) {
+ #[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
+ if yuv420p_simd_inputs_fit(
+ y_data, u_data, v_data, output, width, height, y_stride, uv_stride,
+ ) {
+ return yuv420p_to_rgba_simd(
+ y_data, u_data, v_data, width, height, y_stride, uv_stride, output, matrix,
+ );
+ }
+ yuv420p_to_rgba_scalar(
+ y_data, u_data, v_data, width, height, y_stride, uv_stride, output, matrix,
+ );
+}
+
+#[allow(clippy::too_many_arguments)]
+fn yuv420p_to_rgba_scalar(
+ y_data: &[u8],
+ u_data: &[u8],
+ v_data: &[u8],
+ width: u32,
+ height: u32,
+ y_stride: u32,
+ uv_stride: u32,
+ output: &mut [u8],
+ matrix: YuvMatrix,
) {
let width = width as usize;
let height = height as usize;
@@ -282,7 +336,7 @@ impl SimdLevel {
pub fn pixels_per_iteration(self) -> usize {
match self {
- SimdLevel::Avx2 => 16,
+ SimdLevel::Avx2 => 8,
SimdLevel::Sse2 => 8,
SimdLevel::Scalar => 1,
}
@@ -333,27 +387,8 @@ pub fn nv12_to_rgba_simd_with_progress(
return;
}
- let y_required = (height_usize - 1)
- .saturating_mul(y_stride_usize)
- .saturating_add(width_usize);
-
- let uv_height = height_usize.div_ceil(2);
- let uv_width_bytes = width_usize.div_ceil(2) * 2;
- let uv_required = uv_height
- .saturating_sub(1)
- .saturating_mul(uv_stride_usize)
- .saturating_add(uv_width_bytes);
-
- let output_required = width_usize.saturating_mul(height_usize).saturating_mul(4);
-
- let strides_valid = y_stride_usize >= width_usize && uv_stride_usize >= uv_width_bytes;
-
- if !strides_valid
- || y_data.len() < y_required
- || uv_data.len() < uv_required
- || output.len() < output_required
- {
- return nv12_to_rgba_with_matrix(
+ if !nv12_simd_inputs_fit(y_data, uv_data, output, width, height, y_stride, uv_stride) {
+ return nv12_to_rgba_scalar(
y_data, uv_data, width, height, y_stride, uv_stride, output, matrix,
);
}
@@ -571,16 +606,32 @@ unsafe fn nv12_convert_row_avx2(
matrix: YuvMatrix,
) {
unsafe {
- nv12_convert_row_sse2(
- y_data,
- uv_data,
- width,
- y_row_start,
- uv_row_start,
- out_row_start,
- output,
- matrix,
- );
+ let simd_px = width & !7;
+ if simd_px != 0 {
+ avx2_yuv::nv12(
+ y_data,
+ uv_data,
+ simd_px,
+ y_row_start,
+ uv_row_start,
+ out_row_start,
+ output,
+ matrix,
+ );
+ }
+ if simd_px != width {
+ nv12_convert_span_at(
+ y_data,
+ uv_data,
+ width,
+ simd_px,
+ y_row_start,
+ uv_row_start,
+ out_row_start,
+ output,
+ matrix,
+ );
+ }
}
}
@@ -621,10 +672,39 @@ fn nv12_convert_row_scalar(
output: &mut [u8],
matrix: YuvMatrix,
) {
- let y_row = &y_data[y_row_start..y_row_start + width];
- let uv_row = &uv_data[uv_row_start..uv_row_start + width.div_ceil(2) * 2];
- let output_row = &mut output[out_row_start..out_row_start + width * 4];
+ nv12_convert_span_at(
+ y_data,
+ uv_data,
+ width,
+ 0,
+ y_row_start,
+ uv_row_start,
+ out_row_start,
+ output,
+ matrix,
+ );
+}
+
+#[allow(clippy::too_many_arguments)]
+fn nv12_convert_span_at(
+ y_data: &[u8],
+ uv_data: &[u8],
+ width: usize,
+ start_col: usize,
+ y_row_start: usize,
+ uv_row_start: usize,
+ out_row_start: usize,
+ output: &mut [u8],
+ matrix: YuvMatrix,
+) {
+ debug_assert!(start_col.is_multiple_of(2));
+ let y_row = &y_data[y_row_start + start_col..y_row_start + width];
+ let uv_row = &uv_data[uv_row_start + start_col..uv_row_start + width.div_ceil(2) * 2];
+ let output_row = &mut output[out_row_start + start_col * 4..out_row_start + width * 4];
+ nv12_convert_span(y_row, uv_row, output_row, matrix);
+}
+fn nv12_convert_span(y_row: &[u8], uv_row: &[u8], output_row: &mut [u8], matrix: YuvMatrix) {
for ((luma, chroma), pixels) in y_row
.chunks(2)
.zip(uv_row.chunks_exact(2))
@@ -722,28 +802,10 @@ pub fn yuv420p_to_rgba_simd_with_progress(
return;
}
- let y_required = (height_usize - 1)
- .saturating_mul(y_stride_usize)
- .saturating_add(width_usize);
-
- let uv_height = height_usize.div_ceil(2);
- let uv_width = width_usize.div_ceil(2);
- let uv_required = uv_height
- .saturating_sub(1)
- .saturating_mul(uv_stride_usize)
- .saturating_add(uv_width);
-
- let output_required = width_usize.saturating_mul(height_usize).saturating_mul(4);
-
- let strides_valid = y_stride_usize >= width_usize && uv_stride_usize >= uv_width;
-
- if !strides_valid
- || y_data.len() < y_required
- || u_data.len() < uv_required
- || v_data.len() < uv_required
- || output.len() < output_required
- {
- return yuv420p_to_rgba_with_matrix(
+ if !yuv420p_simd_inputs_fit(
+ y_data, u_data, v_data, output, width, height, y_stride, uv_stride,
+ ) {
+ return yuv420p_to_rgba_scalar(
y_data, u_data, v_data, width, height, y_stride, uv_stride, output, matrix,
);
}
@@ -972,17 +1034,34 @@ unsafe fn yuv420p_convert_row_avx2(
matrix: YuvMatrix,
) {
unsafe {
- yuv420p_convert_row_sse2(
- y_data,
- u_data,
- v_data,
- width,
- y_row_start,
- uv_row_start,
- out_row_start,
- output,
- matrix,
- );
+ let simd_px = width & !7;
+ if simd_px != 0 {
+ avx2_yuv::yuv420p(
+ y_data,
+ u_data,
+ v_data,
+ simd_px,
+ y_row_start,
+ uv_row_start,
+ out_row_start,
+ output,
+ matrix,
+ );
+ }
+ if simd_px != width {
+ yuv420p_convert_span_at(
+ y_data,
+ u_data,
+ v_data,
+ width,
+ simd_px,
+ y_row_start,
+ uv_row_start,
+ out_row_start,
+ output,
+ matrix,
+ );
+ }
}
}
@@ -1026,12 +1105,49 @@ fn yuv420p_convert_row_scalar(
output: &mut [u8],
matrix: YuvMatrix,
) {
- let y_row = &y_data[y_row_start..y_row_start + width];
+ yuv420p_convert_span_at(
+ y_data,
+ u_data,
+ v_data,
+ width,
+ 0,
+ y_row_start,
+ uv_row_start,
+ out_row_start,
+ output,
+ matrix,
+ );
+}
+
+#[allow(clippy::too_many_arguments)]
+fn yuv420p_convert_span_at(
+ y_data: &[u8],
+ u_data: &[u8],
+ v_data: &[u8],
+ width: usize,
+ start_col: usize,
+ y_row_start: usize,
+ uv_row_start: usize,
+ out_row_start: usize,
+ output: &mut [u8],
+ matrix: YuvMatrix,
+) {
+ debug_assert!(start_col.is_multiple_of(2));
let chroma_width = width.div_ceil(2);
- let u_row = &u_data[uv_row_start..uv_row_start + chroma_width];
- let v_row = &v_data[uv_row_start..uv_row_start + chroma_width];
- let output_row = &mut output[out_row_start..out_row_start + width * 4];
+ let y_row = &y_data[y_row_start + start_col..y_row_start + width];
+ let u_row = &u_data[uv_row_start + start_col / 2..uv_row_start + chroma_width];
+ let v_row = &v_data[uv_row_start + start_col / 2..uv_row_start + chroma_width];
+ let output_row = &mut output[out_row_start + start_col * 4..out_row_start + width * 4];
+ yuv420p_convert_span(y_row, u_row, v_row, output_row, matrix);
+}
+fn yuv420p_convert_span(
+ y_row: &[u8],
+ u_row: &[u8],
+ v_row: &[u8],
+ output_row: &mut [u8],
+ matrix: YuvMatrix,
+) {
for (((luma, u), v), pixels) in y_row
.chunks(2)
.zip(u_row)
@@ -1096,6 +1212,249 @@ fn clamp_u8(val: i32) -> u8 {
val.clamp(0, 255) as u8
}
+#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
+#[allow(clippy::too_many_arguments)]
+fn nv12_simd_inputs_fit(
+ y_data: &[u8],
+ uv_data: &[u8],
+ output: &[u8],
+ width: u32,
+ height: u32,
+ y_stride: u32,
+ uv_stride: u32,
+) -> bool {
+ let width = width as usize;
+ let height = height as usize;
+ if width == 0 || height == 0 {
+ return false;
+ }
+ let y_stride = y_stride as usize;
+ let uv_stride = uv_stride as usize;
+ let y_required = (height - 1).saturating_mul(y_stride).saturating_add(width);
+ let uv_height = height.div_ceil(2);
+ let uv_width_bytes = width.div_ceil(2) * 2;
+ let uv_required = uv_height
+ .saturating_sub(1)
+ .saturating_mul(uv_stride)
+ .saturating_add(uv_width_bytes);
+ let output_required = width.saturating_mul(height).saturating_mul(4);
+ y_stride >= width
+ && uv_stride >= uv_width_bytes
+ && y_data.len() >= y_required
+ && uv_data.len() >= uv_required
+ && output.len() >= output_required
+}
+
+#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
+#[allow(clippy::too_many_arguments)]
+fn yuv420p_simd_inputs_fit(
+ y_data: &[u8],
+ u_data: &[u8],
+ v_data: &[u8],
+ output: &[u8],
+ width: u32,
+ height: u32,
+ y_stride: u32,
+ uv_stride: u32,
+) -> bool {
+ let width = width as usize;
+ let height = height as usize;
+ if width == 0 || height == 0 {
+ return false;
+ }
+ let y_stride = y_stride as usize;
+ let uv_stride = uv_stride as usize;
+ let y_required = (height - 1).saturating_mul(y_stride).saturating_add(width);
+ let uv_height = height.div_ceil(2);
+ let uv_width = width.div_ceil(2);
+ let uv_required = uv_height
+ .saturating_sub(1)
+ .saturating_mul(uv_stride)
+ .saturating_add(uv_width);
+ let output_required = width.saturating_mul(height).saturating_mul(4);
+ y_stride >= width
+ && uv_stride >= uv_width
+ && y_data.len() >= y_required
+ && u_data.len() >= uv_required
+ && v_data.len() >= uv_required
+ && output.len() >= output_required
+}
+
+#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
+mod avx2_yuv {
+ use super::YuvMatrix;
+
+ #[cfg(target_arch = "x86")]
+ use std::arch::x86::*;
+ #[cfg(target_arch = "x86_64")]
+ use std::arch::x86_64::*;
+
+ struct Coeffs {
+ rv: __m256i,
+ gu: __m256i,
+ gv: __m256i,
+ bu: __m256i,
+ bias128: __m256i,
+ y16: __m256i,
+ gain: __m256i,
+ alpha: __m256i,
+ zero: __m256i,
+ max: __m256i,
+ nv12_u: __m256i,
+ nv12_v: __m256i,
+ dup4: __m256i,
+ }
+
+ impl Coeffs {
+ #[inline]
+ #[target_feature(enable = "avx2")]
+ unsafe fn new(matrix: YuvMatrix) -> Self {
+ let (rv, gu, gv, bu) = matrix.chroma_gains();
+ Self {
+ rv: _mm256_set1_epi32(rv),
+ gu: _mm256_set1_epi32(gu),
+ gv: _mm256_set1_epi32(gv),
+ bu: _mm256_set1_epi32(bu),
+ bias128: _mm256_set1_epi32(128),
+ y16: _mm256_set1_epi32(16),
+ gain: _mm256_set1_epi32(298),
+ alpha: _mm256_set1_epi32(0xFF00_0000u32 as i32),
+ zero: _mm256_setzero_si256(),
+ max: _mm256_set1_epi32(255),
+ nv12_u: _mm256_setr_epi32(0, 0, 2, 2, 4, 4, 6, 6),
+ nv12_v: _mm256_setr_epi32(1, 1, 3, 3, 5, 5, 7, 7),
+ dup4: _mm256_setr_epi32(0, 0, 1, 1, 2, 2, 3, 3),
+ }
+ }
+ }
+
+ #[inline]
+ #[target_feature(enable = "avx2")]
+ unsafe fn clamp8(sum: __m256i, coeffs: &Coeffs) -> __m256i {
+ let shifted = _mm256_srai_epi32(sum, 8);
+ _mm256_min_epi32(_mm256_max_epi32(shifted, coeffs.zero), coeffs.max)
+ }
+
+ #[inline]
+ #[target_feature(enable = "avx2")]
+ unsafe fn store_rgba(out: *mut u8, luma: __m256i, d: __m256i, e: __m256i, coeffs: &Coeffs) {
+ unsafe {
+ let red = _mm256_add_epi32(_mm256_mullo_epi32(coeffs.rv, e), coeffs.bias128);
+ let green = _mm256_sub_epi32(
+ coeffs.bias128,
+ _mm256_add_epi32(
+ _mm256_mullo_epi32(coeffs.gu, d),
+ _mm256_mullo_epi32(coeffs.gv, e),
+ ),
+ );
+ let blue = _mm256_add_epi32(_mm256_mullo_epi32(coeffs.bu, d), coeffs.bias128);
+ let r = clamp8(_mm256_add_epi32(luma, red), coeffs);
+ let g = clamp8(_mm256_add_epi32(luma, green), coeffs);
+ let b = clamp8(_mm256_add_epi32(luma, blue), coeffs);
+ let pixels = _mm256_or_si256(
+ r,
+ _mm256_or_si256(
+ _mm256_slli_epi32(g, 8),
+ _mm256_or_si256(_mm256_slli_epi32(b, 16), coeffs.alpha),
+ ),
+ );
+ _mm256_storeu_si256(out.cast(), pixels);
+ }
+ }
+
+ #[inline]
+ #[target_feature(enable = "avx2")]
+ unsafe fn luma8(y: *const u8, coeffs: &Coeffs) -> __m256i {
+ unsafe {
+ let y8 = _mm_loadl_epi64(y.cast());
+ let y32 = _mm256_cvtepu8_epi32(y8);
+ _mm256_mullo_epi32(_mm256_sub_epi32(y32, coeffs.y16), coeffs.gain)
+ }
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ #[target_feature(enable = "avx2")]
+ pub(super) unsafe fn nv12(
+ y_data: &[u8],
+ uv_data: &[u8],
+ width: usize,
+ y_row_start: usize,
+ uv_row_start: usize,
+ out_row_start: usize,
+ output: &mut [u8],
+ matrix: YuvMatrix,
+ ) {
+ unsafe {
+ debug_assert!(width.is_multiple_of(8));
+ let y = &y_data[y_row_start..y_row_start + width];
+ let uv = &uv_data[uv_row_start..uv_row_start + width];
+ let out = &mut output[out_row_start..out_row_start + width * 4];
+ let coeffs = Coeffs::new(matrix);
+ let mut x = 0;
+ while x < width {
+ let luma = luma8(y.as_ptr().add(x), &coeffs);
+ let uv32 = _mm256_cvtepu8_epi32(_mm_loadl_epi64(uv.as_ptr().add(x).cast()));
+ let d = _mm256_sub_epi32(
+ _mm256_permutevar8x32_epi32(uv32, coeffs.nv12_u),
+ coeffs.bias128,
+ );
+ let e = _mm256_sub_epi32(
+ _mm256_permutevar8x32_epi32(uv32, coeffs.nv12_v),
+ coeffs.bias128,
+ );
+ store_rgba(out.as_mut_ptr().add(x * 4), luma, d, e, &coeffs);
+ x += 8;
+ }
+ }
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ #[target_feature(enable = "avx2")]
+ pub(super) unsafe fn yuv420p(
+ y_data: &[u8],
+ u_data: &[u8],
+ v_data: &[u8],
+ width: usize,
+ y_row_start: usize,
+ uv_row_start: usize,
+ out_row_start: usize,
+ output: &mut [u8],
+ matrix: YuvMatrix,
+ ) {
+ unsafe {
+ debug_assert!(width.is_multiple_of(8));
+ let chroma_width = width / 2;
+ let y = &y_data[y_row_start..y_row_start + width];
+ let u = &u_data[uv_row_start..uv_row_start + chroma_width];
+ let v = &v_data[uv_row_start..uv_row_start + chroma_width];
+ let out = &mut output[out_row_start..out_row_start + width * 4];
+ let coeffs = Coeffs::new(matrix);
+ let mut x = 0;
+ while x < width {
+ let luma = luma8(y.as_ptr().add(x), &coeffs);
+ let u32 = std::ptr::read_unaligned(u.as_ptr().add(x / 2).cast::<u32>());
+ let v32 = std::ptr::read_unaligned(v.as_ptr().add(x / 2).cast::<u32>());
+ let d = _mm256_sub_epi32(
+ _mm256_permutevar8x32_epi32(
+ _mm256_cvtepu8_epi32(_mm_cvtsi32_si128(u32 as i32)),
+ coeffs.dup4,
+ ),
+ coeffs.bias128,
+ );
+ let e = _mm256_sub_epi32(
+ _mm256_permutevar8x32_epi32(
+ _mm256_cvtepu8_epi32(_mm_cvtsi32_si128(v32 as i32)),
+ coeffs.dup4,
+ ),
+ coeffs.bias128,
+ );
+ store_rgba(out.as_mut_ptr().add(x * 4), luma, d, e, &coeffs);
+ x += 8;
+ }
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -1147,7 +1506,7 @@ mod tests {
let mut output_scalar = vec![0u8; (width * height * 4) as usize];
let mut output_simd = vec![0u8; (width * height * 4) as usize];
- nv12_to_rgba_with_matrix(
+ nv12_to_rgba_scalar(
&y_data,
&uv_data,
width,
@@ -1219,7 +1578,7 @@ mod tests {
let mut output1 = vec![0u8; (width * height * 4) as usize];
let mut output2 = vec![0u8; (width * height * 4) as usize];
- nv12_to_rgba_with_matrix(
+ nv12_to_rgba_scalar(
&y_data,
&uv_data,
width,
@@ -1270,7 +1629,7 @@ mod tests {
let mut output_scalar = vec![0u8; (width * height * 4) as usize];
let mut output_simd = vec![0u8; (width * height * 4) as usize];
- yuv420p_to_rgba_with_matrix(
+ yuv420p_to_rgba_scalar(
&y_data,
&u_data,
&v_data,
@@ -1323,8 +1682,9 @@ mod tests {
let uv_data: Vec<u8> = (0..uv_len).map(|i| (i * 53 + 97) as u8).collect();
let mut expected = vec![0; (width * height * 4) as usize];
let mut actual = vec![0xa5; expected.len()];
+ let mut dispatched = vec![0xa5; expected.len()];
- nv12_to_rgba_with_matrix(
+ nv12_to_rgba_scalar(
&y_data,
&uv_data,
width,
@@ -1344,11 +1704,25 @@ mod tests {
&mut actual,
matrix,
);
+ nv12_to_rgba_with_matrix(
+ &y_data,
+ &uv_data,
+ width,
+ height,
+ y_stride,
+ uv_stride,
+ &mut dispatched,
+ matrix,
+ );
assert_eq!(
actual, expected,
"NV12 {matrix:?} {width}x{height}, y_stride={y_stride}, uv_stride={uv_stride}"
);
+ assert_eq!(
+ dispatched, expected,
+ "NV12 dispatch {matrix:?} {width}x{height}, y_stride={y_stride}, uv_stride={uv_stride}"
+ );
}
}
}
@@ -1374,8 +1748,9 @@ mod tests {
let v_data: Vec<u8> = (0..uv_len).map(|i| (i * 71 + 193) as u8).collect();
let mut expected = vec![0; (width * height * 4) as usize];
let mut actual = vec![0xa5; expected.len()];
+ let mut dispatched = vec![0xa5; expected.len()];
- yuv420p_to_rgba_with_matrix(
+ yuv420p_to_rgba_scalar(
&y_data,
&u_data,
&v_data,
@@ -1397,11 +1772,26 @@ mod tests {
&mut actual,
matrix,
);
+ yuv420p_to_rgba_with_matrix(
+ &y_data,
+ &u_data,
+ &v_data,
+ width,
+ height,
+ y_stride,
+ uv_stride,
+ &mut dispatched,
+ matrix,
+ );
assert_eq!(
actual, expected,
"YUV420P {matrix:?} {width}x{height}, y_stride={y_stride}, uv_stride={uv_stride}"
);
+ assert_eq!(
+ dispatched, expected,
+ "YUV420P dispatch {matrix:?} {width}x{height}, y_stride={y_stride}, uv_stride={uv_stride}"
+ );
}
}
}
@@ -1617,13 +2007,90 @@ mod tests {
for matrix in [YuvMatrix::Bt601, YuvMatrix::Bt709] {
let mut simd = vec![0u8; (width * height * 4) as usize];
let mut scalar = simd.clone();
+ let mut dispatched = simd.clone();
nv12_to_rgba_simd(&y, &uv, width, height, width, width, &mut simd, matrix);
- nv12_to_rgba_with_matrix(&y, &uv, width, height, width, width, &mut scalar, matrix);
+ nv12_to_rgba_scalar(&y, &uv, width, height, width, width, &mut scalar, matrix);
+ nv12_to_rgba_with_matrix(
+ &y,
+ &uv,
+ width,
+ height,
+ width,
+ width,
+ &mut dispatched,
+ matrix,
+ );
assert_eq!(simd, scalar);
+ assert_eq!(dispatched, scalar);
assert_eq!(
&simd[..3],
&matrix.ycbcr_to_rgb(81 - 16, 90 - 128, 240 - 128)
);
}
}
+
+ #[test]
+ fn simd_matches_scalar_on_extremes_and_odd_tail() {
+ let samples = [0u8, 1, 15, 16, 17, 128, 234, 235, 236, 254, 255];
+ let (width, height) = (9u32, 3u32);
+ for matrix in [YuvMatrix::Bt601, YuvMatrix::Bt709] {
+ for &y0 in &samples {
+ for &u0 in &samples {
+ for &v0 in &samples {
+ let y = vec![y0; (width * height) as usize];
+ let uv_rows = height.div_ceil(2);
+ let nv12_row = width.div_ceil(2) * 2;
+ let mut uv = vec![0u8; (nv12_row * uv_rows) as usize];
+ for pair in uv.chunks_exact_mut(2) {
+ pair[0] = u0;
+ pair[1] = v0;
+ }
+ let chroma = (width.div_ceil(2) * uv_rows) as usize;
+ let u = vec![u0; chroma];
+ let v = vec![v0; chroma];
+ let mut expected = vec![7u8; (width * height * 4) as usize];
+ let mut nv12 = vec![9u8; expected.len()];
+ let mut yuv = vec![9u8; expected.len()];
+ nv12_to_rgba_scalar(
+ &y,
+ &uv,
+ width,
+ height,
+ width,
+ nv12_row,
+ &mut expected,
+ matrix,
+ );
+ nv12_to_rgba_with_matrix(
+ &y, &uv, width, height, width, nv12_row, &mut nv12, matrix,
+ );
+ assert_eq!(nv12, expected, "{matrix:?} y={y0} u={u0} v={v0}");
+ yuv420p_to_rgba_scalar(
+ &y,
+ &u,
+ &v,
+ width,
+ height,
+ width,
+ width.div_ceil(2),
+ &mut expected,
+ matrix,
+ );
+ yuv420p_to_rgba_with_matrix(
+ &y,
+ &u,
+ &v,
+ width,
+ height,
+ width,
+ width.div_ceil(2),
+ &mut yuv,
+ matrix,
+ );
+ assert_eq!(yuv, expected, "420 {matrix:?} y={y0} u={u0} v={v0}");
+ }
+ }
+ }
+ }
+ }
}
diff --git a/crates/rendering/src/yuv_converter.rs b/crates/rendering/src/yuv_converter.rs
index f7e8a7983..4fe9e522c 100644
--- a/crates/rendering/src/yuv_converter.rs
+++ b/crates/rendering/src/yuv_converter.rs
@@ -519,6 +519,7 @@ unsafe impl Sync for RetainedImageBuf {}
pub struct YuvToRgbaConverter {
pipelines: Arc<YuvConverterPipelines>,
matrix: YuvMatrix,
+ cpu_rgba_scratch: Vec<u8>,
y_texture: wgpu::Texture,
y_view: wgpu::TextureView,
uv_texture: wgpu::Texture,
@@ -588,6 +589,7 @@ impl YuvToRgbaConverter {
Self {
matrix: YuvMatrix::default(),
+ cpu_rgba_scratch: Vec::new(),
pipelines,
y_texture,
y_view,
@@ -1773,7 +1775,9 @@ impl YuvToRgbaConverter {
self.ensure_texture_size(device, effective_width, effective_height);
self.swap_output_buffer();
- let mut rgba_data = vec![0u8; (width * height * 4) as usize];
+ let byte_len = (width * height * 4) as usize;
+ let matrix = self.matrix;
+ let mut rgba_data = self.take_cpu_rgba_scratch(byte_len);
cpu_yuv::nv12_to_rgba_simd(
y_data,
@@ -1782,8 +1786,8 @@ impl YuvToRgbaConverter {
height,
y_stride,
uv_stride,
- &mut rgba_data,
- self.matrix,
+ &mut rgba_data[..byte_len],
+ matrix,
);
queue.write_texture(
@@ -1793,7 +1797,7 @@ impl YuvToRgbaConverter {
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
- &rgba_data,
+ &rgba_data[..byte_len],
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(width * 4),
@@ -1806,6 +1810,7 @@ impl YuvToRgbaConverter {
},
);
+ self.cpu_rgba_scratch = rgba_data;
Ok(self.current_output_view())
}
@@ -1827,7 +1832,9 @@ impl YuvToRgbaConverter {
self.ensure_texture_size(device, effective_width, effective_height);
self.swap_output_buffer();
- let mut rgba_data = vec![0u8; (width * height * 4) as usize];
+ let byte_len = (width * height * 4) as usize;
+ let matrix = self.matrix;
+ let mut rgba_data = self.take_cpu_rgba_scratch(byte_len);
cpu_yuv::yuv420p_to_rgba_simd(
y_data,
@@ -1837,8 +1844,8 @@ impl YuvToRgbaConverter {
height,
y_stride,
uv_stride,
- &mut rgba_data,
- self.matrix,
+ &mut rgba_data[..byte_len],
+ matrix,
);
queue.write_texture(
@@ -1848,7 +1855,7 @@ impl YuvToRgbaConverter {
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
- &rgba_data,
+ &rgba_data[..byte_len],
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(width * 4),
@@ -1861,9 +1868,20 @@ impl YuvToRgbaConverter {
},
);
+ self.cpu_rgba_scratch = rgba_data;
Ok(self.current_output_view())
}
+ fn take_cpu_rgba_scratch(&mut self, byte_len: usize) -> Vec<u8> {
+ let mut rgba = std::mem::take(&mut self.cpu_rgba_scratch);
+ // Kernels write every output byte, including alpha, so a reused
+ // buffer is cleared only when a larger frame grows it.
+ if rgba.len() < byte_len {
+ rgba.resize(byte_len, 0);
+ }
+ rgba
+ }
+
pub fn output_texture(&self) -> Option<&wgpu::Texture> {
Some(self.current_output_texture())
}