Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

Merge main

Kyzer6f7137d56092
diff --cc crates/enc-ffmpeg/src/video/h264.rs
index 0a5e6d6be,2bc025dcb..cbe05805f
--- a/crates/enc-ffmpeg/src/video/h264.rs
+++ b/crates/enc-ffmpeg/src/video/h264.rs
@@@ -923,7 -923,7 +923,7 @@@ impl H264Encoder
let frame_to_send = if let Some(converter) = &mut self.converter {
let pts = frame.pts();
let converted = self.converted_frame_pool.as_mut().unwrap();
- super::rgb_to_yuv::convert_or_scale(&frame, converted, converter)
- super::packed_yuv::convert_or_scale(&frame, converted, converter)
++ super::convert_or_scale(&frame, converted, converter)
.map_err(QueueFrameError::Converter)?;
converted.set_pts(pts);
converted as &frame::Video
@@@ -952,7 -952,7 +952,7 @@@
let converted = converted_frame.get_or_insert_with(|| {
frame::Video::new(self.output_format, self.output_width, self.output_height)
});
- super::rgb_to_yuv::convert_or_scale(frame, converted, converter)
- super::packed_yuv::convert_or_scale(frame, converted, converter)
++ super::convert_or_scale(frame, converted, converter)
.map_err(QueueFrameError::Converter)?;
converted.set_pts(pts);
converted as &frame::Video
diff --cc crates/enc-ffmpeg/src/video/h264_packet.rs
index 7878ba43c,e344a09e5..dd0fe0e8a
--- a/crates/enc-ffmpeg/src/video/h264_packet.rs
+++ b/crates/enc-ffmpeg/src/video/h264_packet.rs
@@@ -140,7 -140,7 +140,7 @@@ impl H264PacketEncoder
let converted = self.converted_frame_pool.as_mut().expect(
"converted_frame_pool present whenever converter is Some (invariant preserved by from_opened)",
);
- super::rgb_to_yuv::convert_or_scale(&frame, converted, converter)
- super::packed_yuv::convert_or_scale(&frame, converted, converter)
++ super::convert_or_scale(&frame, converted, converter)
.map_err(EncodePacketError::Converter)?;
converted.set_pts(pts);
converted as &frame::Video
@@@ -172,7 -172,7 +172,7 @@@
let converted = converted_frame.get_or_insert_with(|| {
frame::Video::new(self.output_format, self.output_width, self.output_height)
});
- super::rgb_to_yuv::convert_or_scale(frame, converted, converter)
- super::packed_yuv::convert_or_scale(frame, converted, converter)
++ super::convert_or_scale(frame, converted, converter)
.map_err(EncodePacketError::Converter)?;
converted.set_pts(pts);
converted as &frame::Video
diff --cc crates/enc-ffmpeg/src/video/mod.rs
index 3c59c9112,040480d0f..e69e981ed
--- a/crates/enc-ffmpeg/src/video/mod.rs
+++ b/crates/enc-ffmpeg/src/video/mod.rs
@@@ -1,8 -1,8 +1,23 @@@
++use ffmpeg::frame;
++
pub mod colour;
pub mod h264;
pub mod h264_packet;
pub mod hevc;
+ mod packed_yuv;
pub mod prores;
+mod rgb_to_yuv;
#[cfg(target_os = "macos")]
pub mod videotoolbox_hw;
++
++pub(crate) fn convert_or_scale(
++ src: &frame::Video,
++ dst: &mut frame::Video,
++ scaler: &mut ffmpeg::software::scaling::Context,
++) -> Result<(), ffmpeg::Error> {
++ if packed_yuv::try_convert(src, dst) || rgb_to_yuv::try_convert(src, dst) {
++ Ok(())
++ } else {
++ scaler.run(src, dst)
++ }
++}
diff --cc crates/enc-ffmpeg/src/video/packed_yuv.rs
index 000000000,2bc80f474..2259054b3
mode 000000,100644..100644
--- a/crates/enc-ffmpeg/src/video/packed_yuv.rs
+++ b/crates/enc-ffmpeg/src/video/packed_yuv.rs
@@@ -1,0 -1,906 +1,903 @@@
+ use ffmpeg::format::Pixel;
+ use ffmpeg::frame;
+
+ #[derive(Clone, Copy, PartialEq, Eq)]
+ enum Packed422 {
+ Uyvy,
+ Yuyv,
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ struct Job {
+ src: *const u8,
+ src_stride: usize,
+ width: usize,
+ height: usize,
+ kind: Packed422,
+ y: *mut u8,
+ y_stride: usize,
+ u: *mut u8,
+ u_stride: usize,
+ v: *mut u8,
+ v_stride: usize,
+ uv: *mut u8,
+ uv_stride: usize,
+ nv12: bool,
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ fn positive_linesize(frame: &frame::Video, plane: usize) -> Option<usize> {
+ if plane >= frame.planes() {
+ return None;
+ }
+ let raw = unsafe { (*frame.as_ptr()).linesize[plane] };
+ if raw <= 0 { None } else { Some(raw as usize) }
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ fn covers(stride: usize, row_bytes: usize, height: usize, len: usize) -> bool {
+ if row_bytes == 0 || height == 0 {
+ return true;
+ }
+ if stride < row_bytes {
+ return false;
+ }
+ stride
+ .checked_mul(height - 1)
+ .and_then(|last| last.checked_add(row_bytes))
+ .is_some_and(|need| need <= len)
+ }
+
+ /// Same-size UYVY422 or YUYV422 into `dst` when both dimensions are even,
+ /// AVX2 is available at runtime, and `dst` is YUV420P or NV12. Any other
+ /// input returns false so the caller keeps swscale.
+ pub(crate) fn try_convert(src: &frame::Video, dst: &mut frame::Video) -> bool {
+ if src.width() != dst.width() || src.height() != dst.height() {
+ return false;
+ }
+ let kind = match src.format() {
+ Pixel::UYVY422 => Packed422::Uyvy,
+ Pixel::YUYV422 => Packed422::Yuyv,
+ _ => return false,
+ };
+ let nv12 = match dst.format() {
+ Pixel::YUV420P => false,
+ Pixel::NV12 => true,
+ _ => return false,
+ };
+ let width = src.width() as usize;
+ let height = src.height() as usize;
+ if !width.is_multiple_of(2) || !height.is_multiple_of(2) {
+ return false;
+ }
+ if width == 0 || height == 0 {
+ return true;
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ {
+ let _ = (kind, nv12);
+ false
+ }
+ #[cfg(target_arch = "x86_64")]
+ {
+ if !std::arch::is_x86_feature_detected!("avx2") {
+ return false;
+ }
+ let Some(src_stride) = positive_linesize(src, 0) else {
+ return false;
+ };
+ let Some(y_stride) = positive_linesize(dst, 0) else {
+ return false;
+ };
+ if !covers(src_stride, width * 2, height, src.data(0).len())
+ || !covers(y_stride, width, height, dst.data(0).len())
+ {
+ return false;
+ }
+ let chroma_w = width / 2;
+ let chroma_h = height / 2;
+ let (u_stride, v_stride, uv_stride) = if nv12 {
+ let Some(uv_stride) = positive_linesize(dst, 1) else {
+ return false;
+ };
+ if !covers(uv_stride, chroma_w * 2, chroma_h, dst.data(1).len()) {
+ return false;
+ }
+ (0, 0, uv_stride)
+ } else {
+ let Some(u_stride) = positive_linesize(dst, 1) else {
+ return false;
+ };
+ let Some(v_stride) = positive_linesize(dst, 2) else {
+ return false;
+ };
+ if !covers(u_stride, chroma_w, chroma_h, dst.data(1).len())
+ || !covers(v_stride, chroma_w, chroma_h, dst.data(2).len())
+ {
+ return false;
+ }
+ (u_stride, v_stride, 0)
+ };
+
+ let src_ptr = src.data(0).as_ptr();
+ let y_ptr = dst.data_mut(0).as_mut_ptr();
+ let (u_ptr, v_ptr, uv_ptr) = if nv12 {
+ (
+ std::ptr::null_mut(),
+ std::ptr::null_mut(),
+ dst.data_mut(1).as_mut_ptr(),
+ )
+ } else {
+ let u_ptr = dst.data_mut(1).as_mut_ptr();
+ let v_ptr = dst.data_mut(2).as_mut_ptr();
+ (u_ptr, v_ptr, std::ptr::null_mut())
+ };
+
+ let job = Job {
+ src: src_ptr,
+ src_stride,
+ width,
+ height,
+ kind,
+ y: y_ptr,
+ y_stride,
+ u: u_ptr,
+ u_stride,
+ v: v_ptr,
+ v_stride,
+ uv: uv_ptr,
+ uv_stride,
+ nv12,
+ };
+ // SAFETY: AVX2 was detected, and `covers` checked every active plane.
+ unsafe { avx2::execute(&job) };
+ true
+ }
+ }
+
++#[cfg(test)]
+ pub(crate) fn convert_or_scale(
+ src: &frame::Video,
+ dst: &mut frame::Video,
+ scaler: &mut ffmpeg::software::scaling::Context,
+ ) -> Result<(), ffmpeg::Error> {
- if try_convert(src, dst) {
- Ok(())
- } else {
- scaler.run(src, dst)
- }
++ super::convert_or_scale(src, dst, scaler)
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ mod avx2 {
+ use std::arch::x86_64::*;
+
+ use super::{Job, Packed422};
+
+ #[target_feature(enable = "avx2")]
+ pub(super) unsafe fn execute(job: &Job) {
+ unsafe {
+ let masks = match job.kind {
+ Packed422::Uyvy => Masks {
+ y: uyvy_y_mask(),
+ u: uyvy_u_mask(),
+ v: uyvy_v_mask(),
+ },
+ Packed422::Yuyv => Masks {
+ y: yuyv_y_mask(),
+ u: yuyv_u_mask(),
+ v: yuyv_v_mask(),
+ },
+ };
+ if job.nv12 {
+ convert_frame::<true>(job, &masks);
+ } else {
+ convert_frame::<false>(job, &masks);
+ }
+ }
+ }
+
+ struct Masks {
+ y: __m256i,
+ u: __m256i,
+ v: __m256i,
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn convert_frame<const NV12: bool>(job: &Job, masks: &Masks) {
+ unsafe {
+ let mut top = 0;
+ while top < job.height {
+ let row = Row16 {
+ src0: job.src.add(top * job.src_stride),
+ src1: job.src.add((top + 1) * job.src_stride),
+ y0: job.y.add(top * job.y_stride),
+ y1: job.y.add((top + 1) * job.y_stride),
+ top,
+ };
+ let mut x = 0;
+ while x + 32 <= job.width {
+ convert16::<NV12>(job, masks, &row, x);
+ convert16::<NV12>(job, masks, &row, x + 16);
+ x += 32;
+ }
+ while x + 16 <= job.width {
+ convert16::<NV12>(job, masks, &row, x);
+ x += 16;
+ }
+ if x < job.width {
+ scalar_tail::<NV12>(job, top, x);
+ }
+ top += 2;
+ }
+ }
+ }
+
+ struct Row16 {
+ src0: *const u8,
+ src1: *const u8,
+ y0: *mut u8,
+ y1: *mut u8,
+ top: usize,
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn convert16<const NV12: bool>(job: &Job, masks: &Masks, row: &Row16, x: usize) {
+ unsafe {
+ let packed0 = _mm256_loadu_si256(row.src0.add(x * 2).cast());
+ store_luma(packed0, masks.y, row.y0.add(x));
+ let u0 = pack_chroma8(packed0, masks.u);
+ let v0 = pack_chroma8(packed0, masks.v);
+ let packed1 = _mm256_loadu_si256(row.src1.add(x * 2).cast());
+ store_luma(packed1, masks.y, row.y1.add(x));
+ let u1 = pack_chroma8(packed1, masks.u);
+ let v1 = pack_chroma8(packed1, masks.v);
+ let u = _mm_avg_epu8(u0, u1);
+ let v = _mm_avg_epu8(v0, v1);
+ store_chroma8::<NV12>(job, row.top / 2, x / 2, u, v);
+ }
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn store_luma(packed: __m256i, mask: __m256i, dst: *mut u8) {
+ unsafe {
+ let shuffled = _mm256_shuffle_epi8(packed, mask);
+ let lo = _mm256_castsi256_si128(shuffled);
+ let hi = _mm256_extracti128_si256(shuffled, 1);
+ let y = _mm_unpacklo_epi64(lo, hi);
+ _mm_storeu_si128(dst.cast(), y);
+ }
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn pack_chroma8(packed: __m256i, mask: __m256i) -> __m128i {
+ let shuffled = _mm256_shuffle_epi8(packed, mask);
+ let lo = _mm256_castsi256_si128(shuffled);
+ let hi = _mm256_extracti128_si256(shuffled, 1);
+ _mm_unpacklo_epi32(lo, hi)
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn store_chroma8<const NV12: bool>(
+ job: &Job,
+ row: usize,
+ x: usize,
+ u: __m128i,
+ v: __m128i,
+ ) {
+ unsafe {
+ if NV12 {
+ let interleaved = _mm_unpacklo_epi8(u, v);
+ _mm_storeu_si128(job.uv.add(row * job.uv_stride + x * 2).cast(), interleaved);
+ } else {
+ _mm_storel_epi64(job.u.add(row * job.u_stride + x).cast(), u);
+ _mm_storel_epi64(job.v.add(row * job.v_stride + x).cast(), v);
+ }
+ }
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn uyvy_y_mask() -> __m256i {
+ _mm256_setr_epi8(
+ 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, 1, 3, 5, 7, 9, 11, 13, 15,
+ -1, -1, -1, -1, -1, -1, -1, -1,
+ )
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn uyvy_u_mask() -> __m256i {
+ _mm256_setr_epi8(
+ 0, 4, 8, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 4, 8, 12, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ )
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn uyvy_v_mask() -> __m256i {
+ _mm256_setr_epi8(
+ 2, 6, 10, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 6, 10, 14, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ )
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn yuyv_y_mask() -> __m256i {
+ _mm256_setr_epi8(
+ 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, 0, 2, 4, 6, 8, 10, 12, 14,
+ -1, -1, -1, -1, -1, -1, -1, -1,
+ )
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn yuyv_u_mask() -> __m256i {
+ _mm256_setr_epi8(
+ 1, 5, 9, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 5, 9, 13, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ )
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn yuyv_v_mask() -> __m256i {
+ _mm256_setr_epi8(
+ 3, 7, 11, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 3, 7, 11, 15, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ )
+ }
+
+ unsafe fn scalar_tail<const NV12: bool>(job: &Job, top: usize, x_start: usize) {
+ let mut x = x_start;
+ while x < job.width {
+ unsafe {
+ let (y0, y1, u0, v0) = load_macro(job, top, x);
+ let (y2, y3, u1, v1) = load_macro(job, top + 1, x);
+ *job.y.add(top * job.y_stride + x) = y0;
+ *job.y.add(top * job.y_stride + x + 1) = y1;
+ *job.y.add((top + 1) * job.y_stride + x) = y2;
+ *job.y.add((top + 1) * job.y_stride + x + 1) = y3;
+ let cx = x / 2;
+ let cy = top / 2;
+ let u = avg(u0, u1);
+ let v = avg(v0, v1);
+ if NV12 {
+ let dst = job.uv.add(cy * job.uv_stride + cx * 2);
+ *dst = u;
+ *dst.add(1) = v;
+ } else {
+ *job.u.add(cy * job.u_stride + cx) = u;
+ *job.v.add(cy * job.v_stride + cx) = v;
+ }
+ }
+ x += 2;
+ }
+ }
+
+ unsafe fn load_macro(job: &Job, row: usize, x: usize) -> (u8, u8, u8, u8) {
+ unsafe {
+ let pixel = job.src.add(row * job.src_stride + x * 2);
+ let b0 = *pixel;
+ let b1 = *pixel.add(1);
+ let b2 = *pixel.add(2);
+ let b3 = *pixel.add(3);
+ match job.kind {
+ Packed422::Uyvy => (b1, b3, b0, b2),
+ Packed422::Yuyv => (b0, b2, b1, b3),
+ }
+ }
+ }
+
+ fn avg(a: u8, b: u8) -> u8 {
+ ((u16::from(a) + u16::from(b) + 1) >> 1) as u8
+ }
+ }
+
+ #[cfg(test)]
+ mod tests {
+ use super::*;
+ use ffmpeg::software::scaling::{self, flag::Flags};
+
+ fn avx2_ready() -> bool {
+ #[cfg(target_arch = "x86_64")]
+ {
+ std::arch::is_x86_feature_detected!("avx2")
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ {
+ false
+ }
+ }
+
+ fn fast_scaler(src: Pixel, dst: Pixel, width: u32, height: u32) -> scaling::Context {
+ scaling::Context::get(src, width, height, dst, width, height, Flags::FAST_BILINEAR).unwrap()
+ }
+
+ fn zero_frame(frame: &mut frame::Video) {
+ for plane in 0..frame.planes() {
+ frame.data_mut(plane).fill(0);
+ }
+ }
+
+ fn fill_packed(frame: &mut frame::Video) {
+ let uyvy = frame.format() == Pixel::UYVY422;
+ let width = frame.width() as usize;
+ let height = frame.height() as usize;
+ let stride = frame.stride(0);
+ let data = frame.data_mut(0);
+ for y in 0..height {
+ let mut x = 0;
+ while x + 1 < width {
+ let y0 = ((x * 17 + y * 3) % 256) as u8;
+ let y1 = ((x * 13 + y * 11 + 40) % 256) as u8;
+ let u = ((x * 5 + y * 7 + 16) % 256) as u8;
+ let v = ((x * 9 + y * 2 + 80) % 256) as u8;
+ let off = y * stride + x * 2;
+ if uyvy {
+ data[off..off + 4].copy_from_slice(&[u, y0, v, y1]);
+ } else {
+ data[off..off + 4].copy_from_slice(&[y0, u, y1, v]);
+ }
+ x += 2;
+ }
+ }
+ }
+
+ fn reference(src: &frame::Video) -> (Vec<u8>, Vec<u8>, Vec<u8>) {
+ let width = src.width() as usize;
+ let height = src.height() as usize;
+ let stride = src.stride(0);
+ let data = src.data(0);
+ let uyvy = src.format() == Pixel::UYVY422;
+ let cw = width / 2;
+ let ch = height / 2;
+ let mut y_plane = vec![0u8; width * height];
+ let mut u_plane = vec![0u8; cw * ch];
+ let mut v_plane = vec![0u8; cw * ch];
+ let mut row = 0;
+ while row < height {
+ let mut x = 0;
+ while x < width {
+ let (y0, y1, u0, v0) = macro_at(data, stride, row, x, uyvy);
+ let (y2, y3, u1, v1) = macro_at(data, stride, row + 1, x, uyvy);
+ y_plane[row * width + x] = y0;
+ y_plane[row * width + x + 1] = y1;
+ y_plane[(row + 1) * width + x] = y2;
+ y_plane[(row + 1) * width + x + 1] = y3;
+ let cx = x / 2;
+ let cy = row / 2;
+ u_plane[cy * cw + cx] = ((u16::from(u0) + u16::from(u1) + 1) >> 1) as u8;
+ v_plane[cy * cw + cx] = ((u16::from(v0) + u16::from(v1) + 1) >> 1) as u8;
+ x += 2;
+ }
+ row += 2;
+ }
+ (y_plane, u_plane, v_plane)
+ }
+
+ fn macro_at(data: &[u8], stride: usize, row: usize, x: usize, uyvy: bool) -> (u8, u8, u8, u8) {
+ let off = row * stride + x * 2;
+ let b0 = data[off];
+ let b1 = data[off + 1];
+ let b2 = data[off + 2];
+ let b3 = data[off + 3];
+ if uyvy {
+ (b1, b3, b0, b2)
+ } else {
+ (b0, b2, b1, b3)
+ }
+ }
+
+ fn assert_matches_reference(actual: &frame::Video, y: &[u8], u: &[u8], v: &[u8]) {
+ let width = actual.width() as usize;
+ let height = actual.height() as usize;
+ let cw = width / 2;
+ let ch = height / 2;
+ for row in 0..height {
+ let got = &actual.data(0)[row * actual.stride(0)..][..width];
+ assert_eq!(got, &y[row * width..][..width], "y row {row}");
+ }
+ if actual.format() == Pixel::NV12 {
+ for row in 0..ch {
+ let got = &actual.data(1)[row * actual.stride(1)..][..cw * 2];
+ for cx in 0..cw {
+ assert_eq!(got[cx * 2], u[row * cw + cx], "nv12 u {cx},{row}");
+ assert_eq!(got[cx * 2 + 1], v[row * cw + cx], "nv12 v {cx},{row}");
+ }
+ }
+ } else {
+ for row in 0..ch {
+ let got_u = &actual.data(1)[row * actual.stride(1)..][..cw];
+ let got_v = &actual.data(2)[row * actual.stride(2)..][..cw];
+ assert_eq!(got_u, &u[row * cw..][..cw], "u row {row}");
+ assert_eq!(got_v, &v[row * cw..][..cw], "v row {row}");
+ }
+ }
+ }
+
+ struct PlaneStats {
+ max_diff: u8,
+ mean: f64,
+ col: usize,
+ row: usize,
+ }
+
+ fn plane_delta(
+ actual: &frame::Video,
+ expected: &frame::Video,
+ plane: usize,
+ width: usize,
+ height: usize,
+ ) -> PlaneStats {
+ let mut max_diff = 0u8;
+ let mut sum = 0u64;
+ let mut count = 0u64;
+ let mut col = 0;
+ let mut row_at = 0;
+ for row in 0..height {
+ let actual_row = &actual.data(plane)[row * actual.stride(plane)..];
+ let expected_row = &expected.data(plane)[row * expected.stride(plane)..];
+ for x in 0..width {
+ let diff = actual_row[x].abs_diff(expected_row[x]);
+ sum += u64::from(diff);
+ count += 1;
+ if diff > max_diff {
+ max_diff = diff;
+ col = x;
+ row_at = row;
+ }
+ }
+ }
+ PlaneStats {
+ max_diff,
+ mean: sum as f64 / count as f64,
+ col,
+ row: row_at,
+ }
+ }
+
+ fn assert_within_swscale(actual: &frame::Video, expected: &frame::Video) {
+ let width = actual.width() as usize;
+ let height = actual.height() as usize;
+ let y_stats = plane_delta(actual, expected, 0, width, height);
+ let y_limit = if actual.format() == Pixel::NV12 { 4 } else { 2 };
+ assert!(
+ y_stats.max_diff <= y_limit && y_stats.mean < 1.0,
+ "{width}x{height} luma max {} mean {:.3} at {},{}",
+ y_stats.max_diff,
+ y_stats.mean,
+ y_stats.col,
+ y_stats.row
+ );
+ let cw = width / 2;
+ let ch = height / 2;
+ if actual.format() == Pixel::NV12 {
+ let uv_stats = plane_delta(actual, expected, 1, cw * 2, ch);
+ assert!(
+ uv_stats.max_diff <= 3 && uv_stats.mean < 1.5,
+ "{width}x{height} nv12 max {} mean {:.3} at {},{}",
+ uv_stats.max_diff,
+ uv_stats.mean,
+ uv_stats.col,
+ uv_stats.row
+ );
+ } else {
+ let u_stats = plane_delta(actual, expected, 1, cw, ch);
+ let v_stats = plane_delta(actual, expected, 2, cw, ch);
+ assert!(
+ u_stats.max_diff <= 3
+ && v_stats.max_diff <= 3
+ && u_stats.mean < 1.5
+ && v_stats.mean < 1.5,
+ "{width}x{height} chroma u max {} mean {:.3} at {},{} v max {} mean {:.3} at {},{}",
+ u_stats.max_diff,
+ u_stats.mean,
+ u_stats.col,
+ u_stats.row,
+ v_stats.max_diff,
+ v_stats.mean,
+ v_stats.col,
+ v_stats.row
+ );
+ }
+ }
+
+ fn assert_active_eq(actual: &frame::Video, expected: &frame::Video) {
+ for plane in 0..expected.planes() {
+ let row_bytes = if expected.format() == Pixel::NV12 && plane == 1 {
+ expected.width() as usize
+ } else {
+ expected.plane_width(plane) as usize
+ };
+ for row in 0..expected.plane_height(plane) as usize {
+ let got = &actual.data(plane)[row * actual.stride(plane)..][..row_bytes];
+ let want = &expected.data(plane)[row * expected.stride(plane)..][..row_bytes];
+ assert_eq!(got, want, "plane {plane} row {row}");
+ }
+ }
+ }
+
+ fn check_even(format: Pixel, width: u32, height: u32, dst_format: Pixel) {
+ let mut src = frame::Video::new(format, width, height);
+ fill_packed(&mut src);
+ let (y, u, v) = reference(&src);
+ let mut kernel = frame::Video::new(dst_format, width, height);
+ let mut sws = frame::Video::new(dst_format, width, height);
+ let mut scaler = fast_scaler(format, dst_format, width, height);
+ scaler.run(&src, &mut sws).unwrap();
+ if avx2_ready() {
+ assert!(
+ try_convert(&src, &mut kernel),
+ "{format:?} {width}x{height} -> {dst_format:?}"
+ );
+ assert_matches_reference(&kernel, &y, &u, &v);
+ assert_within_swscale(&kernel, &sws);
+ let mut via = frame::Video::new(dst_format, width, height);
+ let mut via_scaler = fast_scaler(format, dst_format, width, height);
+ convert_or_scale(&src, &mut via, &mut via_scaler).unwrap();
+ assert_active_eq(&via, &kernel);
+ } else {
+ assert!(!try_convert(&src, &mut kernel));
+ let mut via = frame::Video::new(dst_format, width, height);
+ let mut via_scaler = fast_scaler(format, dst_format, width, height);
+ convert_or_scale(&src, &mut via, &mut via_scaler).unwrap();
+ assert_active_eq(&via, &sws);
+ }
+ }
+
+ #[test]
+ fn even_uyvy_and_yuyv_match_reference_and_swscale() {
+ ffmpeg::init().unwrap();
+ let sizes = [
+ (2u32, 2u32),
+ (16, 2),
+ (18, 4),
+ (32, 8),
+ (48, 6),
+ (64, 48),
+ (66, 40),
+ (320, 180),
+ ];
+ for format in [Pixel::UYVY422, Pixel::YUYV422] {
+ for (width, height) in sizes {
+ check_even(format, width, height, Pixel::YUV420P);
+ }
+ }
+ }
+
+ #[test]
+ fn even_nv12_matches_where_the_encoder_outputs_nv12() {
+ ffmpeg::init().unwrap();
+ for format in [Pixel::UYVY422, Pixel::YUYV422] {
+ for (width, height) in [(16u32, 2u32), (18, 4), (64, 48), (66, 40)] {
+ check_even(format, width, height, Pixel::NV12);
+ }
+ }
+ }
+
+ #[test]
+ fn odd_sizes_fall_back_to_swscale() {
+ ffmpeg::init().unwrap();
+ let mut compared = 0u32;
+ for format in [Pixel::UYVY422, Pixel::YUYV422] {
+ for (width, height) in [
+ (2u32, 1u32),
+ (64, 63),
+ (32, 15),
+ (1, 2),
+ (65, 64),
+ (31, 17),
+ (65, 63),
+ ] {
+ let mut src = frame::Video::new(format, width, height);
+ fill_packed(&mut src);
+ let mut probe = frame::Video::new(Pixel::YUV420P, width, height);
+ assert!(
+ !try_convert(&src, &mut probe),
+ "{format:?} {width}x{height} should stay on swscale"
+ );
+ let Ok(mut via_scaler) = scaling::Context::get(
+ format,
+ width,
+ height,
+ Pixel::YUV420P,
+ width,
+ height,
+ Flags::FAST_BILINEAR,
+ ) else {
+ continue;
+ };
+ let Ok(mut direct_scaler) = scaling::Context::get(
+ format,
+ width,
+ height,
+ Pixel::YUV420P,
+ width,
+ height,
+ Flags::FAST_BILINEAR,
+ ) else {
+ continue;
+ };
+ let mut via = frame::Video::new(Pixel::YUV420P, width, height);
+ let mut direct = frame::Video::new(Pixel::YUV420P, width, height);
+ // av_frame_get_buffer does not clear memory, and swscale can
+ // leave the odd-height chroma tail untouched.
+ zero_frame(&mut via);
+ zero_frame(&mut direct);
+ convert_or_scale(&src, &mut via, &mut via_scaler).unwrap();
+ direct_scaler.run(&src, &mut direct).unwrap();
+ assert_active_eq(&via, &direct);
+ compared += 1;
+ }
+ }
+ assert!(compared > 0, "swscale rejected every odd size");
+ }
+
+ #[test]
+ fn scaling_and_other_formats_stay_on_swscale() {
+ ffmpeg::init().unwrap();
+ let src = frame::Video::new(Pixel::UYVY422, 64, 48);
+ let mut scaled = frame::Video::new(Pixel::YUV420P, 32, 24);
+ assert!(!try_convert(&src, &mut scaled));
+ let mut yuv422 = frame::Video::new(Pixel::YUV422P, 64, 48);
+ assert!(!try_convert(&src, &mut yuv422));
+ for format in [Pixel::BGRA, Pixel::YUV422P, Pixel::NV12, Pixel::RGB24] {
+ let other = frame::Video::new(format, 32, 16);
+ let mut yuv = frame::Video::new(Pixel::YUV420P, 32, 16);
+ assert!(!try_convert(&other, &mut yuv), "{format:?}");
+ }
+
+ let mut packed = frame::Video::new(Pixel::YUYV422, 64, 48);
+ fill_packed(&mut packed);
+ let mut scaler = scaling::Context::get(
+ Pixel::YUYV422,
+ 64,
+ 48,
+ Pixel::YUV420P,
+ 32,
+ 24,
+ Flags::BICUBIC,
+ )
+ .unwrap();
+ let mut direct_scaler = scaling::Context::get(
+ Pixel::YUYV422,
+ 64,
+ 48,
+ Pixel::YUV420P,
+ 32,
+ 24,
+ Flags::BICUBIC,
+ )
+ .unwrap();
+ let mut via = frame::Video::new(Pixel::YUV420P, 32, 24);
+ let mut direct = frame::Video::new(Pixel::YUV420P, 32, 24);
+ convert_or_scale(&packed, &mut via, &mut scaler).unwrap();
+ direct_scaler.run(&packed, &mut direct).unwrap();
+ assert_active_eq(&via, &direct);
+ }
+
+ #[test]
+ fn libx264_queue_accepts_packed_yuv() {
+ ffmpeg::init().unwrap();
+ queue_one(Pixel::UYVY422);
+ encode_one(Pixel::YUYV422);
+ }
+
+ fn queue_one(format: Pixel) {
+ if ffmpeg::codec::encoder::find_by_name("libx264").is_none() {
+ return;
+ }
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("packed.mp4");
+ let mut output = ffmpeg::format::output_as(&path, "mp4").unwrap();
+ let info = cap_media_info::VideoInfo::from_raw_ffmpeg(format, 64, 48, 30);
+ let mut encoder = super::super::h264::H264Encoder::builder(info)
+ .with_preset(super::super::h264::H264Preset::Ultrafast)
+ .with_crf(18)
+ .with_encoder_priority_override(&["libx264"])
+ .build(&mut output)
+ .unwrap();
+ let requirements = encoder.conversion_requirements();
+ assert_eq!(requirements.output_format, Pixel::YUV420P);
+ assert!(requirements.needs_conversion);
+ let mut frame = frame::Video::new(format, 64, 48);
+ fill_packed(&mut frame);
+ let mut sink = crate::PacketQueue::new(ffmpeg::Rational(1, 90_000));
+ encoder
+ .queue_frame_to(frame, std::time::Duration::ZERO, &mut sink)
+ .unwrap();
+ encoder.flush_to(&mut sink).unwrap();
+ assert!(!sink.take().is_empty());
+ }
+
+ fn encode_one(format: Pixel) {
+ if ffmpeg::codec::encoder::find_by_name("libx264").is_none() {
+ return;
+ }
+ let info = cap_media_info::VideoInfo::from_raw_ffmpeg(format, 64, 48, 30);
+ let mut encoder = super::super::h264::H264Encoder::builder(info)
+ .with_preset(super::super::h264::H264Preset::Ultrafast)
+ .with_crf(18)
+ .with_encoder_priority_override(&["libx264"])
+ .build_standalone()
+ .unwrap();
+ assert_eq!(encoder.output_format(), Pixel::YUV420P);
+ assert!(encoder.needs_conversion());
+ let mut frame = frame::Video::new(format, 64, 48);
+ fill_packed(&mut frame);
+ let mut packets = Vec::new();
+ encoder
+ .encode_frame(frame, std::time::Duration::ZERO, |packet| {
+ packets.push(packet.data);
+ Ok(())
+ })
+ .unwrap();
+ encoder
+ .flush(|packet| {
+ packets.push(packet.data);
+ Ok(())
+ })
+ .unwrap();
+ assert!(!packets.is_empty());
+ }
+
+ #[test]
+ #[ignore = "release timing bench, run with --ignored --nocapture"]
+ fn bench_packed_yuv_conversion() {
+ ffmpeg::init().unwrap();
+ let model = std::fs::read_to_string("/proc/cpuinfo")
+ .ok()
+ .and_then(|text| {
+ text.lines()
+ .find(|line| line.starts_with("model name"))
+ .and_then(|line| line.split(':').nth(1))
+ .map(|value| value.trim().to_string())
+ })
+ .unwrap_or_else(|| "unknown".into());
+ println!("cpu: {model}");
+ println!("avx2: {}", avx2_ready());
+ println!("warmup 5 excluded; median is the middle of 31 sorted samples");
+ for format in [Pixel::UYVY422, Pixel::YUYV422] {
+ for (width, height) in [(1280u32, 720u32), (1920, 1080), (3840, 2160)] {
+ bench_conversion(format, width, height);
+ }
+ }
+ }
+
+ fn bench_conversion(format: Pixel, width: u32, height: u32) {
+ let mut src = frame::Video::new(format, width, height);
+ fill_packed(&mut src);
+ let mut kernel_dst = frame::Video::new(Pixel::YUV420P, width, height);
+ let mut sws_dst = frame::Video::new(Pixel::YUV420P, width, height);
+ let mut kernel_scaler = fast_scaler(format, Pixel::YUV420P, width, height);
+ let mut sws_scaler = fast_scaler(format, Pixel::YUV420P, width, height);
+ let sws = time_median(5, 31, || {
+ sws_scaler.run(&src, &mut sws_dst).unwrap();
+ let _ = std::hint::black_box(sws_dst.data(0));
+ let _ = std::hint::black_box(sws_dst.data(1));
+ let _ = std::hint::black_box(sws_dst.data(2));
+ });
+ let kernel = time_median(5, 31, || {
+ convert_or_scale(&src, &mut kernel_dst, &mut kernel_scaler).unwrap();
+ let _ = std::hint::black_box(kernel_dst.data(0));
+ let _ = std::hint::black_box(kernel_dst.data(1));
+ let _ = std::hint::black_box(kernel_dst.data(2));
+ });
+ println!(
+ "{format:?} {width}x{height} swscale_fast_bilinear median {:.3} ms n=31",
+ sws / 1_000_000.0
+ );
+ println!(
+ "{format:?} {width}x{height} convert_or_scale median {:.3} ms n=31 speedup {:.2}x",
+ kernel / 1_000_000.0,
+ sws / kernel
+ );
+ }
+
+ fn time_median(warmup: usize, runs: usize, mut body: impl FnMut()) -> f64 {
+ for _ in 0..warmup {
+ body();
+ }
+ let mut samples = Vec::with_capacity(runs);
+ for _ in 0..runs {
+ let started = std::time::Instant::now();
+ body();
+ samples.push(started.elapsed().as_nanos());
+ }
+ samples.sort_unstable();
+ samples[runs / 2] as f64
+ }
+ }
diff --cc crates/enc-ffmpeg/src/video/rgb_to_yuv.rs
index 5c5e5b281,000000000..42c39645e
mode 100644,000000..100644
--- a/crates/enc-ffmpeg/src/video/rgb_to_yuv.rs
+++ b/crates/enc-ffmpeg/src/video/rgb_to_yuv.rs
@@@ -1,1235 -1,0 +1,1232 @@@
+use ffmpeg::format::Pixel;
+use ffmpeg::frame;
+
+// 16.16 BT.709, full range RGB to limited YUV. The luma row sums to 56284
+// (219/255) and each chroma row sums to 0, so a grey pixel stays on the
+// limited-range axis (black 16/128/128, white 235/128/128).
+#[cfg(any(target_arch = "x86_64", test))]
+const Y_R: i32 = 11_966;
+#[cfg(any(target_arch = "x86_64", test))]
+const Y_G: i32 = 40_254;
+#[cfg(any(target_arch = "x86_64", test))]
+const Y_B: i32 = 4_064;
+#[cfg(any(target_arch = "x86_64", test))]
+const CB_R: i32 = -6_596;
+#[cfg(any(target_arch = "x86_64", test))]
+const CB_G: i32 = -22_189;
+#[cfg(any(target_arch = "x86_64", test))]
+const CB_B: i32 = 28_785;
+#[cfg(any(target_arch = "x86_64", test))]
+const CR_R: i32 = 28_785;
+#[cfg(any(target_arch = "x86_64", test))]
+const CR_G: i32 = -26_145;
+#[cfg(any(target_arch = "x86_64", test))]
+const CR_B: i32 = -2_640;
+#[cfg(any(target_arch = "x86_64", test))]
+const ROUND: i32 = 1 << 15;
+
+#[cfg(any(target_arch = "x86_64", test))]
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum ChannelOrder {
+ Bgr,
+ Rgb,
+}
+
+#[cfg(any(target_arch = "x86_64", test))]
+struct Job {
+ src: *const u8,
+ src_stride: usize,
+ #[cfg(target_arch = "x86_64")]
+ width: usize,
+ #[cfg(target_arch = "x86_64")]
+ height: usize,
+ order: ChannelOrder,
+ y: *mut u8,
+ y_stride: usize,
+ u: *mut u8,
+ u_stride: usize,
+ v: *mut u8,
+ v_stride: usize,
+ uv: *mut u8,
+ uv_stride: usize,
+ nv12: bool,
+}
+
+#[cfg(any(target_arch = "x86_64", test))]
+fn luma(r: i32, g: i32, b: i32) -> u8 {
+ let y = (Y_R * r + Y_G * g + Y_B * b + ROUND) >> 16;
+ (16 + y).clamp(16, 235) as u8
+}
+
+#[cfg(any(target_arch = "x86_64", test))]
+fn chroma_from_sum(r_sum: i32, g_sum: i32, b_sum: i32, n: i32) -> (u8, u8) {
+ let shift = 16 + n.trailing_zeros();
+ let bias = n << 15;
+ let cb = (CB_R * r_sum + CB_G * g_sum + CB_B * b_sum + bias) >> shift;
+ let cr = (CR_R * r_sum + CR_G * g_sum + CR_B * b_sum + bias) >> shift;
+ (
+ (128 + cb).clamp(16, 240) as u8,
+ (128 + cr).clamp(16, 240) as u8,
+ )
+}
+
+#[cfg(any(target_arch = "x86_64", test))]
+unsafe fn load_rgb(job: &Job, x: usize, y: usize) -> (i32, i32, i32) {
+ unsafe {
+ let pixel = job.src.add(y * job.src_stride + x * 4);
+ let c0 = i32::from(*pixel);
+ let c1 = i32::from(*pixel.add(1));
+ let c2 = i32::from(*pixel.add(2));
+ match job.order {
+ ChannelOrder::Bgr => (c2, c1, c0),
+ ChannelOrder::Rgb => (c0, c1, c2),
+ }
+ }
+}
+
+#[cfg(any(target_arch = "x86_64", test))]
+unsafe fn store_chroma(job: &Job, cx: usize, cy: usize, cb: u8, cr: u8) {
+ unsafe {
+ if job.nv12 {
+ let offset = cy * job.uv_stride + cx * 2;
+ *job.uv.add(offset) = cb;
+ *job.uv.add(offset + 1) = cr;
+ } else {
+ *job.u.add(cy * job.u_stride + cx) = cb;
+ *job.v.add(cy * job.v_stride + cx) = cr;
+ }
+ }
+}
+
+/// `x0` and `y0` are even. `x1` and `y1` are exclusive.
+#[cfg(any(target_arch = "x86_64", test))]
+unsafe fn convert_range(job: &Job, x0: usize, x1: usize, y0: usize, y1: usize) {
+ unsafe {
+ let mut y = y0;
+ while y < y1 {
+ let rows = if y + 1 < y1 { 2 } else { 1 };
+ let mut x = x0;
+ while x < x1 {
+ let cols = if x + 1 < x1 { 2 } else { 1 };
+ let mut r_sum = 0;
+ let mut g_sum = 0;
+ let mut b_sum = 0;
+ let mut row = 0;
+ while row < rows {
+ let mut col = 0;
+ while col < cols {
+ let (r, g, b) = load_rgb(job, x + col, y + row);
+ r_sum += r;
+ g_sum += g;
+ b_sum += b;
+ *job.y.add((y + row) * job.y_stride + (x + col)) = luma(r, g, b);
+ col += 1;
+ }
+ row += 1;
+ }
+ let (cb, cr) = chroma_from_sum(r_sum, g_sum, b_sum, (rows * cols) as i32);
+ store_chroma(job, x / 2, y / 2, cb, cr);
+ x += cols;
+ }
+ y += rows;
+ }
+ }
+}
+
+fn avx2_runtime() -> bool {
+ #[cfg(target_arch = "x86_64")]
+ {
+ std::arch::is_x86_feature_detected!("avx2")
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ {
+ false
+ }
+}
+
+#[cfg(any(target_arch = "x86_64", test))]
+fn execute(job: &Job) {
+ #[cfg(target_arch = "x86_64")]
+ {
+ // SAFETY: `try_convert` calls this only after `avx2_runtime`, and each
+ // plane covers the active rectangle.
+ unsafe { avx2::execute(job) };
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ let _ = job;
+}
+
+fn positive_linesize(frame: &frame::Video, plane: usize) -> Option<usize> {
+ if plane >= frame.planes() {
+ return None;
+ }
+ let raw = unsafe { (*frame.as_ptr()).linesize[plane] };
+ if raw <= 0 { None } else { Some(raw as usize) }
+}
+
+fn covers(stride: usize, row_bytes: usize, height: usize, len: usize) -> bool {
+ if row_bytes == 0 || height == 0 {
+ return true;
+ }
+ if stride < row_bytes {
+ return false;
+ }
+ stride
+ .checked_mul(height - 1)
+ .and_then(|last| last.checked_add(row_bytes))
+ .is_some_and(|need| need <= len)
+}
+
+/// Converts same-size packed BGRZ/BGRA/RGBZ/RGBA into `dst` when both
+/// dimensions are even, `dst` is YUV420P or NV12, and AVX2 is available at
+/// runtime. Returns false so the caller keeps swscale. That includes an odd
+/// width or height, and every machine without AVX2.
+pub(crate) fn try_convert(src: &frame::Video, dst: &mut frame::Video) -> bool {
+ if src.width() != dst.width() || src.height() != dst.height() {
+ return false;
+ }
+ #[cfg(any(target_arch = "x86_64", test))]
+ let order = match src.format() {
+ Pixel::BGRA | Pixel::BGRZ => ChannelOrder::Bgr,
+ Pixel::RGBA | Pixel::RGBZ => ChannelOrder::Rgb,
+ _ => return false,
+ };
+ #[cfg(not(any(target_arch = "x86_64", test)))]
+ if !matches!(
+ src.format(),
+ Pixel::BGRA | Pixel::BGRZ | Pixel::RGBA | Pixel::RGBZ
+ ) {
+ return false;
+ }
+ let nv12 = match dst.format() {
+ Pixel::YUV420P => false,
+ Pixel::NV12 => true,
+ _ => return false,
+ };
+ let width = src.width() as usize;
+ let height = src.height() as usize;
+ if !width.is_multiple_of(2) || !height.is_multiple_of(2) {
+ return false;
+ }
+ if width == 0 || height == 0 {
+ return true;
+ }
+ if !avx2_runtime() {
+ return false;
+ }
+ let Some(src_stride) = positive_linesize(src, 0) else {
+ return false;
+ };
+ let Some(y_stride) = positive_linesize(dst, 0) else {
+ return false;
+ };
+ if !covers(src_stride, width * 4, height, src.data(0).len())
+ || !covers(y_stride, width, height, dst.data(0).len())
+ {
+ return false;
+ }
+
+ let chroma_w = width.div_ceil(2);
+ let chroma_h = height.div_ceil(2);
+ let (u_stride, v_stride, uv_stride) = if nv12 {
+ let Some(uv_stride) = positive_linesize(dst, 1) else {
+ return false;
+ };
+ if !covers(uv_stride, chroma_w * 2, chroma_h, dst.data(1).len()) {
+ return false;
+ }
+ (0, 0, uv_stride)
+ } else {
+ let Some(u_stride) = positive_linesize(dst, 1) else {
+ return false;
+ };
+ let Some(v_stride) = positive_linesize(dst, 2) else {
+ return false;
+ };
+ if !covers(u_stride, chroma_w, chroma_h, dst.data(1).len())
+ || !covers(v_stride, chroma_w, chroma_h, dst.data(2).len())
+ {
+ return false;
+ }
+ (u_stride, v_stride, 0)
+ };
+ #[cfg(not(any(target_arch = "x86_64", test)))]
+ let _ = (u_stride, v_stride, uv_stride);
+
+ #[cfg(any(target_arch = "x86_64", test))]
+ {
+ let src_ptr = src.data(0).as_ptr();
+ let y_ptr = dst.data_mut(0).as_mut_ptr();
+ let (u_ptr, v_ptr, uv_ptr) = if nv12 {
+ (
+ std::ptr::null_mut(),
+ std::ptr::null_mut(),
+ dst.data_mut(1).as_mut_ptr(),
+ )
+ } else {
+ let u_ptr = dst.data_mut(1).as_mut_ptr();
+ let v_ptr = dst.data_mut(2).as_mut_ptr();
+ (u_ptr, v_ptr, std::ptr::null_mut())
+ };
+
+ execute(&Job {
+ src: src_ptr,
+ src_stride,
+ #[cfg(target_arch = "x86_64")]
+ width,
+ #[cfg(target_arch = "x86_64")]
+ height,
+ order,
+ y: y_ptr,
+ y_stride,
+ u: u_ptr,
+ u_stride,
+ v: v_ptr,
+ v_stride,
+ uv: uv_ptr,
+ uv_stride,
+ nv12,
+ });
+ }
+ true
+}
+
++#[cfg(test)]
+pub(crate) fn convert_or_scale(
+ src: &frame::Video,
+ dst: &mut frame::Video,
+ scaler: &mut ffmpeg::software::scaling::Context,
+) -> Result<(), ffmpeg::Error> {
- if try_convert(src, dst) {
- Ok(())
- } else {
- scaler.run(src, dst)
- }
++ super::convert_or_scale(src, dst, scaler)
+}
+
+#[cfg(target_arch = "x86_64")]
+mod avx2 {
+ use std::arch::x86_64::*;
+
+ use super::{Job, ROUND, convert_range};
+
+ struct Coeffs {
+ y_r: __m256i,
+ y_g: __m256i,
+ y_b: __m256i,
+ cb_r: __m256i,
+ cb_g: __m256i,
+ cb_b: __m256i,
+ cr_r: __m256i,
+ cr_g: __m256i,
+ cr_b: __m256i,
+ round: __m256i,
+ chroma_round: __m256i,
+ y_bias: __m256i,
+ c_bias: __m256i,
+ y_lo: __m256i,
+ y_hi: __m256i,
+ c_lo: __m256i,
+ c_hi: __m256i,
+ mask: __m256i,
+ pair_mask: __m256i,
+ gather: __m256i,
+ }
+
+ impl Coeffs {
+ #[target_feature(enable = "avx2")]
+ unsafe fn new() -> Self {
+ Self {
+ y_r: _mm256_set1_epi32(super::Y_R),
+ y_g: _mm256_set1_epi32(super::Y_G),
+ y_b: _mm256_set1_epi32(super::Y_B),
+ cb_r: _mm256_set1_epi32(super::CB_R),
+ cb_g: _mm256_set1_epi32(super::CB_G),
+ cb_b: _mm256_set1_epi32(super::CB_B),
+ cr_r: _mm256_set1_epi32(super::CR_R),
+ cr_g: _mm256_set1_epi32(super::CR_G),
+ cr_b: _mm256_set1_epi32(super::CR_B),
+ round: _mm256_set1_epi32(ROUND),
+ chroma_round: _mm256_set1_epi32(4 * ROUND),
+ y_bias: _mm256_set1_epi32(16),
+ c_bias: _mm256_set1_epi32(128),
+ y_lo: _mm256_set1_epi32(16),
+ y_hi: _mm256_set1_epi32(235),
+ c_lo: _mm256_set1_epi32(16),
+ c_hi: _mm256_set1_epi32(240),
+ mask: _mm256_set1_epi32(0x00FF),
+ pair_mask: _mm256_set1_epi64x(0xFFFF_FFFF),
+ gather: _mm256_setr_epi32(0, 1, 4, 5, 2, 3, 6, 7),
+ }
+ }
+ }
+
+ pub(super) unsafe fn execute(job: &Job) {
+ unsafe {
+ match (job.order, job.nv12) {
+ (super::ChannelOrder::Bgr, false) => convert::<true, false>(job),
+ (super::ChannelOrder::Bgr, true) => convert::<true, true>(job),
+ (super::ChannelOrder::Rgb, false) => convert::<false, false>(job),
+ (super::ChannelOrder::Rgb, true) => convert::<false, true>(job),
+ }
+ }
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn convert<const BGR: bool, const NV12: bool>(job: &Job) {
+ unsafe {
+ let coeffs = Coeffs::new();
+ let mut y = 0;
+ while y + 1 < job.height {
+ let mut x = 0;
+ while x + 16 <= job.width {
+ kernel8::<BGR, NV12>(job, &coeffs, x, y);
+ kernel8::<BGR, NV12>(job, &coeffs, x + 8, y);
+ x += 16;
+ }
+ if x + 8 <= job.width {
+ kernel8::<BGR, NV12>(job, &coeffs, x, y);
+ x += 8;
+ }
+ if x < job.width {
+ convert_range(job, x, job.width, y, y + 2);
+ }
+ y += 2;
+ }
+ if y < job.height {
+ convert_range(job, 0, job.width, y, y + 1);
+ }
+ }
+ }
+
+ #[target_feature(enable = "avx2")]
+ #[inline]
+ unsafe fn kernel8<const BGR: bool, const NV12: bool>(
+ job: &Job,
+ coeffs: &Coeffs,
+ x: usize,
+ y: usize,
+ ) {
+ unsafe {
+ let row0 = job.src.add(y * job.src_stride + x * 4);
+ let row1 = job.src.add((y + 1) * job.src_stride + x * 4);
+ let (r0, g0, b0) =
+ unpack8::<BGR>(_mm256_loadu_si256(row0 as *const __m256i), coeffs.mask);
+ let (r1, g1, b1) =
+ unpack8::<BGR>(_mm256_loadu_si256(row1 as *const __m256i), coeffs.mask);
+ store_y8(job.y.add(y * job.y_stride + x), luma8(r0, g0, b0, coeffs));
+ store_y8(
+ job.y.add((y + 1) * job.y_stride + x),
+ luma8(r1, g1, b1, coeffs),
+ );
+ let r_sum = _mm256_add_epi32(
+ pair_sum(r0, coeffs.pair_mask),
+ pair_sum(r1, coeffs.pair_mask),
+ );
+ let g_sum = _mm256_add_epi32(
+ pair_sum(g0, coeffs.pair_mask),
+ pair_sum(g1, coeffs.pair_mask),
+ );
+ let b_sum = _mm256_add_epi32(
+ pair_sum(b0, coeffs.pair_mask),
+ pair_sum(b1, coeffs.pair_mask),
+ );
+ let cb = chroma8(
+ r_sum,
+ g_sum,
+ b_sum,
+ coeffs.cb_r,
+ coeffs.cb_g,
+ coeffs.cb_b,
+ coeffs,
+ );
+ let cr = chroma8(
+ r_sum,
+ g_sum,
+ b_sum,
+ coeffs.cr_r,
+ coeffs.cr_g,
+ coeffs.cr_b,
+ coeffs,
+ );
+ store_chroma8::<NV12>(job, x, y / 2, cb, cr, coeffs.gather);
+ }
+ }
+
+ #[target_feature(enable = "avx2")]
+ #[inline]
+ unsafe fn unpack8<const BGR: bool>(pix: __m256i, mask: __m256i) -> (__m256i, __m256i, __m256i) {
+ let c0 = _mm256_and_si256(pix, mask);
+ let c1 = _mm256_and_si256(_mm256_srli_epi32(pix, 8), mask);
+ let c2 = _mm256_and_si256(_mm256_srli_epi32(pix, 16), mask);
+ if BGR { (c2, c1, c0) } else { (c0, c1, c2) }
+ }
+
+ #[target_feature(enable = "avx2")]
+ #[inline]
+ unsafe fn luma8(r: __m256i, g: __m256i, b: __m256i, coeffs: &Coeffs) -> __m256i {
+ let mut acc = _mm256_mullo_epi32(r, coeffs.y_r);
+ acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(g, coeffs.y_g));
+ acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(b, coeffs.y_b));
+ acc = _mm256_srai_epi32(_mm256_add_epi32(acc, coeffs.round), 16);
+ acc = _mm256_add_epi32(acc, coeffs.y_bias);
+ _mm256_min_epi32(_mm256_max_epi32(acc, coeffs.y_lo), coeffs.y_hi)
+ }
+
+ #[target_feature(enable = "avx2")]
+ #[inline]
+ unsafe fn pair_sum(v: __m256i, pair_mask: __m256i) -> __m256i {
+ let shifted = _mm256_srli_epi64(v, 32);
+ let sum = _mm256_add_epi32(v, shifted);
+ _mm256_and_si256(sum, pair_mask)
+ }
+
+ #[target_feature(enable = "avx2")]
+ #[inline]
+ unsafe fn chroma8(
+ r_sum: __m256i,
+ g_sum: __m256i,
+ b_sum: __m256i,
+ c_r: __m256i,
+ c_g: __m256i,
+ c_b: __m256i,
+ coeffs: &Coeffs,
+ ) -> __m256i {
+ let mut acc = _mm256_mullo_epi32(r_sum, c_r);
+ acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(g_sum, c_g));
+ acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(b_sum, c_b));
+ acc = _mm256_srai_epi32(_mm256_add_epi32(acc, coeffs.chroma_round), 18);
+ acc = _mm256_add_epi32(acc, coeffs.c_bias);
+ _mm256_min_epi32(_mm256_max_epi32(acc, coeffs.c_lo), coeffs.c_hi)
+ }
+
+ #[target_feature(enable = "avx2")]
+ #[inline]
+ unsafe fn store_y8(ptr: *mut u8, v: __m256i) {
+ unsafe {
+ let packed = _mm256_packs_epi32(v, _mm256_setzero_si256());
+ let grouped = _mm256_permute4x64_epi64(packed, 0xD8);
+ let bytes = _mm_packus_epi16(_mm256_castsi256_si128(grouped), _mm_setzero_si128());
+ _mm_storel_epi64(ptr as *mut __m128i, bytes);
+ }
+ }
+
+ #[target_feature(enable = "avx2")]
+ #[inline]
+ unsafe fn store_chroma8<const NV12: bool>(
+ job: &Job,
+ x: usize,
+ cy: usize,
+ cb: __m256i,
+ cr: __m256i,
+ gather: __m256i,
+ ) {
+ unsafe {
+ let cb = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi32(cb, 0xD8), gather);
+ let cr = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi32(cr, 0xD8), gather);
+ if NV12 {
+ let cb128 = _mm256_castsi256_si128(cb);
+ let cr128 = _mm256_castsi256_si128(cr);
+ let lo = _mm_unpacklo_epi32(cb128, cr128);
+ let hi = _mm_unpackhi_epi32(cb128, cr128);
+ let bytes = _mm_packus_epi16(_mm_packs_epi32(lo, hi), _mm_setzero_si128());
+ _mm_storel_epi64(job.uv.add(cy * job.uv_stride + x) as *mut __m128i, bytes);
+ } else {
+ store_u32(job.u.add(cy * job.u_stride + x / 2), cb);
+ store_u32(job.v.add(cy * job.v_stride + x / 2), cr);
+ }
+ }
+ }
+
+ #[target_feature(enable = "avx2")]
+ #[inline]
+ unsafe fn store_u32(ptr: *mut u8, v: __m256i) {
+ unsafe {
+ let low = _mm256_castsi256_si128(v);
+ let bytes = _mm_packus_epi16(
+ _mm_packs_epi32(low, _mm_setzero_si128()),
+ _mm_setzero_si128(),
+ );
+ _mm_storeu_si32(ptr, bytes);
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[derive(Clone, Copy)]
+ enum Kernel {
+ Scalar,
+ Avx2,
+ }
+
+ struct Plane {
+ data: Vec<u8>,
+ stride: usize,
+ row_bytes: usize,
+ }
+
+ fn plane(row_bytes: usize, pad: usize, height: usize) -> Plane {
+ let stride = row_bytes + pad;
+ Plane {
+ data: vec![0xA5; stride * height],
+ stride,
+ row_bytes,
+ }
+ }
+
+ fn swizzle(order: ChannelOrder, r: u8, g: u8, b: u8) -> [u8; 4] {
+ match order {
+ ChannelOrder::Bgr => [b, g, r, 255],
+ ChannelOrder::Rgb => [r, g, b, 255],
+ }
+ }
+
+ fn run_kernel(
+ kernel: Kernel,
+ order: ChannelOrder,
+ nv12: bool,
+ width: usize,
+ height: usize,
+ pad: usize,
+ rgb: impl Fn(usize, usize) -> (u8, u8, u8),
+ ) -> (Plane, Plane, Plane, Plane) {
+ let mut src = plane(width * 4, pad, height);
+ for y in 0..height {
+ for x in 0..width {
+ let (r, g, b) = rgb(x, y);
+ let pixel = swizzle(order, r, g, b);
+ let offset = y * src.stride + x * 4;
+ src.data[offset..offset + 4].copy_from_slice(&pixel);
+ }
+ }
+ let chroma_w = width.div_ceil(2);
+ let chroma_h = height.div_ceil(2);
+ let mut y_plane = plane(width, pad, height);
+ let mut u_plane = plane(chroma_w, pad, chroma_h);
+ let mut v_plane = plane(chroma_w, pad, chroma_h);
+ let mut uv_plane = plane(chroma_w * 2, pad, chroma_h);
+ let job = Job {
+ src: src.data.as_ptr(),
+ src_stride: src.stride,
+ #[cfg(target_arch = "x86_64")]
+ width,
+ #[cfg(target_arch = "x86_64")]
+ height,
+ order,
+ y: y_plane.data.as_mut_ptr(),
+ y_stride: y_plane.stride,
+ u: u_plane.data.as_mut_ptr(),
+ u_stride: u_plane.stride,
+ v: v_plane.data.as_mut_ptr(),
+ v_stride: v_plane.stride,
+ uv: uv_plane.data.as_mut_ptr(),
+ uv_stride: uv_plane.stride,
+ nv12,
+ };
+ match kernel {
+ Kernel::Scalar => unsafe { convert_range(&job, 0, width, 0, height) },
+ Kernel::Avx2 => {
+ #[cfg(target_arch = "x86_64")]
+ unsafe {
+ avx2::execute(&job);
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ let _ = job;
+ }
+ }
+ (y_plane, u_plane, v_plane, uv_plane)
+ }
+
+ fn assert_plane_eq(actual: &Plane, expected: &Plane) {
+ assert_eq!(actual.data.len(), expected.data.len());
+ assert_eq!(actual.stride, expected.stride);
+ assert_eq!(actual.data, expected.data);
+ }
+
+ fn avx2_ready() -> bool {
+ avx2_runtime()
+ }
+
+ #[test]
+ fn avx2_matches_scalar_on_odd_sizes_and_padding() {
+ if !avx2_ready() {
+ return;
+ }
+ let widths = [
+ 1, 2, 3, 7, 8, 9, 15, 16, 17, 31, 32, 33, 47, 63, 64, 65, 127, 128, 160, 1919,
+ ];
+ let heights = [1, 2, 3, 5, 8, 15, 16, 17, 31, 33];
+ let pads = [0, 1, 7, 16, 63];
+ for width in widths {
+ for height in heights {
+ for pad in pads {
+ let rgb = |x: usize, y: usize| {
+ (
+ (x.wrapping_mul(17).wrapping_add(y.wrapping_mul(31))) as u8,
+ (x.wrapping_mul(13)
+ .wrapping_add(y.wrapping_mul(7))
+ .wrapping_add(40)) as u8,
+ (x.wrapping_mul(3)
+ .wrapping_add(y.wrapping_mul(19))
+ .wrapping_add(90)) as u8,
+ )
+ };
+ for (order, nv12) in [
+ (ChannelOrder::Bgr, false),
+ (ChannelOrder::Bgr, true),
+ (ChannelOrder::Rgb, false),
+ (ChannelOrder::Rgb, true),
+ ] {
+ let scalar =
+ run_kernel(Kernel::Scalar, order, nv12, width, height, pad, rgb);
+ let vector = run_kernel(Kernel::Avx2, order, nv12, width, height, pad, rgb);
+ assert_plane_eq(&vector.0, &scalar.0);
+ assert_plane_eq(&vector.1, &scalar.1);
+ assert_plane_eq(&vector.2, &scalar.2);
+ assert_plane_eq(&vector.3, &scalar.3);
+ }
+ }
+ }
+ }
+ }
+
+ fn assert_flat(width: usize, height: usize, pad: usize, nv12: bool) {
+ let rgb = |_: usize, _: usize| (220u8, 40u8, 40u8);
+ for order in [ChannelOrder::Bgr, ChannelOrder::Rgb] {
+ for kernel in [Kernel::Scalar, Kernel::Avx2] {
+ if matches!(kernel, Kernel::Avx2) && !avx2_ready() {
+ continue;
+ }
+ let (y_plane, u_plane, v_plane, uv_plane) =
+ run_kernel(kernel, order, nv12, width, height, pad, rgb);
+ assert_active(&y_plane, height, 83);
+ if nv12 {
+ assert_nv12(&uv_plane, height.div_ceil(2), 110, 207);
+ } else {
+ assert_active(&u_plane, height.div_ceil(2), 110);
+ assert_active(&v_plane, height.div_ceil(2), 207);
+ }
+ }
+ }
+ }
+
+ fn assert_active(plane: &Plane, rows: usize, value: u8) {
+ for row in 0..rows {
+ let start = row * plane.stride;
+ assert!(
+ plane.data[start..start + plane.row_bytes]
+ .iter()
+ .all(|byte| *byte == value),
+ "row {row} expected {value}, got {:?}",
+ &plane.data[start..start + plane.row_bytes.min(8)]
+ );
+ assert!(
+ plane.data[start + plane.row_bytes..start + plane.stride]
+ .iter()
+ .all(|byte| *byte == 0xA5)
+ );
+ }
+ }
+
+ fn assert_nv12(plane: &Plane, rows: usize, cb: u8, cr: u8) {
+ for row in 0..rows {
+ let start = row * plane.stride;
+ for (index, byte) in plane.data[start..start + plane.row_bytes]
+ .iter()
+ .enumerate()
+ {
+ let expected = if index % 2 == 0 { cb } else { cr };
+ assert_eq!(*byte, expected, "uv {row}:{index}");
+ }
+ assert!(
+ plane.data[start + plane.row_bytes..start + plane.stride]
+ .iter()
+ .all(|byte| *byte == 0xA5)
+ );
+ }
+ }
+
+ #[test]
+ fn flat_220_40_40_is_yuv_83_110_207() {
+ for nv12 in [false, true] {
+ for (width, height, pad) in [
+ (1, 1, 0),
+ (2, 2, 0),
+ (3, 3, 5),
+ (7, 5, 1),
+ (8, 2, 0),
+ (8, 8, 16),
+ (15, 9, 3),
+ (31, 17, 7),
+ (32, 16, 0),
+ (33, 17, 13),
+ (64, 18, 63),
+ ] {
+ assert_flat(width, height, pad, nv12);
+ }
+ }
+ }
+
+ #[test]
+ fn two_by_two_box_average_matches_reference() {
+ // (10,20,30), (40,50,60), (70,80,90), (100,110,120)
+ // luma 32, 58, 84, 109 and chroma 133, 123.
+ let sample = |x: usize, y: usize| match (x, y) {
+ (0, 0) => (10, 20, 30),
+ (1, 0) => (40, 50, 60),
+ (0, 1) => (70, 80, 90),
+ (1, 1) => (100, 110, 120),
+ _ => (0, 0, 0),
+ };
+ let (y_plane, u_plane, v_plane, _) =
+ run_kernel(Kernel::Scalar, ChannelOrder::Bgr, false, 2, 2, 4, sample);
+ assert_eq!(y_plane.data[0], 32);
+ assert_eq!(y_plane.data[1], 58);
+ assert_eq!(y_plane.data[y_plane.stride], 84);
+ assert_eq!(y_plane.data[y_plane.stride + 1], 109);
+ assert_eq!(u_plane.data[0], 133);
+ assert_eq!(v_plane.data[0], 123);
+ assert_eq!(y_plane.data[2], 0xA5);
+
+ if !avx2_ready() {
+ return;
+ }
+ let (vector_y, vector_u, vector_v, _) =
+ run_kernel(Kernel::Avx2, ChannelOrder::Rgb, false, 8, 2, 8, sample);
+ assert_eq!(vector_y.data[0], 32);
+ assert_eq!(vector_y.data[1], 58);
+ assert_eq!(vector_y.data[vector_y.stride], 84);
+ assert_eq!(vector_y.data[vector_y.stride + 1], 109);
+ assert_eq!(vector_u.data[0], 133);
+ assert_eq!(vector_v.data[0], 123);
+ let scalar = run_kernel(Kernel::Scalar, ChannelOrder::Rgb, false, 8, 2, 8, sample);
+ assert_plane_eq(&vector_y, &scalar.0);
+ assert_plane_eq(&vector_u, &scalar.1);
+ assert_plane_eq(&vector_v, &scalar.2);
+ }
+
+ #[test]
+ fn color_bars_match_expected_luma() {
+ let colors = [
+ (255, 0, 0, 63u8, 102u8, 240u8),
+ (0, 255, 0, 173, 42, 26),
+ (0, 0, 255, 32, 240, 118),
+ (0, 0, 0, 16, 128, 128),
+ (255, 255, 255, 235, 128, 128),
+ (128, 128, 128, 126, 128, 128),
+ ];
+ let rgb = |x: usize, _: usize| {
+ let (r, g, b, _, _, _) = colors[x / 32];
+ (r, g, b)
+ };
+ for order in [ChannelOrder::Bgr, ChannelOrder::Rgb] {
+ for nv12 in [false, true] {
+ let (y_plane, u_plane, v_plane, uv_plane) =
+ run_kernel(Kernel::Scalar, order, nv12, 192, 32, 8, rgb);
+ check_bars(&y_plane, &u_plane, &v_plane, &uv_plane, nv12, &colors);
+ if avx2_ready() {
+ let (y_plane, u_plane, v_plane, uv_plane) =
+ run_kernel(Kernel::Avx2, order, nv12, 192, 32, 8, rgb);
+ check_bars(&y_plane, &u_plane, &v_plane, &uv_plane, nv12, &colors);
+ }
+ }
+ }
+ }
+
+ fn check_bars(
+ y_plane: &Plane,
+ u_plane: &Plane,
+ v_plane: &Plane,
+ uv_plane: &Plane,
+ nv12: bool,
+ colors: &[(u8, u8, u8, u8, u8, u8)],
+ ) {
+ for (index, (_, _, _, ey, eu, ev)) in colors.iter().enumerate() {
+ let x = index * 32 + 16;
+ let actual = y_plane.data[16 * y_plane.stride + x];
+ assert!(
+ actual.abs_diff(*ey) <= 1,
+ "luma bar {index}: {actual} vs {ey}"
+ );
+ let (cb, cr) = if nv12 {
+ let offset = 8 * uv_plane.stride + x;
+ (uv_plane.data[offset], uv_plane.data[offset + 1])
+ } else {
+ let cx = x / 2;
+ (
+ u_plane.data[8 * u_plane.stride + cx],
+ v_plane.data[8 * v_plane.stride + cx],
+ )
+ };
+ assert!(cb.abs_diff(*eu) <= 1, "cb bar {index}: {cb} vs {eu}");
+ assert!(cr.abs_diff(*ev) <= 1, "cr bar {index}: {cr} vs {ev}");
+ }
+ }
+
+ #[test]
+ fn swscale_fast_bilinear_stays_within_tolerance() {
+ ffmpeg::init().unwrap();
+ let gradient = |x: usize, y: usize, width: usize, height: usize| {
+ let width = width.max(1);
+ let height = height.max(1);
+ (
+ (x * 255 / width) as u8,
+ (y * 255 / height) as u8,
+ ((x + y) * 255 / (width + height)) as u8,
+ )
+ };
+ for (width, height) in [(32u32, 32u32), (64, 48), (320, 180)] {
+ compare_swscale(Pixel::YUV420P, width, height, gradient);
+ }
+ compare_swscale(Pixel::YUV420P, 48, 24, |_, _, _, _| (220, 40, 40));
+ compare_swscale(Pixel::NV12, 320, 180, gradient);
+ }
+
+ fn compare_swscale(
+ format: Pixel,
+ width: u32,
+ height: u32,
+ rgb: impl Fn(usize, usize, usize, usize) -> (u8, u8, u8),
+ ) {
+ let width_px = width as usize;
+ let height_px = height as usize;
+ let mut src = frame::Video::new(Pixel::BGRZ, width, height);
+ if !avx2_ready() {
+ let mut dst = frame::Video::new(format, width, height);
+ assert!(!try_convert(&src, &mut dst), "{format:?} {width}x{height}");
+ return;
+ }
+ let stride = src.stride(0);
+ for y in 0..height_px {
+ for x in 0..width_px {
+ let (r, g, b) = rgb(x, y, width_px, height_px);
+ let offset = y * stride + x * 4;
+ src.data_mut(0)[offset..offset + 4].copy_from_slice(&[b, g, r, 255]);
+ }
+ }
+ let mut scaler = bt709_scaler(format, width, height);
+ let mut sws = frame::Video::new(format, width, height);
+ scaler.run(&src, &mut sws).unwrap();
+ let mut ours = frame::Video::new(format, width, height);
+ assert!(try_convert(&src, &mut ours), "{format:?} {width}x{height}");
+ let y_stats = plane_delta(&ours, &sws, 0, width as usize, height as usize);
+ let chroma_h = (height as usize).div_ceil(2);
+ assert!(
+ y_stats.max_diff <= 2 && y_stats.mean < 1.0,
+ "{format:?} {width}x{height} luma max {} mean {:.3} at {},{}",
+ y_stats.max_diff,
+ y_stats.mean,
+ y_stats.col,
+ y_stats.row
+ );
+ if format == Pixel::NV12 {
+ let uv_stats = plane_delta(&ours, &sws, 1, width as usize, chroma_h);
+ assert!(
+ uv_stats.max_diff <= 3 && uv_stats.mean < 1.5,
+ "{format:?} {width}x{height} chroma max {} mean {:.3} at {},{}",
+ uv_stats.max_diff,
+ uv_stats.mean,
+ uv_stats.col,
+ uv_stats.row
+ );
+ } else {
+ let chroma_w = (width as usize).div_ceil(2);
+ let u_stats = plane_delta(&ours, &sws, 1, chroma_w, chroma_h);
+ let v_stats = plane_delta(&ours, &sws, 2, chroma_w, chroma_h);
+ assert!(
+ u_stats.max_diff <= 3
+ && v_stats.max_diff <= 3
+ && u_stats.mean < 1.5
+ && v_stats.mean < 1.5,
+ "{format:?} {width}x{height} chroma u max {} mean {:.3} at {},{} v max {} mean {:.3} at {},{}",
+ u_stats.max_diff,
+ u_stats.mean,
+ u_stats.col,
+ u_stats.row,
+ v_stats.max_diff,
+ v_stats.mean,
+ v_stats.col,
+ v_stats.row
+ );
+ }
+ }
+
+ struct PlaneStats {
+ max_diff: u8,
+ mean: f64,
+ col: usize,
+ row: usize,
+ }
+
+ fn plane_delta(
+ actual: &frame::Video,
+ expected: &frame::Video,
+ plane: usize,
+ width: usize,
+ height: usize,
+ ) -> PlaneStats {
+ let mut max_diff = 0u8;
+ let mut sum = 0u64;
+ let mut count = 0u64;
+ let mut col = 0;
+ let mut row = 0;
+ for y in 0..height {
+ let actual_row = &actual.data(plane)[y * actual.stride(plane)..];
+ let expected_row = &expected.data(plane)[y * expected.stride(plane)..];
+ for x in 0..width {
+ let diff = actual_row[x].abs_diff(expected_row[x]);
+ sum += u64::from(diff);
+ count += 1;
+ if diff > max_diff {
+ max_diff = diff;
+ col = x;
+ row = y;
+ }
+ }
+ }
+ PlaneStats {
+ max_diff,
+ mean: sum as f64 / count as f64,
+ col,
+ row,
+ }
+ }
+
+ #[test]
+ fn odd_sizes_convert_or_scale_matches_swscale() {
+ ffmpeg::init().unwrap();
+ for (width, height) in [(1u32, 2), (2, 1), (65, 64), (64, 63), (65, 63), (31, 17)] {
+ let mut src = frame::Video::new(Pixel::BGRZ, width, height);
+ let stride = src.stride(0);
+ for y in 0..height as usize {
+ for x in 0..width as usize {
+ let offset = y * stride + x * 4;
+ src.data_mut(0)[offset..offset + 4].copy_from_slice(&[
+ (x.wrapping_mul(3).wrapping_add(y)) as u8,
+ (y.wrapping_mul(5).wrapping_add(40)) as u8,
+ (x.wrapping_mul(7).wrapping_add(90)) as u8,
+ 255,
+ ]);
+ }
+ }
+ let mut probe = frame::Video::new(Pixel::YUV420P, width, height);
+ assert!(!try_convert(&src, &mut probe), "{width}x{height} yuv420p");
+ let mut nv12 = frame::Video::new(Pixel::NV12, width, height);
+ assert!(!try_convert(&src, &mut nv12), "{width}x{height} nv12");
+
+ let mut via_scaler = bt709_scaler(Pixel::YUV420P, width, height);
+ let mut via = frame::Video::new(Pixel::YUV420P, width, height);
+ convert_or_scale(&src, &mut via, &mut via_scaler).unwrap();
+ let mut direct_scaler = bt709_scaler(Pixel::YUV420P, width, height);
+ let mut direct = frame::Video::new(Pixel::YUV420P, width, height);
+ direct_scaler.run(&src, &mut direct).unwrap();
+ assert_frames_eq(&via, &direct, width, height);
+ }
+ }
+
+ fn bt709_scaler(format: Pixel, width: u32, height: u32) -> ffmpeg::software::scaling::Context {
+ let mut scaler = ffmpeg::software::scaling::Context::get(
+ Pixel::BGRZ,
+ width,
+ height,
+ format,
+ width,
+ height,
+ ffmpeg::software::scaling::flag::Flags::FAST_BILINEAR,
+ )
+ .unwrap();
+ super::super::colour::use_bt709_for_rgb_input(&mut scaler, Pixel::BGRZ, format).unwrap();
+ scaler
+ }
+
+ fn assert_frames_eq(actual: &frame::Video, expected: &frame::Video, width: u32, height: u32) {
+ for plane in 0..expected.planes() {
+ let row_bytes = expected.plane_width(plane) as usize;
+ for row in 0..expected.plane_height(plane) as usize {
+ let got = &actual.data(plane)[row * actual.stride(plane)..][..row_bytes];
+ let want = &expected.data(plane)[row * expected.stride(plane)..][..row_bytes];
+ assert_eq!(got, want, "{width}x{height} plane {plane} row {row}");
+ }
+ }
+ }
+
+ #[test]
+ fn other_formats_and_scaling_stay_on_swscale() {
+ ffmpeg::init().unwrap();
+ let src = frame::Video::new(Pixel::BGRZ, 32, 16);
+ let mut scaled = frame::Video::new(Pixel::YUV420P, 16, 16);
+ assert!(!try_convert(&src, &mut scaled));
+ let rgb24 = frame::Video::new(Pixel::RGB24, 32, 16);
+ let mut yuv = frame::Video::new(Pixel::YUV420P, 32, 16);
+ assert!(!try_convert(&rgb24, &mut yuv));
+ let yuv_src = frame::Video::new(Pixel::YUV420P, 32, 16);
+ assert!(!try_convert(&yuv_src, &mut yuv));
+
+ let mut scaler = ffmpeg::software::scaling::Context::get(
+ Pixel::BGRA,
+ 32,
+ 16,
+ Pixel::YUV420P,
+ 16,
+ 8,
+ ffmpeg::software::scaling::flag::Flags::BICUBIC,
+ )
+ .unwrap();
+ super::super::colour::use_bt709_for_rgb_input(&mut scaler, Pixel::BGRA, Pixel::YUV420P)
+ .unwrap();
+ let mut bgra = frame::Video::new(Pixel::BGRA, 32, 16);
+ for (index, byte) in bgra.data_mut(0).iter_mut().enumerate() {
+ *byte = (index % 251) as u8;
+ }
+ let mut via_helper = frame::Video::empty();
+ let mut direct = frame::Video::empty();
+ convert_or_scale(&bgra, &mut via_helper, &mut scaler).unwrap();
+ scaler.run(&bgra, &mut direct).unwrap();
+ for plane in 0..direct.planes() {
+ let width = direct.plane_width(plane) as usize;
+ for row in 0..direct.plane_height(plane) as usize {
+ let got = &via_helper.data(plane)[row * via_helper.stride(plane)..][..width];
+ let expect = &direct.data(plane)[row * direct.stride(plane)..][..width];
+ assert_eq!(got, expect);
+ }
+ }
+ }
+
+ #[test]
+ #[ignore = "release timing bench, run with --ignored --nocapture"]
+ fn bench_bgrz_to_yuv420p() {
+ ffmpeg::init().unwrap();
+ let model = std::fs::read_to_string("/proc/cpuinfo")
+ .ok()
+ .and_then(|text| {
+ text.lines()
+ .find(|line| line.starts_with("model name"))
+ .and_then(|line| line.split(':').nth(1))
+ .map(|value| value.trim().to_string())
+ })
+ .unwrap_or_else(|| "unknown".into());
+ println!("cpu: {model}");
+ println!("avx2: {}", avx2_ready());
+ println!("warmup excluded from n; median is the middle sorted sample");
+ for (width, height) in [(1280, 720), (1920, 1080), (3840, 2160)] {
+ bench_conversion(width, height);
+ }
+ for (width, height) in [(1280, 720), (1920, 1080), (3840, 2160)] {
+ bench_queue_frame(width, height);
+ }
+ }
+
+ fn bench_conversion(width: u32, height: u32) {
+ let mut src = frame::Video::new(Pixel::BGRZ, width, height);
+ fill_gradient(&mut src);
+ let mut kernel_dst = frame::Video::new(Pixel::YUV420P, width, height);
+ let mut sws_dst = frame::Video::new(Pixel::YUV420P, width, height);
+ let mut scaler = ffmpeg::software::scaling::Context::get(
+ Pixel::BGRZ,
+ width,
+ height,
+ Pixel::YUV420P,
+ width,
+ height,
+ ffmpeg::software::scaling::flag::Flags::FAST_BILINEAR,
+ )
+ .unwrap();
+ super::super::colour::use_bt709_for_rgb_input(&mut scaler, Pixel::BGRZ, Pixel::YUV420P)
+ .unwrap();
+ let sws = time_median(5, 31, || {
+ scaler.run(&src, &mut sws_dst).unwrap();
+ std::hint::black_box(sws_dst.data(0).as_ptr());
+ });
+ let kernel = time_median(5, 31, || {
+ assert!(try_convert(&src, &mut kernel_dst));
+ std::hint::black_box(kernel_dst.data(0).as_ptr());
+ std::hint::black_box(kernel_dst.data(1).as_ptr());
+ });
+ println!(
+ "{width}x{height} swscale_fast_bilinear_bt709 median {:.3} ms n=31",
+ sws / 1_000_000.0
+ );
+ println!(
+ "{width}x{height} bt709_kernel median {:.3} ms n=31 speedup {:.2}x",
+ kernel / 1_000_000.0,
+ sws / kernel
+ );
+ }
+
+ fn bench_queue_frame(width: u32, height: u32) {
+ let info = cap_media_info::VideoInfo::from_raw_ffmpeg(Pixel::BGRZ, width, height, 30);
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("bench.mp4");
+ let mut output = ffmpeg::format::output_as(&path, "mp4").unwrap();
+ let mut encoder = super::super::h264::H264Encoder::builder(info)
+ .with_preset(super::super::h264::H264Preset::Ultrafast)
+ .with_encoder_priority_override(&["libx264"])
+ .build(&mut output)
+ .unwrap();
+ let mut sink = crate::PacketQueue::new(ffmpeg::Rational(1, 90_000));
+ let mut frames = Vec::new();
+ for _ in 0..11 {
+ let mut frame = frame::Video::new(Pixel::BGRZ, width, height);
+ fill_gradient(&mut frame);
+ frames.push(frame);
+ }
+ for index in 0..2 {
+ encoder
+ .queue_frame_to(
+ frames.pop().unwrap(),
+ std::time::Duration::from_millis(index * 33),
+ &mut sink,
+ )
+ .unwrap();
+ sink.take();
+ }
+ let mut samples = Vec::with_capacity(9);
+ for index in 0..9 {
+ let frame = frames.pop().unwrap();
+ let started = std::time::Instant::now();
+ encoder
+ .queue_frame_to(
+ frame,
+ std::time::Duration::from_millis((index + 2) * 33),
+ &mut sink,
+ )
+ .unwrap();
+ samples.push(started.elapsed().as_nanos());
+ sink.take();
+ }
+ samples.sort_unstable();
+ println!(
+ "{width}x{height} queue_frame_to_libx264_ultrafast median {:.3} ms n=9",
+ samples[samples.len() / 2] as f64 / 1_000_000.0
+ );
+ }
+
+ fn fill_gradient(frame: &mut frame::Video) {
+ let width = frame.width() as usize;
+ let height = frame.height() as usize;
+ let stride = frame.stride(0);
+ let data = frame.data_mut(0);
+ for y in 0..height {
+ for x in 0..width {
+ let offset = y * stride + x * 4;
+ data[offset] = (x * 255 / width.max(1)) as u8;
+ data[offset + 1] = (y * 255 / height.max(1)) as u8;
+ data[offset + 2] = ((x + y) % 256) as u8;
+ data[offset + 3] = 255;
+ }
+ }
+ }
+
+ fn time_median(warmup: usize, runs: usize, mut body: impl FnMut()) -> f64 {
+ for _ in 0..warmup {
+ body();
+ }
+ let mut samples = Vec::with_capacity(runs);
+ for _ in 0..runs {
+ let started = std::time::Instant::now();
+ body();
+ samples.push(started.elapsed().as_nanos());
+ }
+ samples.sort_unstable();
+ samples[samples.len() / 2] as f64
+ }
+}