Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf: convert packed camera yuv in the h264 encoder

Kyzer Phneh8734f17bfacb
diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs
index 0042b4610..90645af99 100644
--- a/crates/enc-ffmpeg/src/video/h264.rs
+++ b/crates/enc-ffmpeg/src/video/h264.rs
@@ -915,8 +915,7 @@ impl H264Encoder {
let frame_to_send = if let Some(converter) = &mut self.converter {
let pts = frame.pts();
let converted = self.converted_frame_pool.as_mut().unwrap();
- converter
- .run(&frame, converted)
+ super::packed_yuv::convert_or_scale(&frame, converted, converter)
.map_err(QueueFrameError::Converter)?;
converted.set_pts(pts);
converted as &frame::Video
@@ -945,8 +944,7 @@ impl H264Encoder {
let converted = converted_frame.get_or_insert_with(|| {
frame::Video::new(self.output_format, self.output_width, self.output_height)
});
- converter
- .run(frame, converted)
+ super::packed_yuv::convert_or_scale(frame, converted, converter)
.map_err(QueueFrameError::Converter)?;
converted.set_pts(pts);
converted as &frame::Video
diff --git a/crates/enc-ffmpeg/src/video/h264_packet.rs b/crates/enc-ffmpeg/src/video/h264_packet.rs
index fb7596f7c..e344a09e5 100644
--- a/crates/enc-ffmpeg/src/video/h264_packet.rs
+++ b/crates/enc-ffmpeg/src/video/h264_packet.rs
@@ -140,8 +140,7 @@ impl H264PacketEncoder {
let converted = self.converted_frame_pool.as_mut().expect(
"converted_frame_pool present whenever converter is Some (invariant preserved by from_opened)",
);
- converter
- .run(&frame, converted)
+ super::packed_yuv::convert_or_scale(&frame, converted, converter)
.map_err(EncodePacketError::Converter)?;
converted.set_pts(pts);
converted as &frame::Video
@@ -173,8 +172,7 @@ impl H264PacketEncoder {
let converted = converted_frame.get_or_insert_with(|| {
frame::Video::new(self.output_format, self.output_width, self.output_height)
});
- converter
- .run(frame, converted)
+ super::packed_yuv::convert_or_scale(frame, converted, converter)
.map_err(EncodePacketError::Converter)?;
converted.set_pts(pts);
converted as &frame::Video
diff --git a/crates/enc-ffmpeg/src/video/mod.rs b/crates/enc-ffmpeg/src/video/mod.rs
index 104261fc0..839c61e68 100644
--- a/crates/enc-ffmpeg/src/video/mod.rs
+++ b/crates/enc-ffmpeg/src/video/mod.rs
@@ -1,4 +1,5 @@
pub mod colour;
+mod packed_yuv;
pub mod h264;
pub mod h264_packet;
pub mod hevc;
diff --git a/crates/enc-ffmpeg/src/video/packed_yuv.rs b/crates/enc-ffmpeg/src/video/packed_yuv.rs
new file mode 100644
index 000000000..71ab40425
--- /dev/null
+++ b/crates/enc-ffmpeg/src/video/packed_yuv.rs
@@ -0,0 +1,903 @@
+use ffmpeg::format::Pixel;
+use ffmpeg::frame;
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum Packed422 {
+ Uyvy,
+ Yuyv,
+}
+
+struct Job {
+ src: *const u8,
+ src_stride: usize,
+ width: usize,
+ height: usize,
+ kind: Packed422,
+ y: *mut u8,
+ y_stride: usize,
+ u: *mut u8,
+ u_stride: usize,
+ v: *mut u8,
+ v_stride: usize,
+ uv: *mut u8,
+ uv_stride: usize,
+ nv12: bool,
+}
+
+fn positive_linesize(frame: &frame::Video, plane: usize) -> Option<usize> {
+ if plane >= frame.planes() {
+ return None;
+ }
+ let raw = unsafe { (*frame.as_ptr()).linesize[plane] };
+ if raw <= 0 { None } else { Some(raw as usize) }
+}
+
+fn covers(stride: usize, row_bytes: usize, height: usize, len: usize) -> bool {
+ if row_bytes == 0 || height == 0 {
+ return true;
+ }
+ if stride < row_bytes {
+ return false;
+ }
+ stride
+ .checked_mul(height - 1)
+ .and_then(|last| last.checked_add(row_bytes))
+ .is_some_and(|need| need <= len)
+}
+
+/// Same-size UYVY422 or YUYV422 into `dst` when both dimensions are even,
+/// AVX2 is available at runtime, and `dst` is YUV420P or NV12. Any other
+/// input returns false so the caller keeps swscale.
+pub(crate) fn try_convert(src: &frame::Video, dst: &mut frame::Video) -> bool {
+ if src.width() != dst.width() || src.height() != dst.height() {
+ return false;
+ }
+ let kind = match src.format() {
+ Pixel::UYVY422 => Packed422::Uyvy,
+ Pixel::YUYV422 => Packed422::Yuyv,
+ _ => return false,
+ };
+ let nv12 = match dst.format() {
+ Pixel::YUV420P => false,
+ Pixel::NV12 => true,
+ _ => return false,
+ };
+ let width = src.width() as usize;
+ let height = src.height() as usize;
+ if !width.is_multiple_of(2) || !height.is_multiple_of(2) {
+ return false;
+ }
+ if width == 0 || height == 0 {
+ return true;
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ {
+ let _ = (kind, nv12);
+ return false;
+ }
+ #[cfg(target_arch = "x86_64")]
+ {
+ if !std::arch::is_x86_feature_detected!("avx2") {
+ return false;
+ }
+ let Some(src_stride) = positive_linesize(src, 0) else {
+ return false;
+ };
+ let Some(y_stride) = positive_linesize(dst, 0) else {
+ return false;
+ };
+ if !covers(src_stride, width * 2, height, src.data(0).len())
+ || !covers(y_stride, width, height, dst.data(0).len())
+ {
+ return false;
+ }
+ let chroma_w = width / 2;
+ let chroma_h = height / 2;
+ let (u_stride, v_stride, uv_stride) = if nv12 {
+ let Some(uv_stride) = positive_linesize(dst, 1) else {
+ return false;
+ };
+ if !covers(uv_stride, chroma_w * 2, chroma_h, dst.data(1).len()) {
+ return false;
+ }
+ (0, 0, uv_stride)
+ } else {
+ let Some(u_stride) = positive_linesize(dst, 1) else {
+ return false;
+ };
+ let Some(v_stride) = positive_linesize(dst, 2) else {
+ return false;
+ };
+ if !covers(u_stride, chroma_w, chroma_h, dst.data(1).len())
+ || !covers(v_stride, chroma_w, chroma_h, dst.data(2).len())
+ {
+ return false;
+ }
+ (u_stride, v_stride, 0)
+ };
+
+ let src_ptr = src.data(0).as_ptr();
+ let y_ptr = dst.data_mut(0).as_mut_ptr();
+ let (u_ptr, v_ptr, uv_ptr) = if nv12 {
+ (
+ std::ptr::null_mut(),
+ std::ptr::null_mut(),
+ dst.data_mut(1).as_mut_ptr(),
+ )
+ } else {
+ let u_ptr = dst.data_mut(1).as_mut_ptr();
+ let v_ptr = dst.data_mut(2).as_mut_ptr();
+ (u_ptr, v_ptr, std::ptr::null_mut())
+ };
+
+ let job = Job {
+ src: src_ptr,
+ src_stride,
+ width,
+ height,
+ kind,
+ y: y_ptr,
+ y_stride,
+ u: u_ptr,
+ u_stride,
+ v: v_ptr,
+ v_stride,
+ uv: uv_ptr,
+ uv_stride,
+ nv12,
+ };
+ // SAFETY: AVX2 was detected, and `covers` checked every active plane.
+ unsafe { avx2::execute(&job) };
+ true
+ }
+}
+
+pub(crate) fn convert_or_scale(
+ src: &frame::Video,
+ dst: &mut frame::Video,
+ scaler: &mut ffmpeg::software::scaling::Context,
+) -> Result<(), ffmpeg::Error> {
+ if try_convert(src, dst) {
+ Ok(())
+ } else {
+ scaler.run(src, dst)
+ }
+}
+
+#[cfg(target_arch = "x86_64")]
+mod avx2 {
+ use std::arch::x86_64::*;
+
+ use super::{Job, Packed422};
+
+ #[target_feature(enable = "avx2")]
+ pub(super) unsafe fn execute(job: &Job) {
+ unsafe {
+ let masks = match job.kind {
+ Packed422::Uyvy => Masks {
+ y: uyvy_y_mask(),
+ u: uyvy_u_mask(),
+ v: uyvy_v_mask(),
+ },
+ Packed422::Yuyv => Masks {
+ y: yuyv_y_mask(),
+ u: yuyv_u_mask(),
+ v: yuyv_v_mask(),
+ },
+ };
+ if job.nv12 {
+ convert_frame::<true>(job, &masks);
+ } else {
+ convert_frame::<false>(job, &masks);
+ }
+ }
+ }
+
+ struct Masks {
+ y: __m256i,
+ u: __m256i,
+ v: __m256i,
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn convert_frame<const NV12: bool>(job: &Job, masks: &Masks) {
+ unsafe {
+ let mut top = 0;
+ while top < job.height {
+ let row = Row16 {
+ src0: job.src.add(top * job.src_stride),
+ src1: job.src.add((top + 1) * job.src_stride),
+ y0: job.y.add(top * job.y_stride),
+ y1: job.y.add((top + 1) * job.y_stride),
+ top,
+ };
+ let mut x = 0;
+ while x + 32 <= job.width {
+ convert16::<NV12>(job, masks, &row, x);
+ convert16::<NV12>(job, masks, &row, x + 16);
+ x += 32;
+ }
+ while x + 16 <= job.width {
+ convert16::<NV12>(job, masks, &row, x);
+ x += 16;
+ }
+ if x < job.width {
+ scalar_tail::<NV12>(job, top, x);
+ }
+ top += 2;
+ }
+ }
+ }
+
+ struct Row16 {
+ src0: *const u8,
+ src1: *const u8,
+ y0: *mut u8,
+ y1: *mut u8,
+ top: usize,
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn convert16<const NV12: bool>(job: &Job, masks: &Masks, row: &Row16, x: usize) {
+ unsafe {
+ let packed0 = _mm256_loadu_si256(row.src0.add(x * 2).cast());
+ store_luma(packed0, masks.y, row.y0.add(x));
+ let u0 = pack_chroma8(packed0, masks.u);
+ let v0 = pack_chroma8(packed0, masks.v);
+ let packed1 = _mm256_loadu_si256(row.src1.add(x * 2).cast());
+ store_luma(packed1, masks.y, row.y1.add(x));
+ let u1 = pack_chroma8(packed1, masks.u);
+ let v1 = pack_chroma8(packed1, masks.v);
+ let u = _mm_avg_epu8(u0, u1);
+ let v = _mm_avg_epu8(v0, v1);
+ store_chroma8::<NV12>(job, row.top / 2, x / 2, u, v);
+ }
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn store_luma(packed: __m256i, mask: __m256i, dst: *mut u8) {
+ unsafe {
+ let shuffled = _mm256_shuffle_epi8(packed, mask);
+ let lo = _mm256_castsi256_si128(shuffled);
+ let hi = _mm256_extracti128_si256(shuffled, 1);
+ let y = _mm_unpacklo_epi64(lo, hi);
+ _mm_storeu_si128(dst.cast(), y);
+ }
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn pack_chroma8(packed: __m256i, mask: __m256i) -> __m128i {
+ let shuffled = _mm256_shuffle_epi8(packed, mask);
+ let lo = _mm256_castsi256_si128(shuffled);
+ let hi = _mm256_extracti128_si256(shuffled, 1);
+ _mm_unpacklo_epi32(lo, hi)
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn store_chroma8<const NV12: bool>(
+ job: &Job,
+ row: usize,
+ x: usize,
+ u: __m128i,
+ v: __m128i,
+ ) {
+ unsafe {
+ if NV12 {
+ let interleaved = _mm_unpacklo_epi8(u, v);
+ _mm_storeu_si128(job.uv.add(row * job.uv_stride + x * 2).cast(), interleaved);
+ } else {
+ _mm_storel_epi64(job.u.add(row * job.u_stride + x).cast(), u);
+ _mm_storel_epi64(job.v.add(row * job.v_stride + x).cast(), v);
+ }
+ }
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn uyvy_y_mask() -> __m256i {
+ _mm256_setr_epi8(
+ 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, 1, 3, 5, 7, 9, 11, 13, 15,
+ -1, -1, -1, -1, -1, -1, -1, -1,
+ )
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn uyvy_u_mask() -> __m256i {
+ _mm256_setr_epi8(
+ 0, 4, 8, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 4, 8, 12, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ )
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn uyvy_v_mask() -> __m256i {
+ _mm256_setr_epi8(
+ 2, 6, 10, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 6, 10, 14, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ )
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn yuyv_y_mask() -> __m256i {
+ _mm256_setr_epi8(
+ 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, 0, 2, 4, 6, 8, 10, 12, 14,
+ -1, -1, -1, -1, -1, -1, -1, -1,
+ )
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn yuyv_u_mask() -> __m256i {
+ _mm256_setr_epi8(
+ 1, 5, 9, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 5, 9, 13, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ )
+ }
+
+ #[target_feature(enable = "avx2")]
+ unsafe fn yuyv_v_mask() -> __m256i {
+ _mm256_setr_epi8(
+ 3, 7, 11, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 3, 7, 11, 15, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ )
+ }
+
+ unsafe fn scalar_tail<const NV12: bool>(job: &Job, top: usize, x_start: usize) {
+ let mut x = x_start;
+ while x < job.width {
+ unsafe {
+ let (y0, y1, u0, v0) = load_macro(job, top, x);
+ let (y2, y3, u1, v1) = load_macro(job, top + 1, x);
+ *job.y.add(top * job.y_stride + x) = y0;
+ *job.y.add(top * job.y_stride + x + 1) = y1;
+ *job.y.add((top + 1) * job.y_stride + x) = y2;
+ *job.y.add((top + 1) * job.y_stride + x + 1) = y3;
+ let cx = x / 2;
+ let cy = top / 2;
+ let u = avg(u0, u1);
+ let v = avg(v0, v1);
+ if NV12 {
+ let dst = job.uv.add(cy * job.uv_stride + cx * 2);
+ *dst = u;
+ *dst.add(1) = v;
+ } else {
+ *job.u.add(cy * job.u_stride + cx) = u;
+ *job.v.add(cy * job.v_stride + cx) = v;
+ }
+ }
+ x += 2;
+ }
+ }
+
+ unsafe fn load_macro(job: &Job, row: usize, x: usize) -> (u8, u8, u8, u8) {
+ unsafe {
+ let pixel = job.src.add(row * job.src_stride + x * 2);
+ let b0 = *pixel;
+ let b1 = *pixel.add(1);
+ let b2 = *pixel.add(2);
+ let b3 = *pixel.add(3);
+ match job.kind {
+ Packed422::Uyvy => (b1, b3, b0, b2),
+ Packed422::Yuyv => (b0, b2, b1, b3),
+ }
+ }
+ }
+
+ fn avg(a: u8, b: u8) -> u8 {
+ ((u16::from(a) + u16::from(b) + 1) >> 1) as u8
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use ffmpeg::software::scaling::{self, flag::Flags};
+
+ fn avx2_ready() -> bool {
+ #[cfg(target_arch = "x86_64")]
+ {
+ std::arch::is_x86_feature_detected!("avx2")
+ }
+ #[cfg(not(target_arch = "x86_64"))]
+ {
+ false
+ }
+ }
+
+ fn fast_scaler(src: Pixel, dst: Pixel, width: u32, height: u32) -> scaling::Context {
+ scaling::Context::get(src, width, height, dst, width, height, Flags::FAST_BILINEAR).unwrap()
+ }
+
+ fn zero_frame(frame: &mut frame::Video) {
+ for plane in 0..frame.planes() {
+ frame.data_mut(plane).fill(0);
+ }
+ }
+
+ fn fill_packed(frame: &mut frame::Video) {
+ let uyvy = frame.format() == Pixel::UYVY422;
+ let width = frame.width() as usize;
+ let height = frame.height() as usize;
+ let stride = frame.stride(0);
+ let data = frame.data_mut(0);
+ for y in 0..height {
+ let mut x = 0;
+ while x + 1 < width {
+ let y0 = ((x * 17 + y * 3) % 256) as u8;
+ let y1 = ((x * 13 + y * 11 + 40) % 256) as u8;
+ let u = ((x * 5 + y * 7 + 16) % 256) as u8;
+ let v = ((x * 9 + y * 2 + 80) % 256) as u8;
+ let off = y * stride + x * 2;
+ if uyvy {
+ data[off..off + 4].copy_from_slice(&[u, y0, v, y1]);
+ } else {
+ data[off..off + 4].copy_from_slice(&[y0, u, y1, v]);
+ }
+ x += 2;
+ }
+ }
+ }
+
+ fn reference(src: &frame::Video) -> (Vec<u8>, Vec<u8>, Vec<u8>) {
+ let width = src.width() as usize;
+ let height = src.height() as usize;
+ let stride = src.stride(0);
+ let data = src.data(0);
+ let uyvy = src.format() == Pixel::UYVY422;
+ let cw = width / 2;
+ let ch = height / 2;
+ let mut y_plane = vec![0u8; width * height];
+ let mut u_plane = vec![0u8; cw * ch];
+ let mut v_plane = vec![0u8; cw * ch];
+ let mut row = 0;
+ while row < height {
+ let mut x = 0;
+ while x < width {
+ let (y0, y1, u0, v0) = macro_at(data, stride, row, x, uyvy);
+ let (y2, y3, u1, v1) = macro_at(data, stride, row + 1, x, uyvy);
+ y_plane[row * width + x] = y0;
+ y_plane[row * width + x + 1] = y1;
+ y_plane[(row + 1) * width + x] = y2;
+ y_plane[(row + 1) * width + x + 1] = y3;
+ let cx = x / 2;
+ let cy = row / 2;
+ u_plane[cy * cw + cx] = ((u16::from(u0) + u16::from(u1) + 1) >> 1) as u8;
+ v_plane[cy * cw + cx] = ((u16::from(v0) + u16::from(v1) + 1) >> 1) as u8;
+ x += 2;
+ }
+ row += 2;
+ }
+ (y_plane, u_plane, v_plane)
+ }
+
+ fn macro_at(data: &[u8], stride: usize, row: usize, x: usize, uyvy: bool) -> (u8, u8, u8, u8) {
+ let off = row * stride + x * 2;
+ let b0 = data[off];
+ let b1 = data[off + 1];
+ let b2 = data[off + 2];
+ let b3 = data[off + 3];
+ if uyvy {
+ (b1, b3, b0, b2)
+ } else {
+ (b0, b2, b1, b3)
+ }
+ }
+
+ fn assert_matches_reference(actual: &frame::Video, y: &[u8], u: &[u8], v: &[u8]) {
+ let width = actual.width() as usize;
+ let height = actual.height() as usize;
+ let cw = width / 2;
+ let ch = height / 2;
+ for row in 0..height {
+ let got = &actual.data(0)[row * actual.stride(0)..][..width];
+ assert_eq!(got, &y[row * width..][..width], "y row {row}");
+ }
+ if actual.format() == Pixel::NV12 {
+ for row in 0..ch {
+ let got = &actual.data(1)[row * actual.stride(1)..][..cw * 2];
+ for cx in 0..cw {
+ assert_eq!(got[cx * 2], u[row * cw + cx], "nv12 u {cx},{row}");
+ assert_eq!(got[cx * 2 + 1], v[row * cw + cx], "nv12 v {cx},{row}");
+ }
+ }
+ } else {
+ for row in 0..ch {
+ let got_u = &actual.data(1)[row * actual.stride(1)..][..cw];
+ let got_v = &actual.data(2)[row * actual.stride(2)..][..cw];
+ assert_eq!(got_u, &u[row * cw..][..cw], "u row {row}");
+ assert_eq!(got_v, &v[row * cw..][..cw], "v row {row}");
+ }
+ }
+ }
+
+ struct PlaneStats {
+ max_diff: u8,
+ mean: f64,
+ col: usize,
+ row: usize,
+ }
+
+ fn plane_delta(
+ actual: &frame::Video,
+ expected: &frame::Video,
+ plane: usize,
+ width: usize,
+ height: usize,
+ ) -> PlaneStats {
+ let mut max_diff = 0u8;
+ let mut sum = 0u64;
+ let mut count = 0u64;
+ let mut col = 0;
+ let mut row_at = 0;
+ for row in 0..height {
+ let actual_row = &actual.data(plane)[row * actual.stride(plane)..];
+ let expected_row = &expected.data(plane)[row * expected.stride(plane)..];
+ for x in 0..width {
+ let diff = actual_row[x].abs_diff(expected_row[x]);
+ sum += u64::from(diff);
+ count += 1;
+ if diff > max_diff {
+ max_diff = diff;
+ col = x;
+ row_at = row;
+ }
+ }
+ }
+ PlaneStats {
+ max_diff,
+ mean: sum as f64 / count as f64,
+ col,
+ row: row_at,
+ }
+ }
+
+ fn assert_within_swscale(actual: &frame::Video, expected: &frame::Video) {
+ let width = actual.width() as usize;
+ let height = actual.height() as usize;
+ let y_stats = plane_delta(actual, expected, 0, width, height);
+ let y_limit = if actual.format() == Pixel::NV12 { 4 } else { 2 };
+ assert!(
+ y_stats.max_diff <= y_limit && y_stats.mean < 1.0,
+ "{width}x{height} luma max {} mean {:.3} at {},{}",
+ y_stats.max_diff,
+ y_stats.mean,
+ y_stats.col,
+ y_stats.row
+ );
+ let cw = width / 2;
+ let ch = height / 2;
+ if actual.format() == Pixel::NV12 {
+ let uv_stats = plane_delta(actual, expected, 1, cw * 2, ch);
+ assert!(
+ uv_stats.max_diff <= 3 && uv_stats.mean < 1.5,
+ "{width}x{height} nv12 max {} mean {:.3} at {},{}",
+ uv_stats.max_diff,
+ uv_stats.mean,
+ uv_stats.col,
+ uv_stats.row
+ );
+ } else {
+ let u_stats = plane_delta(actual, expected, 1, cw, ch);
+ let v_stats = plane_delta(actual, expected, 2, cw, ch);
+ assert!(
+ u_stats.max_diff <= 3
+ && v_stats.max_diff <= 3
+ && u_stats.mean < 1.5
+ && v_stats.mean < 1.5,
+ "{width}x{height} chroma u max {} mean {:.3} at {},{} v max {} mean {:.3} at {},{}",
+ u_stats.max_diff,
+ u_stats.mean,
+ u_stats.col,
+ u_stats.row,
+ v_stats.max_diff,
+ v_stats.mean,
+ v_stats.col,
+ v_stats.row
+ );
+ }
+ }
+
+ fn assert_active_eq(actual: &frame::Video, expected: &frame::Video) {
+ for plane in 0..expected.planes() {
+ let row_bytes = if expected.format() == Pixel::NV12 && plane == 1 {
+ expected.width() as usize
+ } else {
+ expected.plane_width(plane) as usize
+ };
+ for row in 0..expected.plane_height(plane) as usize {
+ let got = &actual.data(plane)[row * actual.stride(plane)..][..row_bytes];
+ let want = &expected.data(plane)[row * expected.stride(plane)..][..row_bytes];
+ assert_eq!(got, want, "plane {plane} row {row}");
+ }
+ }
+ }
+
+ fn check_even(format: Pixel, width: u32, height: u32, dst_format: Pixel) {
+ let mut src = frame::Video::new(format, width, height);
+ fill_packed(&mut src);
+ let (y, u, v) = reference(&src);
+ let mut kernel = frame::Video::new(dst_format, width, height);
+ let mut sws = frame::Video::new(dst_format, width, height);
+ let mut scaler = fast_scaler(format, dst_format, width, height);
+ scaler.run(&src, &mut sws).unwrap();
+ if avx2_ready() {
+ assert!(
+ try_convert(&src, &mut kernel),
+ "{format:?} {width}x{height} -> {dst_format:?}"
+ );
+ assert_matches_reference(&kernel, &y, &u, &v);
+ assert_within_swscale(&kernel, &sws);
+ let mut via = frame::Video::new(dst_format, width, height);
+ let mut via_scaler = fast_scaler(format, dst_format, width, height);
+ convert_or_scale(&src, &mut via, &mut via_scaler).unwrap();
+ assert_active_eq(&via, &kernel);
+ } else {
+ assert!(!try_convert(&src, &mut kernel));
+ let mut via = frame::Video::new(dst_format, width, height);
+ let mut via_scaler = fast_scaler(format, dst_format, width, height);
+ convert_or_scale(&src, &mut via, &mut via_scaler).unwrap();
+ assert_active_eq(&via, &sws);
+ }
+ }
+
+ #[test]
+ fn even_uyvy_and_yuyv_match_reference_and_swscale() {
+ ffmpeg::init().unwrap();
+ let sizes = [
+ (2u32, 2u32),
+ (16, 2),
+ (18, 4),
+ (32, 8),
+ (48, 6),
+ (64, 48),
+ (66, 40),
+ (320, 180),
+ ];
+ for format in [Pixel::UYVY422, Pixel::YUYV422] {
+ for (width, height) in sizes {
+ check_even(format, width, height, Pixel::YUV420P);
+ }
+ }
+ }
+
+ #[test]
+ fn even_nv12_matches_where_the_encoder_outputs_nv12() {
+ ffmpeg::init().unwrap();
+ for format in [Pixel::UYVY422, Pixel::YUYV422] {
+ for (width, height) in [(16u32, 2u32), (18, 4), (64, 48), (66, 40)] {
+ check_even(format, width, height, Pixel::NV12);
+ }
+ }
+ }
+
+ #[test]
+ fn odd_sizes_fall_back_to_swscale() {
+ ffmpeg::init().unwrap();
+ let mut compared = 0u32;
+ for format in [Pixel::UYVY422, Pixel::YUYV422] {
+ for (width, height) in [
+ (2u32, 1u32),
+ (64, 63),
+ (32, 15),
+ (1, 2),
+ (65, 64),
+ (31, 17),
+ (65, 63),
+ ] {
+ let mut src = frame::Video::new(format, width, height);
+ fill_packed(&mut src);
+ let mut probe = frame::Video::new(Pixel::YUV420P, width, height);
+ assert!(
+ !try_convert(&src, &mut probe),
+ "{format:?} {width}x{height} should stay on swscale"
+ );
+ let Ok(mut via_scaler) = scaling::Context::get(
+ format,
+ width,
+ height,
+ Pixel::YUV420P,
+ width,
+ height,
+ Flags::FAST_BILINEAR,
+ ) else {
+ continue;
+ };
+ let Ok(mut direct_scaler) = scaling::Context::get(
+ format,
+ width,
+ height,
+ Pixel::YUV420P,
+ width,
+ height,
+ Flags::FAST_BILINEAR,
+ ) else {
+ continue;
+ };
+ let mut via = frame::Video::new(Pixel::YUV420P, width, height);
+ let mut direct = frame::Video::new(Pixel::YUV420P, width, height);
+ // av_frame_get_buffer does not clear memory, and swscale can
+ // leave the odd-height chroma tail untouched.
+ zero_frame(&mut via);
+ zero_frame(&mut direct);
+ convert_or_scale(&src, &mut via, &mut via_scaler).unwrap();
+ direct_scaler.run(&src, &mut direct).unwrap();
+ assert_active_eq(&via, &direct);
+ compared += 1;
+ }
+ }
+ assert!(compared > 0, "swscale rejected every odd size");
+ }
+
+ #[test]
+ fn scaling_and_other_formats_stay_on_swscale() {
+ ffmpeg::init().unwrap();
+ let src = frame::Video::new(Pixel::UYVY422, 64, 48);
+ let mut scaled = frame::Video::new(Pixel::YUV420P, 32, 24);
+ assert!(!try_convert(&src, &mut scaled));
+ let mut yuv422 = frame::Video::new(Pixel::YUV422P, 64, 48);
+ assert!(!try_convert(&src, &mut yuv422));
+ for format in [Pixel::BGRA, Pixel::YUV422P, Pixel::NV12, Pixel::RGB24] {
+ let other = frame::Video::new(format, 32, 16);
+ let mut yuv = frame::Video::new(Pixel::YUV420P, 32, 16);
+ assert!(!try_convert(&other, &mut yuv), "{format:?}");
+ }
+
+ let mut packed = frame::Video::new(Pixel::YUYV422, 64, 48);
+ fill_packed(&mut packed);
+ let mut scaler = scaling::Context::get(
+ Pixel::YUYV422,
+ 64,
+ 48,
+ Pixel::YUV420P,
+ 32,
+ 24,
+ Flags::BICUBIC,
+ )
+ .unwrap();
+ let mut direct_scaler = scaling::Context::get(
+ Pixel::YUYV422,
+ 64,
+ 48,
+ Pixel::YUV420P,
+ 32,
+ 24,
+ Flags::BICUBIC,
+ )
+ .unwrap();
+ let mut via = frame::Video::new(Pixel::YUV420P, 32, 24);
+ let mut direct = frame::Video::new(Pixel::YUV420P, 32, 24);
+ convert_or_scale(&packed, &mut via, &mut scaler).unwrap();
+ direct_scaler.run(&packed, &mut direct).unwrap();
+ assert_active_eq(&via, &direct);
+ }
+
+ #[test]
+ fn libx264_queue_accepts_packed_yuv() {
+ ffmpeg::init().unwrap();
+ queue_one(Pixel::UYVY422);
+ encode_one(Pixel::YUYV422);
+ }
+
+ fn queue_one(format: Pixel) {
+ if ffmpeg::codec::encoder::find_by_name("libx264").is_none() {
+ return;
+ }
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("packed.mp4");
+ let mut output = ffmpeg::format::output_as(&path, "mp4").unwrap();
+ let info = cap_media_info::VideoInfo::from_raw_ffmpeg(format, 64, 48, 30);
+ let mut encoder = super::super::h264::H264Encoder::builder(info)
+ .with_preset(super::super::h264::H264Preset::Ultrafast)
+ .with_crf(18)
+ .with_encoder_priority_override(&["libx264"])
+ .build(&mut output)
+ .unwrap();
+ let requirements = encoder.conversion_requirements();
+ assert_eq!(requirements.output_format, Pixel::YUV420P);
+ assert!(requirements.needs_conversion);
+ let mut frame = frame::Video::new(format, 64, 48);
+ fill_packed(&mut frame);
+ let mut sink = crate::PacketQueue::new(ffmpeg::Rational(1, 90_000));
+ encoder
+ .queue_frame_to(frame, std::time::Duration::ZERO, &mut sink)
+ .unwrap();
+ encoder.flush_to(&mut sink).unwrap();
+ assert!(!sink.take().is_empty());
+ }
+
+ fn encode_one(format: Pixel) {
+ if ffmpeg::codec::encoder::find_by_name("libx264").is_none() {
+ return;
+ }
+ let info = cap_media_info::VideoInfo::from_raw_ffmpeg(format, 64, 48, 30);
+ let mut encoder = super::super::h264::H264Encoder::builder(info)
+ .with_preset(super::super::h264::H264Preset::Ultrafast)
+ .with_crf(18)
+ .with_encoder_priority_override(&["libx264"])
+ .build_standalone()
+ .unwrap();
+ assert_eq!(encoder.output_format(), Pixel::YUV420P);
+ assert!(encoder.needs_conversion());
+ let mut frame = frame::Video::new(format, 64, 48);
+ fill_packed(&mut frame);
+ let mut packets = Vec::new();
+ encoder
+ .encode_frame(frame, std::time::Duration::ZERO, |packet| {
+ packets.push(packet.data);
+ Ok(())
+ })
+ .unwrap();
+ encoder
+ .flush(|packet| {
+ packets.push(packet.data);
+ Ok(())
+ })
+ .unwrap();
+ assert!(!packets.is_empty());
+ }
+
+ #[test]
+ #[ignore = "release timing bench, run with --ignored --nocapture"]
+ fn bench_packed_yuv_conversion() {
+ ffmpeg::init().unwrap();
+ let model = std::fs::read_to_string("/proc/cpuinfo")
+ .ok()
+ .and_then(|text| {
+ text.lines()
+ .find(|line| line.starts_with("model name"))
+ .and_then(|line| line.split(':').nth(1))
+ .map(|value| value.trim().to_string())
+ })
+ .unwrap_or_else(|| "unknown".into());
+ println!("cpu: {model}");
+ println!("avx2: {}", avx2_ready());
+ println!("warmup 5 excluded; median is the middle of 31 sorted samples");
+ for format in [Pixel::UYVY422, Pixel::YUYV422] {
+ for (width, height) in [(1280u32, 720u32), (1920, 1080), (3840, 2160)] {
+ bench_conversion(format, width, height);
+ }
+ }
+ }
+
+ fn bench_conversion(format: Pixel, width: u32, height: u32) {
+ let mut src = frame::Video::new(format, width, height);
+ fill_packed(&mut src);
+ let mut kernel_dst = frame::Video::new(Pixel::YUV420P, width, height);
+ let mut sws_dst = frame::Video::new(Pixel::YUV420P, width, height);
+ let mut kernel_scaler = fast_scaler(format, Pixel::YUV420P, width, height);
+ let mut sws_scaler = fast_scaler(format, Pixel::YUV420P, width, height);
+ let sws = time_median(5, 31, || {
+ sws_scaler.run(&src, &mut sws_dst).unwrap();
+ let _ = std::hint::black_box(sws_dst.data(0));
+ let _ = std::hint::black_box(sws_dst.data(1));
+ let _ = std::hint::black_box(sws_dst.data(2));
+ });
+ let kernel = time_median(5, 31, || {
+ convert_or_scale(&src, &mut kernel_dst, &mut kernel_scaler).unwrap();
+ let _ = std::hint::black_box(kernel_dst.data(0));
+ let _ = std::hint::black_box(kernel_dst.data(1));
+ let _ = std::hint::black_box(kernel_dst.data(2));
+ });
+ println!(
+ "{format:?} {width}x{height} swscale_fast_bilinear median {:.3} ms n=31",
+ sws / 1_000_000.0
+ );
+ println!(
+ "{format:?} {width}x{height} convert_or_scale median {:.3} ms n=31 speedup {:.2}x",
+ kernel / 1_000_000.0,
+ sws / kernel
+ );
+ }
+
+ fn time_median(warmup: usize, runs: usize, mut body: impl FnMut()) -> f64 {
+ for _ in 0..warmup {
+ body();
+ }
+ let mut samples = Vec::with_capacity(runs);
+ for _ in 0..runs {
+ let started = std::time::Instant::now();
+ body();
+ samples.push(started.elapsed().as_nanos());
+ }
+ samples.sort_unstable();
+ samples[runs / 2] as f64
+ }
+}
diff --git a/crates/frame-converter/src/frame_pool.rs b/crates/frame-converter/src/frame_pool.rs
index 4bba2af3e..51d880fbf 100644
--- a/crates/frame-converter/src/frame_pool.rs
+++ b/crates/frame-converter/src/frame_pool.rs
@@ -1,4 +1,3 @@
-use crate::output_frame::cached_output_frame;
use ffmpeg::{format::Pixel, frame};
use parking_lot::Mutex;
use std::sync::Arc;
@@ -15,7 +14,7 @@ impl VideoFramePool {
pub fn new(capacity: usize, format: Pixel, width: u32, height: u32) -> Arc<Self> {
let mut frames = Vec::with_capacity(capacity);
for _ in 0..capacity {
- frames.push(cached_output_frame(format, width, height));
+ frames.push(frame::Video::new(format, width, height));
}
Arc::new(Self {
@@ -31,7 +30,7 @@ impl VideoFramePool {
self.frames
.lock()
.pop()
- .unwrap_or_else(|| cached_output_frame(self.format, self.width, self.height))
+ .unwrap_or_else(|| frame::Video::new(self.format, self.width, self.height))
}
pub fn put(&self, frame: frame::Video) {
@@ -57,33 +56,3 @@ impl VideoFramePool {
(self.width, self.height)
}
}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use std::sync::Once;
-
- fn init_ffmpeg() {
- static ONCE: Once = Once::new();
- ONCE.call_once(|| {
- ffmpeg::init().expect("ffmpeg init");
- });
- }
-
- #[test]
- fn pooled_frames_match_ffmpeg_geometry() {
- init_ffmpeg();
- let pool = VideoFramePool::new(2, Pixel::NV12, 64, 32);
- assert_eq!(pool.available(), 2);
- let frame = pool.get();
- let fresh = frame::Video::new(Pixel::NV12, 64, 32);
- assert_eq!(frame.format(), fresh.format());
- assert_eq!(frame.width(), fresh.width());
- assert_eq!(frame.height(), fresh.height());
- assert_eq!(frame.stride(0), fresh.stride(0));
- assert_eq!(frame.stride(1), fresh.stride(1));
- assert_eq!(pool.available(), 1);
- pool.put(frame);
- assert_eq!(pool.available(), 2);
- }
-}
diff --git a/crates/frame-converter/src/lib.rs b/crates/frame-converter/src/lib.rs
index 82811bcc5..cbe4c5810 100644
--- a/crates/frame-converter/src/lib.rs
+++ b/crates/frame-converter/src/lib.rs
@@ -1,6 +1,3 @@
-mod output_frame;
-mod packed_yuv;
-
mod swscale;
pub use swscale::*;
diff --git a/crates/frame-converter/src/output_frame.rs b/crates/frame-converter/src/output_frame.rs
deleted file mode 100644
index 83195a9f9..000000000
--- a/crates/frame-converter/src/output_frame.rs
+++ /dev/null
@@ -1,269 +0,0 @@
-use ffmpeg::format::Pixel;
-use ffmpeg::frame;
-use parking_lot::Mutex;
-use std::ffi::c_void;
-
-// Image bytes stay in this pool after the last AVFrame reference drops, so
-// `convert` can hand out the same allocation again. Callers still receive an
-// exclusively owned frame::Video.
-const POOLED_PER_SIZE: usize = 16;
-
-struct PooledBuffer {
- ptr: *mut u8,
- size: usize,
-}
-
-// Exclusive ownership of `ptr` moves with the value under OUTPUT_POOL.
-unsafe impl Send for PooledBuffer {}
-
-static OUTPUT_POOL: Mutex<Vec<PooledBuffer>> = Mutex::new(Vec::new());
-static LAYOUTS: Mutex<Vec<FrameLayout>> = Mutex::new(Vec::new());
-
-#[derive(Clone, Copy)]
-struct FrameLayout {
- format: Pixel,
- width: u32,
- height: u32,
- linesize: [i32; 8],
- offset: [isize; 8],
- size: usize,
-}
-
-pub(crate) fn cached_output_frame(format: Pixel, width: u32, height: u32) -> frame::Video {
- if width == 0 || height == 0 {
- return frame::Video::new(format, width, height);
- }
- let Some(layout) = layout_for(format, width, height) else {
- return frame::Video::new(format, width, height);
- };
- assemble(&layout).unwrap_or_else(|| frame::Video::new(format, width, height))
-}
-
-fn layout_for(format: Pixel, width: u32, height: u32) -> Option<FrameLayout> {
- if let Some(layout) = find_layout(format, width, height) {
- return Some(layout);
- }
- let probe = frame::Video::new(format, width, height);
- let layout = capture(&probe)?;
- drop(probe);
- let mut layouts = LAYOUTS.lock();
- if let Some(existing) = layouts
- .iter()
- .copied()
- .find(|layout| layout.format == format && layout.width == width && layout.height == height)
- {
- return Some(existing);
- }
- layouts.push(layout);
- Some(layout)
-}
-
-fn find_layout(format: Pixel, width: u32, height: u32) -> Option<FrameLayout> {
- LAYOUTS
- .lock()
- .iter()
- .copied()
- .find(|layout| layout.format == format && layout.width == width && layout.height == height)
-}
-
-fn capture(frame: &frame::Video) -> Option<FrameLayout> {
- unsafe {
- let raw = frame.as_ptr();
- if raw.is_null() || (*raw).buf[0].is_null() {
- return None;
- }
- for index in 1..8 {
- if !(*raw).buf[index].is_null() {
- return None;
- }
- }
- let buf = (*raw).buf[0];
- let base = (*buf).data;
- let size = (*buf).size;
- if base.is_null() || size == 0 {
- return None;
- }
- let mut linesize = [0i32; 8];
- let mut offset = [-1isize; 8];
- for index in 0..8 {
- linesize[index] = (*raw).linesize[index];
- let plane = (*raw).data[index];
- if plane.is_null() {
- continue;
- }
- let off = (plane as usize).wrapping_sub(base as usize);
- if off >= size {
- return None;
- }
- offset[index] = off as isize;
- }
- if offset[0] < 0 {
- return None;
- }
- Some(FrameLayout {
- format: frame.format(),
- width: frame.width(),
- height: frame.height(),
- linesize,
- offset,
- size,
- })
- }
-}
-
-fn assemble(layout: &FrameLayout) -> Option<frame::Video> {
- let data = take_pooled(layout.size).or_else(|| malloc_zeroed(layout.size))?;
- let mut frame = frame::Video::empty();
- if unsafe { frame.as_ptr() }.is_null() {
- free_data(data);
- return None;
- }
- frame.set_format(layout.format);
- frame.set_width(layout.width);
- frame.set_height(layout.height);
- unsafe {
- let raw = frame.as_mut_ptr();
- let buf = ffmpeg::ffi::av_buffer_create(
- data,
- layout.size,
- Some(release_output_buffer),
- layout.size as *mut c_void,
- 0,
- );
- if buf.is_null() {
- free_data(data);
- return None;
- }
- (*raw).buf[0] = buf;
- for index in 0..8 {
- (*raw).linesize[index] = layout.linesize[index];
- if layout.offset[index] >= 0 {
- (*raw).data[index] = data.offset(layout.offset[index]);
- }
- }
- (*raw).extended_data = (*raw).data.as_mut_ptr();
- }
- Some(frame)
-}
-
-fn take_pooled(size: usize) -> Option<*mut u8> {
- let mut pool = OUTPUT_POOL.lock();
- let index = pool.iter().rposition(|buf| buf.size == size)?;
- Some(pool.swap_remove(index).ptr)
-}
-
-fn malloc_zeroed(size: usize) -> Option<*mut u8> {
- if size == 0 {
- return None;
- }
- unsafe {
- let ptr = ffmpeg::ffi::av_malloc(size) as *mut u8;
- if ptr.is_null() {
- None
- } else {
- std::ptr::write_bytes(ptr, 0, size);
- Some(ptr)
- }
- }
-}
-
-fn free_data(ptr: *mut u8) {
- unsafe {
- ffmpeg::ffi::av_free(ptr.cast());
- }
-}
-
-unsafe extern "C" fn release_output_buffer(opaque: *mut c_void, data: *mut u8) {
- if data.is_null() {
- return;
- }
- let size = opaque as usize;
- if size == 0 {
- free_data(data);
- return;
- }
- let mut pool = OUTPUT_POOL.lock();
- let same_size = pool.iter().filter(|buf| buf.size == size).count();
- if same_size < POOLED_PER_SIZE {
- pool.push(PooledBuffer { ptr: data, size });
- } else {
- free_data(data);
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use std::sync::Once;
-
- fn init_ffmpeg() {
- static ONCE: Once = Once::new();
- ONCE.call_once(|| {
- ffmpeg::init().expect("ffmpeg init");
- });
- }
-
- fn plane_offset(frame: &frame::Video, plane: usize) -> usize {
- unsafe {
- let raw = frame.as_ptr();
- let base = (*(*raw).buf[0]).data as usize;
- (*raw).data[plane] as usize - base
- }
- }
-
- #[test]
- fn cached_frame_matches_ffmpeg_geometry() {
- init_ffmpeg();
- let mut cached = cached_output_frame(Pixel::NV12, 94, 46);
- let fresh = frame::Video::new(Pixel::NV12, 94, 46);
- assert_eq!(cached.format(), fresh.format());
- assert_eq!(cached.width(), fresh.width());
- assert_eq!(cached.height(), fresh.height());
- assert_eq!(cached.stride(0), fresh.stride(0));
- assert_eq!(cached.stride(1), fresh.stride(1));
- assert_eq!(cached.planes(), fresh.planes());
- assert_eq!(plane_offset(&cached, 0), plane_offset(&fresh, 0));
- assert_eq!(plane_offset(&cached, 1), plane_offset(&fresh, 1));
- unsafe {
- assert_eq!(ffmpeg::ffi::av_frame_is_writable(cached.as_mut_ptr()), 1);
- }
- }
-
- #[test]
- fn dropped_frame_returns_its_buffer() {
- init_ffmpeg();
- let first = cached_output_frame(Pixel::NV12, 95, 46);
- let ptr = unsafe { (*first.as_ptr()).data[0] };
- drop(first);
- let mut second = cached_output_frame(Pixel::NV12, 95, 46);
- let reused = unsafe { (*second.as_ptr()).data[0] };
- assert_eq!(reused, ptr);
- second.data_mut(0).fill(7);
- assert!(second.data(0).iter().all(|byte| *byte == 7));
- }
-
- #[test]
- fn live_reference_keeps_the_buffer_out_of_the_pool() {
- init_ffmpeg();
- let mut held = frame::Video::empty();
- let ptr = {
- let mut frame = cached_output_frame(Pixel::YUYV422, 96, 40);
- frame.data_mut(0).fill(19);
- unsafe {
- let status = ffmpeg::ffi::av_frame_ref(held.as_mut_ptr(), frame.as_ptr());
- assert!(status >= 0);
- (*frame.as_ptr()).data[0]
- }
- };
- let next = cached_output_frame(Pixel::YUYV422, 96, 40);
- let next_ptr = unsafe { (*next.as_ptr()).data[0] };
- assert_ne!(next_ptr, ptr);
- assert!(held.data(0).iter().all(|byte| *byte == 19));
- drop(held);
- let recycled = cached_output_frame(Pixel::YUYV422, 96, 40);
- let recycled_ptr = unsafe { (*recycled.as_ptr()).data[0] };
- assert_eq!(recycled_ptr, ptr);
- assert_ne!(next_ptr, recycled_ptr);
- drop(next);
- }
-}
diff --git a/crates/frame-converter/src/packed_yuv.rs b/crates/frame-converter/src/packed_yuv.rs
deleted file mode 100644
index d8fd8f61d..000000000
--- a/crates/frame-converter/src/packed_yuv.rs
+++ /dev/null
@@ -1,665 +0,0 @@
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub(crate) enum Packed422 {
- Uyvy,
- Yuyv,
-}
-
-pub(crate) struct Nv12Mut<'a> {
- pub y: &'a mut [u8],
- pub y_stride: usize,
- pub uv: &'a mut [u8],
- pub uv_stride: usize,
-}
-
-struct PackedView<'a> {
- kind: Packed422,
- src: &'a [u8],
- stride: usize,
- width: usize,
- height: usize,
-}
-
-pub(crate) fn convert_packed422_to_nv12(
- kind: Packed422,
- src: &[u8],
- src_stride: usize,
- width: usize,
- height: usize,
- dst: &mut Nv12Mut<'_>,
-) -> bool {
- if width == 0 || height == 0 {
- return true;
- }
- let view = PackedView {
- kind,
- src,
- stride: src_stride,
- width,
- height,
- };
- if !buffers_fit(&view, dst) {
- return false;
- }
- if avx2_available() {
- unsafe {
- avx_convert(&view, dst);
- }
- } else {
- scalar_rows(&view, dst);
- }
- true
-}
-
-#[cfg(test)]
-pub(crate) fn convert_packed422_scalar(
- kind: Packed422,
- src: &[u8],
- src_stride: usize,
- width: usize,
- height: usize,
- dst: &mut Nv12Mut<'_>,
-) -> bool {
- if width == 0 || height == 0 {
- return true;
- }
- let view = PackedView {
- kind,
- src,
- stride: src_stride,
- width,
- height,
- };
- if !buffers_fit(&view, dst) {
- return false;
- }
- scalar_rows(&view, dst);
- true
-}
-
-fn avx2_available() -> bool {
- #[cfg(target_arch = "x86_64")]
- {
- is_x86_feature_detected!("avx2")
- }
- #[cfg(not(target_arch = "x86_64"))]
- {
- false
- }
-}
-
-fn chroma_row_bytes(width: usize) -> usize {
- width.div_ceil(2) * 2
-}
-
-fn buffers_fit(src: &PackedView<'_>, dst: &Nv12Mut<'_>) -> bool {
- let Some(packed) = src.width.checked_mul(2) else {
- return false;
- };
- let chroma = chroma_row_bytes(src.width);
- if src.stride < packed || dst.y_stride < src.width || dst.uv_stride < chroma {
- return false;
- }
- let Some(src_need) = src.stride.checked_mul(src.height) else {
- return false;
- };
- let Some(y_need) = dst.y_stride.checked_mul(src.height) else {
- return false;
- };
- let Some(uv_need) = dst.uv_stride.checked_mul(src.height.div_ceil(2)) else {
- return false;
- };
- src.src.len() >= src_need && dst.y.len() >= y_need && dst.uv.len() >= uv_need
-}
-
-fn scalar_rows(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>) {
- let uv_rows = src.height.div_ceil(2);
- for row in 0..uv_rows {
- write_pixels(src, dst, row, 0);
- }
-}
-
-fn sample(row: &[u8], packed_len: usize, index: usize) -> u8 {
- if index < packed_len { row[index] } else { 128 }
-}
-
-fn avg_u8(a: u8, b: u8) -> u8 {
- ((u16::from(a) + u16::from(b) + 1) >> 1) as u8
-}
-
-// (y_left, y_right, u, v). A short last macropixel reads missing bytes as 128.
-fn load_macro(kind: Packed422, row: &[u8], pair: usize, packed_len: usize) -> (u8, u8, u8, u8) {
- let off = pair * 4;
- match kind {
- Packed422::Uyvy => (
- sample(row, packed_len, off + 1),
- sample(row, packed_len, off + 3),
- sample(row, packed_len, off),
- sample(row, packed_len, off + 2),
- ),
- Packed422::Yuyv => (
- sample(row, packed_len, off),
- sample(row, packed_len, off + 2),
- sample(row, packed_len, off + 1),
- sample(row, packed_len, off + 3),
- ),
- }
-}
-
-fn write_pixels(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>, row: usize, x_start: usize) {
- debug_assert!(x_start.is_multiple_of(2));
- let width = src.width;
- let height = src.height;
- let src_stride = src.stride;
- let y_stride = dst.y_stride;
- let uv_stride = dst.uv_stride;
- let top = row * 2;
- let bot = top + 1;
- let average = bot < height;
- let packed_len = width * 2;
- let src0 = top * src_stride;
- let src1 = bot * src_stride;
- let y0 = top * y_stride;
- let y1 = bot * y_stride;
- let uv_base = row * uv_stride;
- let mut x = x_start;
- while x < width {
- let pair = x / 2;
- let (left, right, u0, v0) = load_macro(
- src.kind,
- &src.src[src0..src0 + src_stride],
- pair,
- packed_len,
- );
- dst.y[y0 + x] = left;
- if x + 1 < width {
- dst.y[y0 + x + 1] = right;
- }
- let (u, v) = if average {
- let (left1, right1, u1, v1) = load_macro(
- src.kind,
- &src.src[src1..src1 + src_stride],
- pair,
- packed_len,
- );
- dst.y[y1 + x] = left1;
- if x + 1 < width {
- dst.y[y1 + x + 1] = right1;
- }
- (avg_u8(u0, u1), avg_u8(v0, v1))
- } else {
- (u0, v0)
- };
- let uv_at = uv_base + pair * 2;
- dst.uv[uv_at] = u;
- dst.uv[uv_at + 1] = v;
- x += 2;
- }
-}
-
-#[cfg(target_arch = "x86_64")]
-struct RowPtrs {
- src0: *const u8,
- src1: *const u8,
- y0: *mut u8,
- y1: *mut u8,
- uv: *mut u8,
- average: bool,
-}
-
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn avx_convert(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>) {
- let masks = match src.kind {
- Packed422::Uyvy => Masks {
- y: unsafe { uyvy_y_mask() },
- u: unsafe { uyvy_u_mask() },
- v: unsafe { uyvy_v_mask() },
- },
- Packed422::Yuyv => Masks {
- y: unsafe { yuyv_y_mask() },
- u: unsafe { yuyv_u_mask() },
- v: unsafe { yuyv_v_mask() },
- },
- };
- let width = src.width;
- let height = src.height;
- let src_stride = src.stride;
- let y_stride = dst.y_stride;
- let uv_stride = dst.uv_stride;
- let uv_rows = height.div_ceil(2);
- for row in 0..uv_rows {
- let top = row * 2;
- let bot = top + 1;
- let average = bot < height;
- let src0 = src.src.as_ptr().wrapping_add(top * src_stride);
- let src1 = if average {
- src.src.as_ptr().wrapping_add(bot * src_stride)
- } else {
- std::ptr::null()
- };
- let y0 = dst.y.as_mut_ptr().wrapping_add(top * y_stride);
- let y1 = if average {
- dst.y.as_mut_ptr().wrapping_add(bot * y_stride)
- } else {
- std::ptr::null_mut()
- };
- let uv = dst.uv.as_mut_ptr().wrapping_add(row * uv_stride);
- let ptrs = RowPtrs {
- src0,
- src1,
- y0,
- y1,
- uv,
- average,
- };
- let mut x = 0usize;
- while x + 32 <= width {
- unsafe {
- convert16(&masks, &ptrs, x);
- convert16(&masks, &ptrs, x + 16);
- }
- x += 32;
- }
- while x + 16 <= width {
- unsafe {
- convert16(&masks, &ptrs, x);
- }
- x += 16;
- }
- if x < width {
- write_pixels(src, dst, row, x);
- }
- }
-}
-
-#[cfg(not(target_arch = "x86_64"))]
-unsafe fn avx_convert(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>) {
- scalar_rows(src, dst);
-}
-
-#[cfg(target_arch = "x86_64")]
-struct Masks {
- y: std::arch::x86_64::__m256i,
- u: std::arch::x86_64::__m256i,
- v: std::arch::x86_64::__m256i,
-}
-
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn convert16(masks: &Masks, row: &RowPtrs, x: usize) {
- use std::arch::x86_64::*;
-
- let packed0 = unsafe { _mm256_loadu_si256(row.src0.add(x * 2).cast()) };
- unsafe { store_luma(packed0, masks.y, row.y0.add(x)) };
- let u0 = unsafe { pack_chroma8(packed0, masks.u) };
- let v0 = unsafe { pack_chroma8(packed0, masks.v) };
- let (u, v) = if row.average {
- let packed1 = unsafe { _mm256_loadu_si256(row.src1.add(x * 2).cast()) };
- unsafe { store_luma(packed1, masks.y, row.y1.add(x)) };
- let u1 = unsafe { pack_chroma8(packed1, masks.u) };
- let v1 = unsafe { pack_chroma8(packed1, masks.v) };
- // avg_epu8 is (a + b + 1) >> 1 for each byte.
- (_mm_avg_epu8(u0, u1), _mm_avg_epu8(v0, v1))
- } else {
- (u0, v0)
- };
- let interleaved = _mm_unpacklo_epi8(u, v);
- unsafe { _mm_storeu_si128(row.uv.add(x).cast(), interleaved) };
-}
-
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn store_luma(
- packed: std::arch::x86_64::__m256i,
- mask: std::arch::x86_64::__m256i,
- dst: *mut u8,
-) {
- use std::arch::x86_64::*;
- let shuffled = _mm256_shuffle_epi8(packed, mask);
- let lo = _mm256_castsi256_si128(shuffled);
- let hi = _mm256_extracti128_si256(shuffled, 1);
- let y = _mm_unpacklo_epi64(lo, hi);
- unsafe { _mm_storeu_si128(dst.cast(), y) };
-}
-
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn pack_chroma8(
- packed: std::arch::x86_64::__m256i,
- mask: std::arch::x86_64::__m256i,
-) -> std::arch::x86_64::__m128i {
- use std::arch::x86_64::*;
- let shuffled = _mm256_shuffle_epi8(packed, mask);
- let lo = _mm256_castsi256_si128(shuffled);
- let hi = _mm256_extracti128_si256(shuffled, 1);
- _mm_unpacklo_epi32(lo, hi)
-}
-
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn uyvy_y_mask() -> std::arch::x86_64::__m256i {
- use std::arch::x86_64::*;
- _mm256_setr_epi8(
- 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, 1, 3, 5, 7, 9, 11, 13, 15, -1,
- -1, -1, -1, -1, -1, -1, -1,
- )
-}
-
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn uyvy_u_mask() -> std::arch::x86_64::__m256i {
- use std::arch::x86_64::*;
- _mm256_setr_epi8(
- 0, 4, 8, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 4, 8, 12, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1,
- )
-}
-
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn uyvy_v_mask() -> std::arch::x86_64::__m256i {
- use std::arch::x86_64::*;
- _mm256_setr_epi8(
- 2, 6, 10, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 6, 10, 14, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1,
- )
-}
-
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn yuyv_y_mask() -> std::arch::x86_64::__m256i {
- use std::arch::x86_64::*;
- _mm256_setr_epi8(
- 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, 0, 2, 4, 6, 8, 10, 12, 14, -1,
- -1, -1, -1, -1, -1, -1, -1,
- )
-}
-
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn yuyv_u_mask() -> std::arch::x86_64::__m256i {
- use std::arch::x86_64::*;
- _mm256_setr_epi8(
- 1, 5, 9, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 5, 9, 13, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1,
- )
-}
-
-#[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-unsafe fn yuyv_v_mask() -> std::arch::x86_64::__m256i {
- use std::arch::x86_64::*;
- _mm256_setr_epi8(
- 3, 7, 11, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 3, 7, 11, 15, -1, -1, -1, -1,
- -1, -1, -1, -1, -1, -1, -1, -1,
- )
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- fn fill(buf: &mut [u8], mut seed: u32) {
- for byte in buf.iter_mut() {
- seed = seed.wrapping_mul(1664525).wrapping_add(1013904223);
- *byte = (seed >> 24) as u8;
- }
- }
-
- fn ref_byte(row: &[u8], packed_len: usize, index: usize) -> u8 {
- if index < packed_len { row[index] } else { 128 }
- }
-
- fn ref_sample(kind: Packed422, row: &[u8], packed_len: usize, x: usize) -> (u8, u8, u8) {
- let off = (x / 2) * 4;
- let (y, u, v) = match kind {
- Packed422::Uyvy => {
- let y = if x.is_multiple_of(2) {
- ref_byte(row, packed_len, off + 1)
- } else {
- ref_byte(row, packed_len, off + 3)
- };
- (
- y,
- ref_byte(row, packed_len, off),
- ref_byte(row, packed_len, off + 2),
- )
- }
- Packed422::Yuyv => {
- let y = if x.is_multiple_of(2) {
- ref_byte(row, packed_len, off)
- } else {
- ref_byte(row, packed_len, off + 2)
- };
- (
- y,
- ref_byte(row, packed_len, off + 1),
- ref_byte(row, packed_len, off + 3),
- )
- }
- };
- (y, u, v)
- }
-
- fn reference(
- src: &PackedView<'_>,
- y: &mut [u8],
- y_stride: usize,
- uv: &mut [u8],
- uv_stride: usize,
- ) {
- let packed_len = src.width * 2;
- for row in 0..src.height {
- let src_row = &src.src[row * src.stride..row * src.stride + src.stride];
- for x in 0..src.width {
- let (sample_y, _, _) = ref_sample(src.kind, src_row, packed_len, x);
- y[row * y_stride + x] = sample_y;
- }
- }
- let chroma_pairs = src.width.div_ceil(2);
- for row in 0..src.height.div_ceil(2) {
- let top = row * 2;
- let src_top = &src.src[top * src.stride..top * src.stride + src.stride];
- let bot = top + 1;
- let src_bot = (bot < src.height)
- .then(|| &src.src[bot * src.stride..bot * src.stride + src.stride]);
- for pair in 0..chroma_pairs {
- let x = pair * 2;
- let (_, u0, v0) = ref_sample(src.kind, src_top, packed_len, x);
- let (u, v) = if let Some(src_bot) = src_bot {
- let (_, u1, v1) = ref_sample(src.kind, src_bot, packed_len, x);
- (
- ((u16::from(u0) + u16::from(u1) + 1) >> 1) as u8,
- ((u16::from(v0) + u16::from(v1) + 1) >> 1) as u8,
- )
- } else {
- (u0, v0)
- };
- uv[row * uv_stride + pair * 2] = u;
- uv[row * uv_stride + pair * 2 + 1] = v;
- }
- }
- }
-
- fn run_scalar(
- kind: Packed422,
- width: usize,
- height: usize,
- src_pad: usize,
- y_pad: usize,
- uv_pad: usize,
- seed: u32,
- ) -> (Vec<u8>, Vec<u8>, Vec<u8>, Vec<u8>) {
- let src_stride = width * 2 + src_pad;
- let y_stride = width + y_pad;
- let uv_stride = chroma_row_bytes(width) + uv_pad;
- let mut src = vec![0u8; src_stride * height];
- fill(&mut src, seed);
- let mut y = vec![0x5Au8; y_stride * height];
- let mut uv = vec![0x5Au8; uv_stride * height.div_ceil(2)];
- let mut y_ref = y.clone();
- let mut uv_ref = uv.clone();
- let mut dst = Nv12Mut {
- y: &mut y,
- y_stride,
- uv: &mut uv,
- uv_stride,
- };
- let view = PackedView {
- kind,
- src: &src,
- stride: src_stride,
- width,
- height,
- };
- assert!(convert_rows_scalar(&view, &mut dst));
- reference(&view, &mut y_ref, y_stride, &mut uv_ref, uv_stride);
- (y, uv, y_ref, uv_ref)
- }
-
- fn convert_rows_scalar(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>) -> bool {
- if !buffers_fit(src, dst) {
- return false;
- }
- scalar_rows(src, dst);
- true
- }
-
- fn case(
- kind: Packed422,
- width: usize,
- height: usize,
- src_pad: usize,
- y_pad: usize,
- uv_pad: usize,
- ) {
- let seed = (width as u32)
- .wrapping_mul(1000)
- .wrapping_add(height as u32 * 17)
- .wrapping_add(src_pad as u32 * 3)
- .wrapping_add(y_pad as u32 * 5)
- .wrapping_add(uv_pad as u32 * 9)
- .wrapping_add(match kind {
- Packed422::Uyvy => 1,
- Packed422::Yuyv => 2,
- });
- let (y, uv, y_ref, uv_ref) = run_scalar(kind, width, height, src_pad, y_pad, uv_pad, seed);
- assert_eq!(
- y, y_ref,
- "{kind:?} {width}x{height} y pad {src_pad}/{y_pad}/{uv_pad}"
- );
- assert_eq!(
- uv, uv_ref,
- "{kind:?} {width}x{height} uv pad {src_pad}/{y_pad}/{uv_pad}"
- );
- }
-
- #[test]
- fn scalar_matches_reference_across_shapes() {
- let widths = [
- 1usize, 2, 3, 7, 8, 15, 16, 17, 31, 32, 33, 47, 63, 64, 65, 100, 127, 128,
- ];
- let heights = [1usize, 2, 3, 4, 5, 27];
- let src_pads = [0usize, 7, 32];
- let y_pads = [0usize, 13];
- let uv_pads = [0usize, 15];
- for kind in [Packed422::Uyvy, Packed422::Yuyv] {
- for &width in &widths {
- for &height in &heights {
- for &src_pad in &src_pads {
- for &y_pad in &y_pads {
- for &uv_pad in &uv_pads {
- case(kind, width, height, src_pad, y_pad, uv_pad);
- }
- }
- }
- }
- }
- case(kind, 1920, 2, 16, 8, 24);
- case(kind, 1919, 3, 1, 0, 5);
- case(kind, 1280, 4, 0, 0, 0);
- }
- }
-
- #[cfg(target_arch = "x86_64")]
- #[test]
- fn avx2_matches_scalar_across_shapes() {
- if !is_x86_feature_detected!("avx2") {
- return;
- }
- let widths = [
- 1usize, 2, 3, 7, 8, 15, 16, 17, 31, 32, 33, 47, 63, 64, 65, 100, 127, 128,
- ];
- let heights = [1usize, 2, 3, 5, 27];
- let src_pads = [0usize, 7, 32];
- let y_pads = [0usize, 13];
- let uv_pads = [0usize, 15];
- for kind in [Packed422::Uyvy, Packed422::Yuyv] {
- for &width in &widths {
- for &height in &heights {
- for &src_pad in &src_pads {
- for &y_pad in &y_pads {
- for &uv_pad in &uv_pads {
- compare_avx(kind, width, height, src_pad, y_pad, uv_pad);
- }
- }
- }
- }
- }
- compare_avx(kind, 1920, 2, 16, 8, 24);
- compare_avx(kind, 1919, 3, 3, 1, 9);
- compare_avx(kind, 3840, 2, 0, 0, 0);
- }
- }
-
- #[cfg(target_arch = "x86_64")]
- fn compare_avx(
- kind: Packed422,
- width: usize,
- height: usize,
- src_pad: usize,
- y_pad: usize,
- uv_pad: usize,
- ) {
- let src_stride = width * 2 + src_pad;
- let y_stride = width + y_pad;
- let uv_stride = chroma_row_bytes(width) + uv_pad;
- let mut src = vec![0u8; src_stride * height];
- fill(&mut src, width as u32 * 13 + height as u32);
- let mut y_avx = vec![0x5Au8; y_stride * height];
- let mut uv_avx = vec![0x5Au8; uv_stride * height.div_ceil(2)];
- let mut y_scalar = y_avx.clone();
- let mut uv_scalar = uv_avx.clone();
- {
- let mut dst = Nv12Mut {
- y: &mut y_avx,
- y_stride,
- uv: &mut uv_avx,
- uv_stride,
- };
- assert!(convert_packed422_to_nv12(
- kind, &src, src_stride, width, height, &mut dst
- ));
- }
- {
- let mut dst = Nv12Mut {
- y: &mut y_scalar,
- y_stride,
- uv: &mut uv_scalar,
- uv_stride,
- };
- assert!(convert_rows_scalar(
- &PackedView {
- kind,
- src: &src,
- stride: src_stride,
- width,
- height,
- },
- &mut dst
- ));
- }
- assert_eq!(y_avx, y_scalar, "{kind:?} {width}x{height} avx y");
- assert_eq!(uv_avx, uv_scalar, "{kind:?} {width}x{height} avx uv");
- }
-}
diff --git a/crates/frame-converter/src/swscale.rs b/crates/frame-converter/src/swscale.rs
index 18ee4bd80..eafff2924 100644
--- a/crates/frame-converter/src/swscale.rs
+++ b/crates/frame-converter/src/swscale.rs
@@ -1,5 +1,3 @@
-use crate::output_frame::cached_output_frame;
-use crate::packed_yuv::{self, Nv12Mut, Packed422};
use crate::{ConversionConfig, ConvertError, ConverterBackend, FrameConverter};
use ffmpeg::{format::Pixel, frame, software::scaling};
use std::cell::UnsafeCell;
@@ -63,124 +61,12 @@ impl SwscaleConverter {
self.contexts
.get_or_try(|| Self::create_context(&self.config).map(SendableContext::new))
}
-
- #[cfg(test)]
- pub(crate) fn convert_swscale_into(
- &self,
- input: &frame::Video,
- output: &mut frame::Video,
- ) -> Result<(), ConvertError> {
- let context = self.get_or_create_context()?;
- context
- .run(input, output)
- .map_err(|e| ConvertError::ConversionFailed(e.to_string()))?;
- Ok(())
- }
-}
-
-fn positive_linesize(frame: &frame::Video, plane: usize) -> Option<usize> {
- if plane >= frame.planes() {
- return None;
- }
- let linesize = unsafe { (*frame.as_ptr()).linesize[plane] };
- if linesize <= 0 {
- None
- } else {
- Some(linesize as usize)
- }
-}
-
-fn planes_disjoint(a: *const u8, a_len: usize, b: *const u8, b_len: usize) -> bool {
- if a_len == 0 || b_len == 0 {
- return true;
- }
- let Some(a_end) = (a as usize).checked_add(a_len) else {
- return false;
- };
- let Some(b_end) = (b as usize).checked_add(b_len) else {
- return false;
- };
- a_end <= b as usize || b_end <= a as usize
-}
-
-pub(crate) fn try_packed_yuv_nv12(input: &frame::Video, output: &mut frame::Video) -> bool {
- let kind = match input.format() {
- Pixel::UYVY422 => Packed422::Uyvy,
- Pixel::YUYV422 => Packed422::Yuyv,
- _ => return false,
- };
- if output.format() != Pixel::NV12 {
- return false;
- }
- let width_px = input.width();
- let height_px = input.height();
- if output.width() != width_px || output.height() != height_px {
- return false;
- }
- if width_px == 0 || height_px == 0 {
- return true;
- }
- if input.planes() < 1 || output.planes() < 2 {
- return false;
- }
- let Some(src_stride) = positive_linesize(input, 0) else {
- return false;
- };
- let Some(y_stride) = positive_linesize(output, 0) else {
- return false;
- };
- let Some(uv_stride) = positive_linesize(output, 1) else {
- return false;
- };
- let width = width_px as usize;
- let height = height_px as usize;
- let src_h = input.plane_height(0) as usize;
- let y_h = output.plane_height(0) as usize;
- let uv_h = output.plane_height(1) as usize;
- if src_h < height || y_h < height || uv_h < height.div_ceil(2) {
- return false;
- }
- let Some(src_len) = src_stride.checked_mul(src_h) else {
- return false;
- };
- let Some(y_len) = y_stride.checked_mul(y_h) else {
- return false;
- };
- let Some(uv_len) = uv_stride.checked_mul(uv_h) else {
- return false;
- };
- unsafe {
- let in_raw = input.as_ptr();
- let out_raw = output.as_mut_ptr();
- if in_raw.is_null() || out_raw.is_null() {
- return false;
- }
- let src_ptr = (*in_raw).data[0];
- let y_ptr = (*out_raw).data[0];
- let uv_ptr = (*out_raw).data[1];
- if src_ptr.is_null() || y_ptr.is_null() || uv_ptr.is_null() {
- return false;
- }
- if !planes_disjoint(y_ptr, y_len, uv_ptr, uv_len) {
- return false;
- }
- let src = std::slice::from_raw_parts(src_ptr, src_len);
- let y = std::slice::from_raw_parts_mut(y_ptr, y_len);
- let uv = std::slice::from_raw_parts_mut(uv_ptr, uv_len);
- let mut dst = Nv12Mut {
- y,
- y_stride,
- uv,
- uv_stride,
- };
- packed_yuv::convert_packed422_to_nv12(kind, src, src_stride, width, height, &mut dst)
- }
}
impl FrameConverter for SwscaleConverter {
fn convert(&self, input: frame::Video) -> Result<frame::Video, ConvertError> {
let mut output =
- cached_output_frame(self.output_format, self.output_width, self.output_height);
+ frame::Video::new(self.output_format, self.output_width, self.output_height);
self.convert_into(input, &mut output)?;
Ok(output)
}
@@ -191,12 +77,12 @@ impl FrameConverter for SwscaleConverter {
output: &mut frame::Video,
) -> Result<(), ConvertError> {
let pts = input.pts();
- if !try_packed_yuv_nv12(&input, output) {
- let context = self.get_or_create_context()?;
- context
- .run(&input, output)
- .map_err(|e| ConvertError::ConversionFailed(e.to_string()))?;
- }
+
+ let context = self.get_or_create_context()?;
+ context
+ .run(&input, output)
+ .map_err(|e| ConvertError::ConversionFailed(e.to_string()))?;
+
output.set_pts(pts);
Ok(())
}
@@ -212,369 +98,3 @@ impl FrameConverter for SwscaleConverter {
unsafe impl Send for SwscaleConverter {}
unsafe impl Sync for SwscaleConverter {}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::packed_yuv::{Packed422, convert_packed422_scalar};
- use std::sync::Once;
- use std::time::{Duration, Instant};
-
- fn init_ffmpeg() {
- static ONCE: Once = Once::new();
- ONCE.call_once(|| {
- ffmpeg::init().expect("ffmpeg init");
- });
- }
-
- fn fill(buf: &mut [u8], mut seed: u32) {
- for byte in buf.iter_mut() {
- seed = seed.wrapping_mul(1664525).wrapping_add(1013904223);
- *byte = (seed >> 24) as u8;
- }
- }
-
- fn fill_frame(frame: &mut frame::Video, seed: u32) {
- for plane in 0..frame.planes() {
- fill(frame.data_mut(plane), seed.wrapping_add(plane as u32 * 97));
- }
- }
-
- fn active_rows(plane: &[u8], stride: usize, rows: usize, active: usize) -> Vec<u8> {
- let mut out = Vec::with_capacity(rows * active);
- for row in 0..rows {
- let start = row * stride;
- out.extend_from_slice(&plane[start..start + active]);
- }
- out
- }
-
- fn kind_of(format: Pixel) -> Packed422 {
- match format {
- Pixel::UYVY422 => Packed422::Uyvy,
- Pixel::YUYV422 => Packed422::Yuyv,
- _ => panic!("packed 4:2:2 format"),
- }
- }
-
- #[test]
- fn convert_into_matches_scalar_kernel() {
- init_ffmpeg();
- let shapes = [
- (16u32, 8u32),
- (17, 3),
- (15, 2),
- (31, 5),
- (32, 1),
- (33, 4),
- (64, 27),
- (47, 9),
- ];
- for format in [Pixel::UYVY422, Pixel::YUYV422] {
- for (width, height) in shapes {
- let config =
- ConversionConfig::new(format, width, height, Pixel::NV12, width, height);
- let converter = SwscaleConverter::new(config).expect("converter");
- let mut input = frame::Video::new(format, width, height);
- fill_frame(&mut input, width.wrapping_mul(13).wrapping_add(height));
- input.set_pts(Some(i64::from(width) * 1000 + i64::from(height)));
- let mut output = frame::Video::new(Pixel::NV12, width, height);
- converter
- .convert_into(input.clone(), &mut output)
- .expect("convert_into");
- assert_eq!(
- output.pts(),
- Some(i64::from(width) * 1000 + i64::from(height))
- );
- assert_eq!(output.format(), Pixel::NV12);
-
- let src_stride = input.stride(0);
- let y_stride = output.stride(0);
- let uv_stride = output.stride(1);
- let mut y = vec![0u8; y_stride * output.plane_height(0) as usize];
- let mut uv = vec![0u8; uv_stride * output.plane_height(1) as usize];
- {
- let mut dst = crate::packed_yuv::Nv12Mut {
- y: &mut y,
- y_stride,
- uv: &mut uv,
- uv_stride,
- };
- assert!(convert_packed422_scalar(
- kind_of(format),
- input.data(0),
- src_stride,
- width as usize,
- height as usize,
- &mut dst,
- ));
- }
- assert_eq!(
- active_rows(output.data(0), y_stride, height as usize, width as usize),
- active_rows(&y, y_stride, height as usize, width as usize),
- "{format:?} {width}x{height} y"
- );
- let chroma_bytes = (width as usize).div_ceil(2) * 2;
- assert_eq!(
- active_rows(
- output.data(1),
- uv_stride,
- height.div_ceil(2) as usize,
- chroma_bytes
- ),
- active_rows(&uv, uv_stride, height.div_ceil(2) as usize, chroma_bytes),
- "{format:?} {width}x{height} uv"
- );
- }
- }
- }
-
- // FAST_BILINEAR is not an identity luma copy and is not (a + b + 1) >> 1.
- // On even frames through 1920x1080 the max absolute difference was 20 luma
- // and 19 chroma. Odd heights use a different swscale vertical filter
- // (about 120 levels on 64x33) and stay in the exact kernel tests.
- const LUMA_TOLERANCE: i32 = 24;
- const CHROMA_TOLERANCE: i32 = 24;
-
- fn max_plane_diff(left: &[u8], right: &[u8], stride: usize, rows: usize, active: usize) -> i32 {
- let mut max_diff = 0i32;
- for row in 0..rows {
- let start = row * stride;
- for offset in 0..active {
- let delta = i32::from(left[start + offset]) - i32::from(right[start + offset]);
- max_diff = max_diff.max(delta.abs());
- }
- }
- max_diff
- }
-
- #[test]
- fn packed_path_stays_close_to_swscale() {
- init_ffmpeg();
- let shapes = [
- (64u32, 48u32),
- (320, 180),
- (640, 360),
- (1280, 720),
- (1920, 1080),
- ];
- for format in [Pixel::UYVY422, Pixel::YUYV422] {
- for (width, height) in shapes {
- let config =
- ConversionConfig::new(format, width, height, Pixel::NV12, width, height);
- let converter = SwscaleConverter::new(config).expect("converter");
- let mut input = frame::Video::new(format, width, height);
- fill_frame(
- &mut input,
- 0x51C0_u32.wrapping_add(width).wrapping_add(height),
- );
- let mut fast = frame::Video::new(Pixel::NV12, width, height);
- let mut sws = frame::Video::new(Pixel::NV12, width, height);
- assert!(try_packed_yuv_nv12(&input, &mut fast));
- converter
- .convert_swscale_into(&input, &mut sws)
- .expect("swscale");
- let luma = max_plane_diff(
- fast.data(0),
- sws.data(0),
- fast.stride(0),
- height as usize,
- width as usize,
- );
- let chroma_bytes = (width as usize / 2) * 2;
- let chroma = max_plane_diff(
- fast.data(1),
- sws.data(1),
- fast.stride(1),
- height.div_ceil(2) as usize,
- chroma_bytes,
- );
- assert!(
- luma <= LUMA_TOLERANCE && chroma <= CHROMA_TOLERANCE,
- "{format:?} {width}x{height} luma {luma} chroma {chroma}"
- );
- }
- }
- }
-
- #[test]
- fn other_formats_and_scaling_still_use_swscale() {
- init_ffmpeg();
- let scaled = SwscaleConverter::new(ConversionConfig::new(
- Pixel::UYVY422,
- 32,
- 16,
- Pixel::NV12,
- 16,
- 8,
- ))
- .expect("scaled");
- let mut input = frame::Video::new(Pixel::UYVY422, 32, 16);
- fill_frame(&mut input, 9);
- let output = scaled.convert(input).expect("scale");
- assert_eq!(
- (output.width(), output.height(), output.format()),
- (16, 8, Pixel::NV12)
- );
-
- let bgra = SwscaleConverter::new(ConversionConfig::new(
- Pixel::BGRA,
- 16,
- 16,
- Pixel::NV12,
- 16,
- 16,
- ))
- .expect("bgra");
- let mut input = frame::Video::new(Pixel::BGRA, 16, 16);
- fill_frame(&mut input, 4);
- let output = bgra.convert(input).expect("bgra convert");
- assert_eq!(output.format(), Pixel::NV12);
- assert_eq!((output.width(), output.height()), (16, 16));
- }
-
- #[test]
- fn convert_keeps_live_frames_independent() {
- init_ffmpeg();
- let converter = SwscaleConverter::new(ConversionConfig::new(
- Pixel::UYVY422,
- 82,
- 18,
- Pixel::NV12,
- 82,
- 18,
- ))
- .expect("converter");
- let mut first_src = frame::Video::new(Pixel::UYVY422, 82, 18);
- first_src.data_mut(0).fill(30);
- first_src.set_pts(Some(42));
- let mut first = converter.convert(first_src).expect("first");
- assert_eq!(first.pts(), Some(42));
- let first_y = first.data(0).to_vec();
- let first_uv = first.data(1).to_vec();
- unsafe {
- assert_eq!(ffmpeg::ffi::av_frame_is_writable(first.as_mut_ptr()), 1);
- }
- let mut second_src = frame::Video::new(Pixel::UYVY422, 82, 18);
- second_src.data_mut(0).fill(90);
- let mut second = converter.convert(second_src).expect("second");
- assert_eq!(first.data(0), first_y.as_slice());
- assert_eq!(first.data(1), first_uv.as_slice());
- assert!(second.data(0).contains(&90));
- assert!(!second.data(0).iter().take(82).all(|byte| *byte == 30));
- unsafe {
- assert_eq!(ffmpeg::ffi::av_frame_is_writable(first.as_mut_ptr()), 1);
- assert_eq!(ffmpeg::ffi::av_frame_is_writable(second.as_mut_ptr()), 1);
- }
- drop(first);
- drop(second);
- let mut third_src = frame::Video::new(Pixel::UYVY422, 82, 18);
- third_src.data_mut(0).fill(11);
- let third = converter.convert(third_src).expect("third");
- assert!(third.data(0).iter().take(82).all(|byte| *byte == 11));
- }
-
- #[inline(never)]
- fn time_swscale(converter: &SwscaleConverter, input: &frame::Video, output: &mut frame::Video) {
- converter
- .convert_swscale_into(input, output)
- .expect("swscale");
- let index = output.stride(0);
- std::hint::black_box(output.data(0)[index]);
- }
-
- #[inline(never)]
- fn time_packed(input: &frame::Video, output: &mut frame::Video) {
- assert!(try_packed_yuv_nv12(input, output));
- let index = output.stride(0);
- std::hint::black_box(output.data(0)[index]);
- }
-
- fn median(samples: &mut [Duration]) -> Duration {
- samples.sort_unstable();
- samples[samples.len() / 2]
- }
-
- fn cpu_model() -> String {
- let Ok(text) = std::fs::read_to_string("/proc/cpuinfo") else {
- return "unknown".to_string();
- };
- for line in text.lines() {
- let Some(rest) = line.strip_prefix("model name") else {
- continue;
- };
- return rest.trim_start_matches([' ', ':', '\t']).to_string();
- }
- "unknown".to_string()
- }
-
- #[test]
- #[ignore]
- fn bench_packed_yuv_nv12() {
- init_ffmpeg();
- if cfg!(debug_assertions) {
- eprintln!("debug build; rerun with --release for representative medians");
- }
- #[cfg(target_arch = "x86_64")]
- let avx2 = is_x86_feature_detected!("avx2");
- #[cfg(not(target_arch = "x86_64"))]
- let avx2 = false;
- let warmup = 16;
- let runs = 51;
- println!("cpu: {}", cpu_model());
- println!("avx2: {avx2}");
- println!("warmup: {warmup}");
- println!("timed runs: {runs}");
- println!(
- "before: swscale FAST_BILINEAR, after: packed kernel, frames allocated outside the timer"
- );
- let shapes = [(1280u32, 720u32), (1920, 1080), (3840, 2160)];
- for format in [Pixel::UYVY422, Pixel::YUYV422] {
- for (width, height) in shapes {
- let config =
- ConversionConfig::new(format, width, height, Pixel::NV12, width, height);
- let converter = SwscaleConverter::new(config).expect("converter");
- let mut input = frame::Video::new(format, width, height);
- fill_frame(&mut input, width.wrapping_mul(height));
- let mut sws_out = frame::Video::new(Pixel::NV12, width, height);
- let mut fast_out = frame::Video::new(Pixel::NV12, width, height);
- for _ in 0..warmup {
- time_swscale(&converter, &input, &mut sws_out);
- }
- let mut sws_samples = Vec::with_capacity(runs);
- for _ in 0..runs {
- let start = Instant::now();
- time_swscale(&converter, &input, &mut sws_out);
- sws_samples.push(start.elapsed());
- }
- for _ in 0..warmup {
- time_packed(&input, &mut fast_out);
- }
- let mut fast_samples = Vec::with_capacity(runs);
- for _ in 0..runs {
- let start = Instant::now();
- time_packed(&input, &mut fast_out);
- fast_samples.push(start.elapsed());
- }
- let sws_min = sws_samples.iter().copied().min().unwrap_or_default();
- let fast_min = fast_samples.iter().copied().min().unwrap_or_default();
- let sws_max = sws_samples.iter().copied().max().unwrap_or_default();
- let fast_max = fast_samples.iter().copied().max().unwrap_or_default();
- let sws_med = median(&mut sws_samples);
- let fast_med = median(&mut fast_samples);
- let speedup = sws_med.as_secs_f64() / fast_med.as_secs_f64();
- println!(
- "{width}x{height} {format:?} before {:.3} ms after {:.3} ms speedup {speedup:.2}x n {runs} before_min {:.3} before_max {:.3} after_min {:.3} after_max {:.3}",
- sws_med.as_secs_f64() * 1000.0,
- fast_med.as_secs_f64() * 1000.0,
- sws_min.as_secs_f64() * 1000.0,
- sws_max.as_secs_f64() * 1000.0,
- fast_min.as_secs_f64() * 1000.0,
- fast_max.as_secs_f64() * 1000.0,
- );
- let _ = std::hint::black_box(sws_out.data(0)[0]);
- let _ = std::hint::black_box(fast_out.data(0)[0]);
- }
- }
- }
-}