Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(recording): add aarch64 NEON paths for screenshot pixel packers
shubhookfeee25f967b5
diff --git a/crates/recording/examples/screenshot-pack-benchmark.rs b/crates/recording/examples/screenshot-pack-benchmark.rsnew file mode 100644index 000000000..d9148ff7c--- /dev/null+++ b/crates/recording/examples/screenshot-pack-benchmark.rs@@ -0,0 +1,515 @@+//! Compares the macOS fast-capture pixel packers in `src/screenshot.rs` before and after the+//! aarch64 NEON paths: BGRA to RGB (display and area captures) and BGRA to RGBA (window+//! captures). The old and new code live in this one file, so both sides share compiler flags.+//! Every timed call allocates its output buffer, as the production functions do.+//!+//! The file uses only `std`, so it builds without the crate's native dependencies:+//!+//! rustc +1.88.0 --edition 2024 -C opt-level=s -C lto=thin -C codegen-units=1 \+//! -C debuginfo=line-tables-only -C panic=unwind \+//! crates/recording/examples/screenshot-pack-benchmark.rs -o /tmp/screenshot-pack-benchmark+//! /tmp/screenshot-pack-benchmark+//!+//! or, where the crate builds: `cargo run --release -p cap-recording --example screenshot-pack-benchmark`.++#[cfg(not(target_arch = "aarch64"))]+fn main() {+ eprintln!("screenshot-pack-benchmark: skipped (not aarch64)");+}++#[cfg(target_arch = "aarch64")]+fn main() {+ bench::run();+}++#[cfg(target_arch = "aarch64")]+#[derive(Clone, Copy, Debug)]+enum ChannelOrder {+ Rgba,+ Bgra,+}++#[cfg(target_arch = "aarch64")]+mod old {+ use super::ChannelOrder;++ pub fn rgb_from_rgba(+ data: &[u8],+ width: usize,+ height: usize,+ bytes_per_row: usize,+ order: ChannelOrder,+ ) -> Option<Vec<u8>> {+ let row_bytes = width.checked_mul(4)?;+ if bytes_per_row < row_bytes {+ return None;+ }++ let required_len = height.checked_mul(bytes_per_row)?;+ if data.len() < required_len {+ return None;+ }++ let width_stride = width.checked_mul(3)?;+ let rgb_len = height.checked_mul(width_stride)?;+ let mut rgb = vec![0u8; rgb_len];++ for y in 0..height {+ let (src_row, dst_row) =+ rgb_row_pair(data, &mut rgb, y, bytes_per_row, row_bytes, width_stride)?;+ reorder_rgb_row_scalar(src_row, dst_row, order);+ }++ Some(rgb)+ }++ fn rgb_row_pair<'a>(+ data: &'a [u8],+ rgb: &'a mut [u8],+ y: usize,+ bytes_per_row: usize,+ row_bytes: usize,+ width_stride: usize,+ ) -> Option<(&'a [u8], &'a mut [u8])> {+ let src_start = y.checked_mul(bytes_per_row)?;+ let src_end = src_start.checked_add(row_bytes)?;+ let dst_start = y.checked_mul(width_stride)?;+ let dst_end = dst_start.checked_add(width_stride)?;+ let src_row = data.get(src_start..src_end)?;+ let dst_row = rgb.get_mut(dst_start..dst_end)?;+ Some((src_row, dst_row))+ }++ fn reorder_rgb_row_scalar(src: &[u8], dst: &mut [u8], order: ChannelOrder) {+ match order {+ ChannelOrder::Rgba => rgba_to_rgb_scalar(src, dst),+ ChannelOrder::Bgra => bgra_to_rgb_scalar(src, dst),+ }+ }++ fn rgba_to_rgb_scalar(src: &[u8], dst: &mut [u8]) {+ for (src, dst) in src.chunks_exact(4).zip(dst.chunks_exact_mut(3)) {+ dst[0] = src[0];+ dst[1] = src[1];+ dst[2] = src[2];+ }+ }++ fn bgra_to_rgb_scalar(src: &[u8], dst: &mut [u8]) {+ for (src, dst) in src.chunks_exact(4).zip(dst.chunks_exact_mut(3)) {+ dst[0] = src[2];+ dst[1] = src[1];+ dst[2] = src[0];+ }+ }++ pub fn rgba_from_raw(+ data: &[u8],+ width: usize,+ height: usize,+ bytes_per_row: usize,+ order: ChannelOrder,+ ) -> Option<Vec<u8>> {+ let row_bytes = width.checked_mul(4)?;+ if bytes_per_row < row_bytes {+ return None;+ }++ let required_len = height.checked_mul(bytes_per_row)?;+ if data.len() < required_len {+ return None;+ }++ let width_stride = width.checked_mul(4)?;+ let rgba_len = height.checked_mul(width_stride)?;+ let mut rgba = vec![0u8; rgba_len];++ for y in 0..height {+ let src_start = y.checked_mul(bytes_per_row)?;+ let src_end = src_start.checked_add(row_bytes)?;+ let dst_start = y.checked_mul(width_stride)?;+ let dst_end = dst_start.checked_add(width_stride)?;++ let src_row = data.get(src_start..src_end)?;+ let dst_row = rgba.get_mut(dst_start..dst_end)?;++ for (src, dst) in src_row.chunks_exact(4).zip(dst_row.chunks_exact_mut(4)) {+ let (r, b) = match order {+ ChannelOrder::Rgba => (src[0], src[2]),+ ChannelOrder::Bgra => (src[2], src[0]),+ };++ dst[0] = r;+ dst[1] = src[1];+ dst[2] = b;+ dst[3] = src[3];+ }+ }++ Some(rgba)+ }+}++#[cfg(target_arch = "aarch64")]+mod new {+ use super::ChannelOrder;++ pub fn rgb_from_rgba(+ data: &[u8],+ width: usize,+ height: usize,+ bytes_per_row: usize,+ order: ChannelOrder,+ ) -> Option<Vec<u8>> {+ let row_bytes = width.checked_mul(4)?;+ if bytes_per_row < row_bytes {+ return None;+ }+ let required_len = height.checked_mul(bytes_per_row)?;+ if data.len() < required_len {+ return None;+ }+ let width_stride = width.checked_mul(3)?;+ // SAFETY: reorder_rgb_row_neon initializes every element of each destination row.+ unsafe {+ pack_rows_uninit(+ data,+ height,+ bytes_per_row,+ row_bytes,+ width_stride,+ |src_row, dst_row| reorder_rgb_row_neon(src_row, dst_row, order),+ )+ }+ }++ pub fn rgba_from_raw(+ data: &[u8],+ width: usize,+ height: usize,+ bytes_per_row: usize,+ order: ChannelOrder,+ ) -> Option<Vec<u8>> {+ let row_bytes = width.checked_mul(4)?;+ if bytes_per_row < row_bytes {+ return None;+ }+ let required_len = height.checked_mul(bytes_per_row)?;+ if data.len() < required_len {+ return None;+ }+ let width_stride = width.checked_mul(4)?;+ // SAFETY: reorder_rgba_row_neon initializes every element of each destination row.+ unsafe {+ pack_rows_uninit(+ data,+ height,+ bytes_per_row,+ row_bytes,+ width_stride,+ |src_row, dst_row| reorder_rgba_row_neon(src_row, dst_row, order),+ )+ }+ }++ pub fn rgb_from_rgba_zeroed(+ data: &[u8],+ width: usize,+ height: usize,+ bytes_per_row: usize,+ order: ChannelOrder,+ ) -> Option<Vec<u8>> {+ let row_bytes = width.checked_mul(4)?;+ let width_stride = width.checked_mul(3)?;+ let mut rgb = vec![0u8; height.checked_mul(width_stride)?];+ for y in 0..height {+ let src_row = data.get(y * bytes_per_row..y * bytes_per_row + row_bytes)?;+ let dst_row = rgb.get_mut(y * width_stride..(y + 1) * width_stride)?;+ reorder_rgb_row_neon(src_row, as_uninit(dst_row), order);+ }+ Some(rgb)+ }++ pub fn rgba_from_raw_zeroed(+ data: &[u8],+ width: usize,+ height: usize,+ bytes_per_row: usize,+ order: ChannelOrder,+ ) -> Option<Vec<u8>> {+ let row_bytes = width.checked_mul(4)?;+ let mut rgba = vec![0u8; height.checked_mul(row_bytes)?];+ for y in 0..height {+ let src_row = data.get(y * bytes_per_row..y * bytes_per_row + row_bytes)?;+ let dst_row = rgba.get_mut(y * row_bytes..(y + 1) * row_bytes)?;+ reorder_rgba_row_neon(src_row, as_uninit(dst_row), order);+ }+ Some(rgba)+ }++ fn as_uninit(bytes: &mut [u8]) -> &mut [std::mem::MaybeUninit<u8>] {+ // SAFETY: MaybeUninit<u8> has the layout of u8, and the kernels only store initialized+ // bytes, so the slice stays initialized.+ unsafe { &mut *(bytes as *mut [u8] as *mut [std::mem::MaybeUninit<u8>]) }+ }++ /// # Safety+ ///+ /// `write_row` must initialize every element of the destination slice it receives.+ unsafe fn pack_rows_uninit(+ data: &[u8],+ height: usize,+ bytes_per_row: usize,+ row_bytes: usize,+ dst_stride: usize,+ mut write_row: impl FnMut(&[u8], &mut [std::mem::MaybeUninit<u8>]),+ ) -> Option<Vec<u8>> {+ let dst_len = height.checked_mul(dst_stride)?;+ let mut out = Vec::<u8>::with_capacity(dst_len);+ let spare = out.spare_capacity_mut().get_mut(..dst_len)?;++ for y in 0..height {+ let src_start = y.checked_mul(bytes_per_row)?;+ let src_row = data.get(src_start..src_start.checked_add(row_bytes)?)?;+ let dst_start = y.checked_mul(dst_stride)?;+ let dst_row = spare.get_mut(dst_start..dst_start.checked_add(dst_stride)?)?;+ write_row(src_row, dst_row);+ }++ // SAFETY: the rows 0..height tile 0..dst_len exactly, and the caller guarantees write_row+ // initialized every byte of each row. dst_len <= capacity.+ unsafe { out.set_len(dst_len) };+ Some(out)+ }++ fn reorder_rgb_row_neon(+ src: &[u8],+ dst: &mut [std::mem::MaybeUninit<u8>],+ order: ChannelOrder,+ ) {+ use std::arch::aarch64::*;++ assert_eq!(+ dst.len() % 3,+ 0,+ "rgb row length must be a whole number of pixels"+ );+ let pixels = dst.len() / 3;+ let src = &src[..pixels * 4];+ let swap = matches!(order, ChannelOrder::Bgra);+ let simd_pixels = pixels & !15;++ for i in (0..simd_pixels).step_by(16) {+ // SAFETY: i + 16 <= simd_pixels <= pixels, so the 64 byte load at i * 4 is inside src+ // (pixels * 4 bytes) and the 48 byte store at i * 3 is inside dst (pixels * 3 bytes).+ // NEON is part of the aarch64 baseline, and both intrinsics accept unaligned pointers.+ unsafe {+ let px = vld4q_u8(src.as_ptr().add(i * 4));+ let rgb = if swap {+ uint8x16x3_t(px.2, px.1, px.0)+ } else {+ uint8x16x3_t(px.0, px.1, px.2)+ };+ vst3q_u8(dst.as_mut_ptr().add(i * 3).cast::<u8>(), rgb);+ }+ }++ for (s, d) in src[simd_pixels * 4..]+ .chunks_exact(4)+ .zip(dst[simd_pixels * 3..].chunks_exact_mut(3))+ {+ let (r, b) = if swap { (s[2], s[0]) } else { (s[0], s[2]) };+ d[0].write(r);+ d[1].write(s[1]);+ d[2].write(b);+ }+ }++ fn reorder_rgba_row_neon(+ src: &[u8],+ dst: &mut [std::mem::MaybeUninit<u8>],+ order: ChannelOrder,+ ) {+ use std::arch::aarch64::*;++ assert_eq!(+ dst.len() % 4,+ 0,+ "rgba row length must be a whole number of pixels"+ );+ let pixels = dst.len() / 4;+ let src = &src[..pixels * 4];+ let swap = matches!(order, ChannelOrder::Bgra);+ let simd_pixels = pixels & !15;++ for i in (0..simd_pixels).step_by(16) {+ // SAFETY: i + 16 <= simd_pixels <= pixels, so the 64 byte load and the 64 byte store at+ // i * 4 are inside src and dst (both pixels * 4 bytes). NEON is part of the aarch64+ // baseline, and both intrinsics accept unaligned pointers.+ unsafe {+ let px = vld4q_u8(src.as_ptr().add(i * 4));+ let rgba = if swap {+ uint8x16x4_t(px.2, px.1, px.0, px.3)+ } else {+ px+ };+ vst4q_u8(dst.as_mut_ptr().add(i * 4).cast::<u8>(), rgba);+ }+ }++ for (s, d) in src[simd_pixels * 4..]+ .chunks_exact(4)+ .zip(dst[simd_pixels * 4..].chunks_exact_mut(4))+ {+ let (r, b) = if swap { (s[2], s[0]) } else { (s[0], s[2]) };+ d[0].write(r);+ d[1].write(s[1]);+ d[2].write(b);+ d[3].write(s[3]);+ }+ }+}++#[cfg(target_arch = "aarch64")]+mod bench {+ use super::{ChannelOrder, new, old};+ use std::hint::black_box;+ use std::time::{Duration, Instant};++ const SIZES: [(usize, usize); 3] = [(1920, 1080), (2880, 1800), (5120, 2880)];+ const STRIDE_PAD: usize = 64;+ const WARMUP: usize = 2;+ const RUNS: usize = 9;++ type Packer = fn(&[u8], usize, usize, usize, ChannelOrder) -> Option<Vec<u8>>;++ fn paint(len: usize, seed: u32) -> Vec<u8> {+ let mut state = seed;+ let mut out = vec![0u8; len];+ for byte in &mut out {+ state = state.wrapping_mul(1664525).wrapping_add(1013904223);+ *byte = (state >> 16) as u8;+ }+ out+ }++ fn time_once(packer: Packer, data: &[u8], w: usize, h: usize, stride: usize) -> Duration {+ let start = Instant::now();+ let out = packer(+ black_box(data),+ black_box(w),+ black_box(h),+ black_box(stride),+ black_box(ChannelOrder::Bgra),+ );+ let elapsed = start.elapsed();+ black_box(out.expect("packer rejected a valid frame"));+ elapsed+ }++ fn median_ms(mut samples: Vec<Duration>) -> f64 {+ samples.sort();+ samples[samples.len() / 2].as_secs_f64() * 1000.0+ }++ fn command_line(program: &str, args: &[&str]) -> String {+ std::process::Command::new(program)+ .args(args)+ .output()+ .ok()+ .map(|out| {+ String::from_utf8_lossy(&out.stdout)+ .trim()+ .replace('\n', " ")+ })+ .unwrap_or_else(|| "unknown".to_string())+ }++ fn verify(data: &[u8], w: usize, h: usize, stride: usize) -> bool {+ let mut ok = true;+ for order in [ChannelOrder::Bgra, ChannelOrder::Rgba] {+ let pairs: [(&str, Packer, Packer); 4] = [+ ("rgb", old::rgb_from_rgba, new::rgb_from_rgba),+ ("rgb zeroed", old::rgb_from_rgba, new::rgb_from_rgba_zeroed),+ ("rgba", old::rgba_from_raw, new::rgba_from_raw),+ ("rgba zeroed", old::rgba_from_raw, new::rgba_from_raw_zeroed),+ ];+ for (label, before, after) in pairs {+ let expected = before(data, w, h, stride, order);+ let got = after(data, w, h, stride, order);+ if expected.is_none() || expected != got {+ eprintln!("MISMATCH {label} {w}x{h} stride {stride} {order:?}");+ ok = false;+ }+ }+ }+ ok+ }++ pub fn run() {+ println!(+ "machine: {} | {}",+ command_line("sysctl", &["-n", "machdep.cpu.brand_string", "hw.memsize"]),+ command_line("sw_vers", &["-productVersion"]),+ );+ println!(+ "build: {} | {}",+ option_env!("RUSTC_VERSION").unwrap_or("rustc version not recorded"),+ option_env!("OPT_LABEL").unwrap_or("flags not recorded"),+ );+ println!(+ "input: BGRA, bytes_per_row = width * 4 + {STRIDE_PAD}, warmup {WARMUP}, median of {RUNS} single-frame calls"+ );+ println!();+ println!(+ "| Frame | BGRA to RGB before | NEON, zeroed buffer | NEON, uninit buffer | Speedup | BGRA to RGBA before | NEON, zeroed buffer | NEON, uninit buffer | Speedup |"+ );+ println!("| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |");++ let mut all_ok = true;+ for (w, h) in SIZES {+ let stride = w * 4 + STRIDE_PAD;+ let data = paint(stride * h, (w as u32) ^ (h as u32).rotate_left(16));++ if !verify(&data, w, h, stride) {+ all_ok = false;+ continue;+ }++ let packers: [Packer; 6] = [+ old::rgb_from_rgba,+ new::rgb_from_rgba_zeroed,+ new::rgb_from_rgba,+ old::rgba_from_raw,+ new::rgba_from_raw_zeroed,+ new::rgba_from_raw,+ ];+ let mut samples: [Vec<Duration>; 6] = Default::default();+ for run in 0..WARMUP + RUNS {+ for (packer, bucket) in packers.iter().zip(samples.iter_mut()) {+ let elapsed = time_once(*packer, &data, w, h, stride);+ if run >= WARMUP {+ bucket.push(elapsed);+ }+ }+ }+ let ms = samples.map(median_ms);+ println!(+ "| {w}x{h} | {:.3} | {:.3} | {:.3} | {:.2}x | {:.3} | {:.3} | {:.3} | {:.2}x |",+ ms[0],+ ms[1],+ ms[2],+ ms[0] / ms[2],+ ms[3],+ ms[4],+ ms[5],+ ms[3] / ms[5],+ );+ }++ if !all_ok {+ eprintln!("outputs differ, failing");+ std::process::exit(1);+ }+ }+}diff --git a/crates/recording/src/screenshot.rs b/crates/recording/src/screenshot.rsindex 537c4fdb7..4551794eb 100644--- a/crates/recording/src/screenshot.rs+++ b/crates/recording/src/screenshot.rs@@ -80,33 +80,173 @@ fn rgb_from_rgba(}let width_stride = width.checked_mul(3)?;- let rgb_len = height.checked_mul(width_stride)?;- let mut rgb = vec![0u8; rgb_len];- #[cfg(target_arch = "x86_64")]+ #[cfg(target_arch = "aarch64")]{- if std::is_x86_feature_detected!("ssse3") {- for y in 0..height {- let (src_row, dst_row) =- rgb_row_pair(data, &mut rgb, y, bytes_per_row, row_bytes, width_stride)?;- unsafe {- reorder_rgb_row_ssse3(src_row, dst_row, order);+ // SAFETY: reorder_rgb_row_neon initializes every element of each destination row.+ let rgb = unsafe {+ pack_rows_uninit(+ data,+ height,+ bytes_per_row,+ row_bytes,+ width_stride,+ |src_row, dst_row| reorder_rgb_row_neon(src_row, dst_row, order),+ )?+ };+ RgbImage::from_raw(width as u32, height as u32, rgb)+ }++ #[cfg(not(target_arch = "aarch64"))]+ {+ let rgb_len = height.checked_mul(width_stride)?;+ let mut rgb = vec![0u8; rgb_len];++ #[cfg(target_arch = "x86_64")]+ {+ if std::is_x86_feature_detected!("ssse3") {+ for y in 0..height {+ let (src_row, dst_row) =+ rgb_row_pair(data, &mut rgb, y, bytes_per_row, row_bytes, width_stride)?;+ unsafe {+ reorder_rgb_row_ssse3(src_row, dst_row, order);+ }}+ return RgbImage::from_raw(width as u32, height as u32, rgb);}- return RgbImage::from_raw(width as u32, height as u32, rgb);}++ for y in 0..height {+ let (src_row, dst_row) =+ rgb_row_pair(data, &mut rgb, y, bytes_per_row, row_bytes, width_stride)?;+ reorder_rgb_row_scalar(src_row, dst_row, order);+ }++ RgbImage::from_raw(width as u32, height as u32, rgb)}+}++/// # Safety+///+/// `write_row` must initialize every element of the destination slice it receives.+#[cfg(all(target_arch = "aarch64", any(not(target_os = "linux"), test)))]+unsafe fn pack_rows_uninit(+ data: &[u8],+ height: usize,+ bytes_per_row: usize,+ row_bytes: usize,+ dst_stride: usize,+ mut write_row: impl FnMut(&[u8], &mut [std::mem::MaybeUninit<u8>]),+) -> Option<Vec<u8>> {+ let dst_len = height.checked_mul(dst_stride)?;+ let mut out = Vec::<u8>::with_capacity(dst_len);+ let spare = out.spare_capacity_mut().get_mut(..dst_len)?;for y in 0..height {- let (src_row, dst_row) =- rgb_row_pair(data, &mut rgb, y, bytes_per_row, row_bytes, width_stride)?;- reorder_rgb_row_scalar(src_row, dst_row, order);+ let src_start = y.checked_mul(bytes_per_row)?;+ let src_row = data.get(src_start..src_start.checked_add(row_bytes)?)?;+ let dst_start = y.checked_mul(dst_stride)?;+ let dst_row = spare.get_mut(dst_start..dst_start.checked_add(dst_stride)?)?;+ write_row(src_row, dst_row);}- RgbImage::from_raw(width as u32, height as u32, rgb)+ // SAFETY: the rows 0..height tile 0..dst_len exactly, and the caller guarantees write_row+ // initialized every byte of each row. dst_len <= capacity.+ unsafe { out.set_len(dst_len) };+ Some(out)}-#[cfg(any(target_os = "macos", target_os = "windows", test))]+#[cfg(all(+ any(target_os = "macos", target_os = "windows", test),+ target_arch = "aarch64"+))]+fn reorder_rgb_row_neon(src: &[u8], dst: &mut [std::mem::MaybeUninit<u8>], order: ChannelOrder) {+ use std::arch::aarch64::*;++ assert_eq!(+ dst.len() % 3,+ 0,+ "rgb row length must be a whole number of pixels"+ );+ let pixels = dst.len() / 3;+ let src = &src[..pixels * 4];+ let swap = matches!(order, ChannelOrder::Bgra);+ let simd_pixels = pixels & !15;++ for i in (0..simd_pixels).step_by(16) {+ // SAFETY: i + 16 <= simd_pixels <= pixels, so the 64 byte load at i * 4 is inside src+ // (pixels * 4 bytes) and the 48 byte store at i * 3 is inside dst (pixels * 3 bytes).+ // NEON is part of the aarch64 baseline, and both intrinsics accept unaligned pointers.+ unsafe {+ let px = vld4q_u8(src.as_ptr().add(i * 4));+ let rgb = if swap {+ uint8x16x3_t(px.2, px.1, px.0)+ } else {+ uint8x16x3_t(px.0, px.1, px.2)+ };+ vst3q_u8(dst.as_mut_ptr().add(i * 3).cast::<u8>(), rgb);+ }+ }++ for (s, d) in src[simd_pixels * 4..]+ .chunks_exact(4)+ .zip(dst[simd_pixels * 3..].chunks_exact_mut(3))+ {+ let (r, b) = if swap { (s[2], s[0]) } else { (s[0], s[2]) };+ d[0].write(r);+ d[1].write(s[1]);+ d[2].write(b);+ }+}++#[cfg(all(not(target_os = "linux"), target_arch = "aarch64"))]+fn reorder_rgba_row_neon(src: &[u8], dst: &mut [std::mem::MaybeUninit<u8>], order: ChannelOrder) {+ use std::arch::aarch64::*;++ assert_eq!(+ dst.len() % 4,+ 0,+ "rgba row length must be a whole number of pixels"+ );+ let pixels = dst.len() / 4;+ let src = &src[..pixels * 4];+ let swap = matches!(order, ChannelOrder::Bgra);+ let simd_pixels = pixels & !15;++ for i in (0..simd_pixels).step_by(16) {+ // SAFETY: i + 16 <= simd_pixels <= pixels, so the 64 byte load and the 64 byte store at+ // i * 4 are inside src and dst (both pixels * 4 bytes). NEON is part of the aarch64+ // baseline, and both intrinsics accept unaligned pointers.+ unsafe {+ let px = vld4q_u8(src.as_ptr().add(i * 4));+ let rgba = if swap {+ uint8x16x4_t(px.2, px.1, px.0, px.3)+ } else {+ px+ };+ vst4q_u8(dst.as_mut_ptr().add(i * 4).cast::<u8>(), rgba);+ }+ }++ for (s, d) in src[simd_pixels * 4..]+ .chunks_exact(4)+ .zip(dst[simd_pixels * 4..].chunks_exact_mut(4))+ {+ let (r, b) = if swap { (s[2], s[0]) } else { (s[0], s[2]) };+ d[0].write(r);+ d[1].write(s[1]);+ d[2].write(b);+ d[3].write(s[3]);+ }+}++#[cfg(any(+ all(+ any(target_os = "macos", target_os = "windows"),+ not(target_arch = "aarch64")+ ),+ test+))]fn rgb_row_pair<'a>(data: &'a [u8],rgb: &'a mut [u8],@@ -124,7 +264,13 @@ fn rgb_row_pair<'a>(Some((src_row, dst_row))}-#[cfg(any(target_os = "macos", target_os = "windows", test))]+#[cfg(any(+ all(+ any(target_os = "macos", target_os = "windows"),+ not(target_arch = "aarch64")+ ),+ test+))]fn reorder_rgb_row_scalar(src: &[u8], dst: &mut [u8], order: ChannelOrder) {match order {ChannelOrder::Rgba => rgba_to_rgb_scalar(src, dst),@@ -132,7 +278,13 @@ fn reorder_rgb_row_scalar(src: &[u8], dst: &mut [u8], order: ChannelOrder) {}}-#[cfg(any(target_os = "macos", target_os = "windows", test))]+#[cfg(any(+ all(+ any(target_os = "macos", target_os = "windows"),+ not(target_arch = "aarch64")+ ),+ test+))]fn rgba_to_rgb_scalar(src: &[u8], dst: &mut [u8]) {for (src, dst) in src.chunks_exact(4).zip(dst.chunks_exact_mut(3)) {dst[0] = src[0];@@ -141,7 +293,13 @@ fn rgba_to_rgb_scalar(src: &[u8], dst: &mut [u8]) {}}-#[cfg(any(target_os = "macos", target_os = "windows", test))]+#[cfg(any(+ all(+ any(target_os = "macos", target_os = "windows"),+ not(target_arch = "aarch64")+ ),+ test+))]fn bgra_to_rgb_scalar(src: &[u8], dst: &mut [u8]) {for (src, dst) in src.chunks_exact(4).zip(dst.chunks_exact_mut(3)) {dst[0] = src[2];@@ -203,32 +361,52 @@ fn rgba_from_raw(}let width_stride = width.checked_mul(4)?;- let rgba_len = height.checked_mul(width_stride)?;- let mut rgba = vec![0u8; rgba_len];- for y in 0..height {- let src_start = y.checked_mul(bytes_per_row)?;- let src_end = src_start.checked_add(row_bytes)?;- let dst_start = y.checked_mul(width_stride)?;- let dst_end = dst_start.checked_add(width_stride)?;+ #[cfg(target_arch = "aarch64")]+ {+ // SAFETY: reorder_rgba_row_neon initializes every element of each destination row.+ let rgba = unsafe {+ pack_rows_uninit(+ data,+ height,+ bytes_per_row,+ row_bytes,+ width_stride,+ |src_row, dst_row| reorder_rgba_row_neon(src_row, dst_row, order),+ )?+ };+ RgbaImage::from_raw(width as u32, height as u32, rgba)+ }- let src_row = data.get(src_start..src_end)?;- let dst_row = rgba.get_mut(dst_start..dst_end)?;+ #[cfg(not(target_arch = "aarch64"))]+ {+ let rgba_len = height.checked_mul(width_stride)?;+ let mut rgba = vec![0u8; rgba_len];- for (src, dst) in src_row.chunks_exact(4).zip(dst_row.chunks_exact_mut(4)) {- let (r, b) = match order {- ChannelOrder::Rgba => (src[0], src[2]),- ChannelOrder::Bgra => (src[2], src[0]),- };+ for y in 0..height {+ let src_start = y.checked_mul(bytes_per_row)?;+ let src_end = src_start.checked_add(row_bytes)?;+ let dst_start = y.checked_mul(width_stride)?;+ let dst_end = dst_start.checked_add(width_stride)?;- dst[0] = r;- dst[1] = src[1];- dst[2] = b;- dst[3] = src[3];+ let src_row = data.get(src_start..src_end)?;+ let dst_row = rgba.get_mut(dst_start..dst_end)?;++ for (src, dst) in src_row.chunks_exact(4).zip(dst_row.chunks_exact_mut(4)) {+ let (r, b) = match order {+ ChannelOrder::Rgba => (src[0], src[2]),+ ChannelOrder::Bgra => (src[2], src[0]),+ };++ dst[0] = r;+ dst[1] = src[1];+ dst[2] = b;+ dst[3] = src[3];+ }}- }- RgbaImage::from_raw(width as u32, height as u32, rgba)+ RgbaImage::from_raw(width as u32, height as u32, rgba)+ }}fn apply_window_rounded_corners(image: RgbImage, target: &ScreenCaptureTarget) -> RgbaImage {@@ -1933,7 +2111,8 @@ mod screenshot_pixel_tests {fn rgb_cases() -> Vec<(usize, usize, usize)> {let mut cases = Vec::new();let widths = [- 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 16, 17, 31, 63, 64, 65, 191, 1920, 1921,+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 16, 17, 31, 32, 33, 47, 48, 49, 63, 64, 65, 191,+ 1920, 1921, 2879,];let heights = [0, 1, 2, 3, 5, 7];let pads = [0usize, 1, 7, 32];@@ -2066,6 +2245,91 @@ mod screenshot_pixel_tests {assert_eq!(image.as_raw(), &[30, 20, 10]);}+ #[cfg(target_arch = "aarch64")]+ #[test]+ fn rgb_neon_matches_scalar_including_tail() {+ for (width, height, pad) in rgb_cases() {+ let stride = width * 4 + pad;+ let data = paint(+ height * stride + 8,+ (width as u32).wrapping_mul(31).wrapping_add(pad as u32),+ );+ for order in [ChannelOrder::Rgba, ChannelOrder::Bgra] {+ let label = format!("{width}x{height} stride {stride} {order:?}");+ let expected = scalar_rgb(&data, width, height, stride, order);+ let got = rgb_from_rgba(&data, width, height, stride, order)+ .map(|image| image.into_raw());+ assert_rgb_path_matches_original("neon", got, &expected, &label);+ }+ }+ }++ #[cfg(not(target_os = "linux"))]+ fn reference_rgba(+ data: &[u8],+ width: usize,+ height: usize,+ bytes_per_row: usize,+ order: ChannelOrder,+ ) -> Option<Vec<u8>> {+ let row_bytes = width.checked_mul(4)?;+ if bytes_per_row < row_bytes {+ return None;+ }+ let required_len = height.checked_mul(bytes_per_row)?;+ if data.len() < required_len {+ return None;+ }+ let mut rgba = vec![0u8; height.checked_mul(row_bytes)?];+ for y in 0..height {+ let src_row = data.get(y * bytes_per_row..y * bytes_per_row + row_bytes)?;+ let dst_row = rgba.get_mut(y * row_bytes..(y + 1) * row_bytes)?;+ for (src, dst) in src_row.chunks_exact(4).zip(dst_row.chunks_exact_mut(4)) {+ let (r, b) = match order {+ ChannelOrder::Rgba => (src[0], src[2]),+ ChannelOrder::Bgra => (src[2], src[0]),+ };+ dst[0] = r;+ dst[1] = src[1];+ dst[2] = b;+ dst[3] = src[3];+ }+ }+ Some(rgba)+ }++ #[cfg(not(target_os = "linux"))]+ #[test]+ fn rgba_dispatch_matches_original_for_both_channel_orders() {+ for (width, height, pad) in rgb_cases() {+ let stride = width * 4 + pad;+ let data = paint(+ height * stride + 8,+ (width as u32).wrapping_mul(37).wrapping_add(height as u32),+ );+ for order in [ChannelOrder::Rgba, ChannelOrder::Bgra] {+ let label = format!("{width}x{height} stride {stride} {order:?}");+ let expected = reference_rgba(&data, width, height, stride, order);+ let got = rgba_from_raw(&data, width, height, stride, order)+ .map(|image| image.into_raw());+ assert_rgb_path_matches_original("rgba", got, &expected, &label);+ }+ }+ }++ #[cfg(not(target_os = "linux"))]+ #[test]+ fn rgba_packing_rejects_undersized_input() {+ assert!(rgba_from_raw(&[0, 1, 2, 3], 2, 1, 4, ChannelOrder::Rgba).is_none());+ assert!(rgba_from_raw(&[0, 1, 2, 3], 1, 2, 4, ChannelOrder::Bgra).is_none());+ assert!(rgba_from_raw(&[0, 1, 2, 3, 4, 5, 6], 1, 1, 8, ChannelOrder::Rgba).is_none());+ let image = rgba_from_raw(&[10, 20, 30, 40, 1, 2, 3, 4], 1, 1, 8, ChannelOrder::Bgra)+ .expect("padded row");+ assert_eq!(image.as_raw(), &[30, 20, 10, 40]);+ let empty = rgba_from_raw(&[], 0, 0, 0, ChannelOrder::Bgra).expect("empty image");+ assert!(empty.as_raw().is_empty());+ }+fn rgb_image(width: u32, height: u32, seed: u32) -> RgbImage {let buf = paint((width as usize) * (height as usize) * 3, seed);RgbImage::from_raw(width, height, buf).expect("rgb image")