Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(linux-screen): cache screenshot swscale context

Kyzer623ef7d7feb7
diff --git a/crates/recording/src/screenshot.rs b/crates/recording/src/screenshot.rs
index 4551794eb..521243fa3 100644
--- a/crates/recording/src/screenshot.rs
+++ b/crates/recording/src/screenshot.rs
@@ -11,6 +11,8 @@ use anyhow::anyhow;
use image::{DynamicImage, RgbImage, RgbaImage};
#[cfg(target_os = "macos")]
use scap_ffmpeg::AsFFmpeg;
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+use std::cell::RefCell;
#[cfg(not(target_os = "linux"))]
use std::sync::{Arc, Mutex};
#[cfg(not(target_os = "linux"))]
@@ -1723,48 +1725,218 @@ fn crop_area_if_needed(
Ok(image)
}
+/// `Video::alloc` asks swscale for 32-byte lines. A row that is already a multiple
+/// of that is tightly packed, matching the buffer swscale would allocate itself.
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+const SCREENSHOT_RGB_STRIDE_ALIGN: usize = 32;
+
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+struct ScreenshotScaler {
+ context: ffmpeg::software::scaling::context::Context,
+ src_format: ffmpeg::format::Pixel,
+ src_width: u32,
+ src_height: u32,
+ dst_format: ffmpeg::format::Pixel,
+ dst_width: u32,
+ dst_height: u32,
+}
+
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+impl ScreenshotScaler {
+ fn matches(&self, frame: &ffmpeg::frame::Video) -> bool {
+ self.src_format == frame.format()
+ && self.src_width == frame.width()
+ && self.src_height == frame.height()
+ && self.dst_format == ffmpeg::format::Pixel::RGB24
+ && self.dst_width == frame.width()
+ && self.dst_height == frame.height()
+ }
+}
+
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+struct ScreenshotScalerSlot {
+ scaler: Option<ScreenshotScaler>,
+ #[cfg(test)]
+ builds: u64,
+}
+
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+thread_local! {
+ static SCREENSHOT_SCALER: RefCell<ScreenshotScalerSlot> = const {
+ RefCell::new(ScreenshotScalerSlot {
+ scaler: None,
+ #[cfg(test)]
+ builds: 0,
+ })
+ };
+}
+
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+fn with_screenshot_scaler<R>(
+ frame: &ffmpeg::frame::Video,
+ body: impl FnOnce(&mut ffmpeg::software::scaling::context::Context) -> R,
+) -> anyhow::Result<R> {
+ SCREENSHOT_SCALER.with(|slot| {
+ let mut slot = slot.borrow_mut();
+ if slot
+ .scaler
+ .as_ref()
+ .is_none_or(|scaler| !scaler.matches(frame))
+ {
+ let context = ffmpeg::software::scaling::context::Context::get(
+ frame.format(),
+ frame.width(),
+ frame.height(),
+ ffmpeg::format::Pixel::RGB24,
+ frame.width(),
+ frame.height(),
+ ffmpeg::software::scaling::flag::Flags::BILINEAR,
+ )
+ .context("Failed to create scaler")?;
+ slot.scaler = Some(ScreenshotScaler {
+ context,
+ src_format: frame.format(),
+ src_width: frame.width(),
+ src_height: frame.height(),
+ dst_format: ffmpeg::format::Pixel::RGB24,
+ dst_width: frame.width(),
+ dst_height: frame.height(),
+ });
+ #[cfg(test)]
+ {
+ slot.builds = slot.builds.saturating_add(1);
+ }
+ }
+ let scaler = slot
+ .scaler
+ .as_mut()
+ .expect("screenshot scaler is inserted before use");
+ Ok(body(&mut scaler.context))
+ })
+}
+
+#[cfg(test)]
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+fn clear_screenshot_scaler_cache() {
+ SCREENSHOT_SCALER.with(|slot| {
+ *slot.borrow_mut() = ScreenshotScalerSlot {
+ scaler: None,
+ builds: 0,
+ };
+ });
+}
+
+#[cfg(test)]
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+fn screenshot_scaler_builds() -> u64 {
+ SCREENSHOT_SCALER.with(|slot| slot.borrow().builds)
+}
+
#[cfg(any(target_os = "macos", target_os = "linux"))]
fn convert_ffmpeg_frame_to_image(frame: &ffmpeg::frame::Video) -> anyhow::Result<RgbImage> {
- let mut scaler = ffmpeg::software::scaling::context::Context::get(
- frame.format(),
- frame.width(),
- frame.height(),
- ffmpeg::format::Pixel::RGB24,
- frame.width(),
- frame.height(),
- ffmpeg::software::scaling::flag::Flags::BILINEAR,
- )
- .context("Failed to create scaler")?;
+ let width = frame.width() as usize;
+ let height = frame.height() as usize;
+ let row_bytes = width
+ .checked_mul(3)
+ .context("screenshot dimensions overflow")?;
+ let image_bytes = height
+ .checked_mul(row_bytes)
+ .context("screenshot dimensions overflow")?;
+ if width == 0 || height == 0 {
+ return RgbImage::from_raw(frame.width(), frame.height(), Vec::new())
+ .ok_or_else(|| anyhow!("Failed to create image buffer"));
+ }
+
+ if row_bytes % SCREENSHOT_RGB_STRIDE_ALIGN == 0
+ && let Some(image) = scale_screenshot_into_tight_rgb(frame, row_bytes, image_bytes)?
+ {
+ return Ok(image);
+ }
let mut rgb_frame = ffmpeg::frame::Video::empty();
- scaler
- .run(frame, &mut rgb_frame)
+ with_screenshot_scaler(frame, |scaler| scaler.run(frame, &mut rgb_frame))?
.context("Failed to scale frame")?;
+ pack_rgb_frame(&rgb_frame)
+}
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+fn scale_screenshot_into_tight_rgb(
+ frame: &ffmpeg::frame::Video,
+ row_bytes: usize,
+ image_bytes: usize,
+) -> anyhow::Result<Option<RgbImage>> {
+ let mut image = vec![0u8; image_bytes];
+ let mut rgb_frame = ffmpeg::frame::Video::empty();
+ rgb_frame.set_format(ffmpeg::format::Pixel::RGB24);
+ rgb_frame.set_width(frame.width());
+ rgb_frame.set_height(frame.height());
+ let linesize = libc::c_int::try_from(row_bytes).context("screenshot stride exceeds c_int")?;
+ // SAFETY: plane 0 aliases `image` and the frame does not own it (`buf[0]` stays
+ // null, so av_frame_free will not release it). `image` outlives `rgb_frame`.
+ // The stride equals width*3 and is a multiple of Video::alloc's 32-byte
+ // alignment, so this is the same layout swscale uses for its own buffer.
+ unsafe {
+ let raw = rgb_frame.as_mut_ptr();
+ (*raw).data[0] = image.as_mut_ptr();
+ (*raw).linesize[0] = linesize;
+ }
+
+ with_screenshot_scaler(frame, |scaler| scaler.run(frame, &mut rgb_frame))?
+ .context("Failed to scale frame")?;
+
+ let stride = rgb_frame.stride(0);
+ let same_buffer = std::ptr::eq(rgb_frame.data(0).as_ptr(), image.as_ptr());
+ drop(rgb_frame);
+ if stride != row_bytes || !same_buffer {
+ return Ok(None);
+ }
+
+ RgbImage::from_raw(frame.width(), frame.height(), image)
+ .map(Some)
+ .ok_or_else(|| anyhow!("Failed to create image buffer"))
+}
+
+#[cfg(any(target_os = "macos", target_os = "linux"))]
+fn pack_rgb_frame(rgb_frame: &ffmpeg::frame::Video) -> anyhow::Result<RgbImage> {
let width = rgb_frame.width() as usize;
let height = rgb_frame.height() as usize;
- let dst_stride = width * 3;
+ let dst_stride = width
+ .checked_mul(3)
+ .context("screenshot dimensions overflow")?;
let src_stride = rgb_frame.stride(0);
if src_stride < dst_stride {
return Err(anyhow!(
- "Source stride ({}) is less than destination stride ({}); width={}, height={}",
- src_stride,
- dst_stride,
- width,
- height
+ "Source stride ({src_stride}) is less than destination stride ({dst_stride}); width={width}, height={height}"
));
}
- let mut img_buffer = vec![0u8; height * dst_stride];
-
- for y in 0..height {
- let src_slice = &rgb_frame.data(0)[y * src_stride..y * src_stride + dst_stride];
- let dst_slice = &mut img_buffer[y * dst_stride..(y + 1) * dst_stride];
- dst_slice.copy_from_slice(src_slice);
+ let buf_len = height
+ .checked_mul(dst_stride)
+ .context("screenshot dimensions overflow")?;
+ let mut img_buffer = vec![0u8; buf_len];
+ let src = rgb_frame.data(0);
+ if src_stride == dst_stride {
+ let copy_len = height
+ .checked_mul(src_stride)
+ .context("screenshot dimensions overflow")?;
+ let src = src
+ .get(..copy_len)
+ .ok_or_else(|| anyhow!("Failed to create image buffer"))?;
+ img_buffer.copy_from_slice(src);
+ } else {
+ for y in 0..height {
+ let start = y * src_stride;
+ let end = start + dst_stride;
+ let dst_start = y * dst_stride;
+ let src_row = src
+ .get(start..end)
+ .ok_or_else(|| anyhow!("Failed to create image buffer"))?;
+ img_buffer[dst_start..dst_start + dst_stride].copy_from_slice(src_row);
+ }
}
- RgbImage::from_raw(width as u32, height as u32, img_buffer)
+ RgbImage::from_raw(rgb_frame.width(), rgb_frame.height(), img_buffer)
.ok_or_else(|| anyhow!("Failed to create image buffer"))
}
@@ -2423,3 +2595,218 @@ mod screenshot_pixel_tests {
assert_bytes_eq(got.as_raw(), expected.as_raw(), "camera only");
}
}
+
+#[cfg(all(test, any(target_os = "macos", target_os = "linux")))]
+mod screenshot_swscale_cache_tests {
+ use super::{
+ clear_screenshot_scaler_cache, convert_ffmpeg_frame_to_image, screenshot_scaler_builds,
+ };
+ use anyhow::Context;
+ use image::RgbImage;
+ use std::time::{Duration, Instant};
+
+ fn filled_bgrx(format: ffmpeg::format::Pixel, width: u32, height: u32) -> ffmpeg::frame::Video {
+ let mut frame = ffmpeg::frame::Video::new(format, width, height);
+ let stride = frame.stride(0);
+ let data = frame.data_mut(0);
+ for y in 0..height as usize {
+ for x in 0..width as usize {
+ let i = y * stride + x * 4;
+ data[i] = (x.wrapping_mul(3).wrapping_add(y)) as u8;
+ data[i + 1] = (x.wrapping_mul(5).wrapping_add(7)) as u8;
+ data[i + 2] = (x.wrapping_mul(11).wrapping_add(y.wrapping_mul(2))) as u8;
+ data[i + 3] = 0xA5;
+ }
+ }
+ frame
+ }
+
+ fn expected_bgrx_rgb(frame: &ffmpeg::frame::Video) -> Vec<u8> {
+ let width = frame.width() as usize;
+ let height = frame.height() as usize;
+ let stride = frame.stride(0);
+ let src = frame.data(0);
+ let mut rgb = vec![0u8; width * height * 3];
+ for y in 0..height {
+ for x in 0..width {
+ let s = y * stride + x * 4;
+ let d = (y * width + x) * 3;
+ rgb[d] = src[s + 2];
+ rgb[d + 1] = src[s + 1];
+ rgb[d + 2] = src[s];
+ }
+ }
+ rgb
+ }
+
+ /// The pre-cache conversion: a new swscale context and a row pack every call.
+ fn uncached_packed_rgb(frame: &ffmpeg::frame::Video) -> anyhow::Result<RgbImage> {
+ let mut scaler = ffmpeg::software::scaling::context::Context::get(
+ frame.format(),
+ frame.width(),
+ frame.height(),
+ ffmpeg::format::Pixel::RGB24,
+ frame.width(),
+ frame.height(),
+ ffmpeg::software::scaling::flag::Flags::BILINEAR,
+ )
+ .context("Failed to create scaler")?;
+ let mut rgb_frame = ffmpeg::frame::Video::empty();
+ scaler
+ .run(frame, &mut rgb_frame)
+ .context("Failed to scale frame")?;
+ let width = rgb_frame.width() as usize;
+ let height = rgb_frame.height() as usize;
+ let dst_stride = width * 3;
+ let src_stride = rgb_frame.stride(0);
+ anyhow::ensure!(
+ src_stride >= dst_stride,
+ "Source stride ({src_stride}) is less than destination stride ({dst_stride})"
+ );
+ let mut img_buffer = vec![0u8; height * dst_stride];
+ for y in 0..height {
+ let src_slice = &rgb_frame.data(0)[y * src_stride..y * src_stride + dst_stride];
+ img_buffer[y * dst_stride..(y + 1) * dst_stride].copy_from_slice(src_slice);
+ }
+ RgbImage::from_raw(rgb_frame.width(), rgb_frame.height(), img_buffer)
+ .ok_or_else(|| anyhow::anyhow!("Failed to create image buffer"))
+ }
+
+ #[test]
+ fn bgrx_screenshot_reuses_scaler_and_matches_channel_swap() {
+ ffmpeg::init().expect("ffmpeg");
+ clear_screenshot_scaler_cache();
+
+ let aligned = filled_bgrx(ffmpeg::format::Pixel::BGRZ, 3840, 2);
+ let image = convert_ffmpeg_frame_to_image(&aligned).expect("aligned convert");
+ assert_eq!(image.as_raw(), expected_bgrx_rgb(&aligned).as_slice());
+ assert_eq!(
+ image.as_raw(),
+ uncached_packed_rgb(&aligned)
+ .expect("aligned reference")
+ .as_raw()
+ );
+ assert_eq!(screenshot_scaler_builds(), 1);
+ let again = convert_ffmpeg_frame_to_image(&aligned).expect("aligned reuse");
+ assert_eq!(again.as_raw(), image.as_raw());
+ assert_eq!(screenshot_scaler_builds(), 1);
+
+ let padded = filled_bgrx(ffmpeg::format::Pixel::BGRZ, 17, 3);
+ let padded_image = convert_ffmpeg_frame_to_image(&padded).expect("padded convert");
+ assert_eq!(padded_image.as_raw(), expected_bgrx_rgb(&padded).as_slice());
+ assert_eq!(
+ padded_image.as_raw(),
+ uncached_packed_rgb(&padded)
+ .expect("padded reference")
+ .as_raw()
+ );
+ assert_eq!(screenshot_scaler_builds(), 2);
+ convert_ffmpeg_frame_to_image(&padded).expect("padded reuse");
+ assert_eq!(screenshot_scaler_builds(), 2);
+
+ let bgra = filled_bgrx(ffmpeg::format::Pixel::BGRA, 32, 2);
+ let bgra_image = convert_ffmpeg_frame_to_image(&bgra).expect("bgra convert");
+ assert_eq!(bgra_image.as_raw(), expected_bgrx_rgb(&bgra).as_slice());
+ assert_eq!(screenshot_scaler_builds(), 3);
+
+ convert_ffmpeg_frame_to_image(&aligned).expect("return to aligned");
+ assert_eq!(screenshot_scaler_builds(), 4);
+
+ let tiny = filled_bgrx(ffmpeg::format::Pixel::BGRZ, 1, 1);
+ let tiny_image = convert_ffmpeg_frame_to_image(&tiny).expect("tiny convert");
+ assert_eq!(
+ tiny_image.as_raw(),
+ &[tiny.data(0)[2], tiny.data(0)[1], tiny.data(0)[0]]
+ );
+ }
+
+ fn thread_cpu_time() -> Duration {
+ let mut ts = libc::timespec {
+ tv_sec: 0,
+ tv_nsec: 0,
+ };
+ let rc = unsafe { libc::clock_gettime(libc::CLOCK_THREAD_CPUTIME_ID, &mut ts) };
+ assert_eq!(rc, 0);
+ Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32)
+ }
+
+ fn summarize(samples: &mut [Duration]) -> (Duration, Duration, Duration) {
+ samples.sort_unstable();
+ let min = samples[0];
+ let sum = samples.iter().copied().sum::<Duration>();
+ let mean = sum / u32::try_from(samples.len()).expect("sample count");
+ let mid = samples.len() / 2;
+ let median = if samples.len().is_multiple_of(2) {
+ (samples[mid - 1] + samples[mid]) / 2
+ } else {
+ samples[mid]
+ };
+ (min, median, mean)
+ }
+
+ fn time_convert(
+ frame: &ffmpeg::frame::Video,
+ runs: usize,
+ convert: impl Fn(&ffmpeg::frame::Video) -> RgbImage,
+ ) -> (Vec<Duration>, Vec<Duration>) {
+ let mut wall = Vec::with_capacity(runs);
+ let mut cpu = Vec::with_capacity(runs);
+ for _ in 0..runs {
+ let cpu_start = thread_cpu_time();
+ let start = Instant::now();
+ let image = convert(frame);
+ let elapsed = start.elapsed();
+ let cpu_elapsed = thread_cpu_time().saturating_sub(cpu_start);
+ std::hint::black_box(image);
+ wall.push(elapsed);
+ cpu.push(cpu_elapsed);
+ }
+ (wall, cpu)
+ }
+
+ #[test]
+ #[ignore = "manual 4K BGRZ screenshot conversion benchmark"]
+ fn bench_bgrz_screenshot_conversion() {
+ ffmpeg::init().expect("ffmpeg");
+ const WARMUP: usize = 5;
+ const RUNS: usize = 30;
+ for (width, height) in [(3840, 2160), (1920, 1080)] {
+ let frame = filled_bgrx(ffmpeg::format::Pixel::BGRZ, width, height);
+ let baseline = uncached_packed_rgb(&frame).expect("baseline");
+ clear_screenshot_scaler_cache();
+ let cached = convert_ffmpeg_frame_to_image(&frame).expect("cached");
+ assert_eq!(baseline.as_raw(), cached.as_raw());
+
+ for _ in 0..WARMUP {
+ std::hint::black_box(uncached_packed_rgb(&frame).expect("warmup baseline"));
+ }
+ let (mut base_wall, mut base_cpu) = time_convert(&frame, RUNS, |frame| {
+ uncached_packed_rgb(frame).expect("baseline")
+ });
+ clear_screenshot_scaler_cache();
+ for _ in 0..WARMUP {
+ std::hint::black_box(convert_ffmpeg_frame_to_image(&frame).expect("warmup cached"));
+ }
+ let builds_before = screenshot_scaler_builds();
+ let (mut new_wall, mut new_cpu) = time_convert(&frame, RUNS, |frame| {
+ convert_ffmpeg_frame_to_image(frame).expect("cached")
+ });
+ let builds_after = screenshot_scaler_builds();
+
+ let (base_wall_min, base_wall_med, base_wall_mean) = summarize(&mut base_wall);
+ let (base_cpu_min, base_cpu_med, base_cpu_mean) = summarize(&mut base_cpu);
+ let (new_wall_min, new_wall_med, new_wall_mean) = summarize(&mut new_wall);
+ let (new_cpu_min, new_cpu_med, new_cpu_mean) = summarize(&mut new_cpu);
+ println!(
+ "size={width}x{height} warmup={WARMUP} runs={RUNS} scaler_builds_during_timed_runs={}",
+ builds_after - builds_before
+ );
+ println!(
+ "baseline wall min={base_wall_min:?} median={base_wall_med:?} mean={base_wall_mean:?} cpu min={base_cpu_min:?} median={base_cpu_med:?} mean={base_cpu_mean:?}"
+ );
+ println!(
+ "cached wall min={new_wall_min:?} median={new_wall_med:?} mean={new_wall_mean:?} cpu min={new_cpu_min:?} median={new_cpu_med:?} mean={new_cpu_mean:?}"
+ );
+ }
+ }
+}