Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
improve: copy Linux camera frames once
Kyzer39016e9635fe
diff --git a/crates/camera-ffmpeg/src/linux.rs b/crates/camera-ffmpeg/src/linux.rsindex 89e42e2a8..ad28a4be3 100644--- a/crates/camera-ffmpeg/src/linux.rs+++ b/crates/camera-ffmpeg/src/linux.rs@@ -72,24 +72,85 @@ fn decode_mjpeg(bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {})}-impl CapturedFrameExt for CapturedFrame {+impl CapturedFrameExt for CapturedFrame<'_> {fn as_ffmpeg(&self) -> Result<FFVideo, AsFFmpegError> {let native = self.native();- let format = native.format;- let bytes = native.bytes.as_slice();-- match fourcc_str(format.fourcc).as_str() {- "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2),- "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2),- "RGB3" => copy_packed(bytes, format, Pixel::RGB24, 3),- "BGR3" => copy_packed(bytes, format, Pixel::BGR24, 3),- "NV12" => copy_nv12(bytes, format),- "YU12" => copy_yuv420(bytes, format, false),- "YV12" => copy_yuv420(bytes, format, true),- "MJPG" | "JPEG" => decode_mjpeg(bytes),- other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())),- }+ frame_from_bytes(native.bytes, native.format)+ }+}++fn frame_from_bytes(bytes: &[u8], format: NativeFrameFormat) -> Result<FFVideo, AsFFmpegError> {+ match fourcc_str(format.fourcc).as_str() {+ "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2),+ "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2),+ "RGB3" => copy_packed(bytes, format, Pixel::RGB24, 3),+ "BGR3" => copy_packed(bytes, format, Pixel::BGR24, 3),+ "NV12" => copy_nv12(bytes, format),+ "YU12" => copy_yuv420(bytes, format, false),+ "YV12" => copy_yuv420(bytes, format, true),+ "MJPG" | "JPEG" => decode_mjpeg(bytes),+ other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())),+ }+}++const FRAME_POOL_CAPACITY: usize = 4;++struct FramePool {+ frames: Vec<FFVideo>,+}++thread_local! {+ static FRAME_POOL: RefCell<FramePool> = const { RefCell::new(FramePool { frames: Vec::new() }) };+}++fn with_pooled_frame(+ pixel: Pixel,+ width: u32,+ height: u32,+ fill: impl FnOnce(&mut FFVideo) -> Result<(), AsFFmpegError>,+) -> Result<FFVideo, AsFFmpegError> {+ FRAME_POOL.with(|pool| {+ let mut pool = pool.borrow_mut();+ let index = if let Some(index) = matching_reusable(&pool.frames, pixel, width, height) {+ index+ } else if let Some(index) = pool.frames.iter().position(frame_is_reusable) {+ pool.frames[index] = FFVideo::new(pixel, width, height);+ index+ } else if pool.frames.len() < FRAME_POOL_CAPACITY {+ pool.frames.push(FFVideo::new(pixel, width, height));+ pool.frames.len() - 1+ } else {+ let mut frame = FFVideo::new(pixel, width, height);+ fill(&mut frame)?;+ return Ok(frame);+ };+ fill(&mut pool.frames[index])?;+ Ok(reference_frame(&pool.frames[index]))+ })+}++fn matching_reusable(frames: &[FFVideo], pixel: Pixel, width: u32, height: u32) -> Option<usize> {+ frames.iter().position(|frame| {+ frame.format() == pixel+ && frame.width() == width+ && frame.height() == height+ && frame_is_reusable(frame)+ })+}++fn frame_is_reusable(frame: &FFVideo) -> bool {+ // av_frame_ref hands this buffer to the feed. It can be filled again only+ // after those references are gone, or a live frame would change underneath.+ unsafe { ffmpeg::ffi::av_frame_is_writable(frame.as_ptr().cast_mut()) != 0 }+}++fn reference_frame(frame: &FFVideo) -> FFVideo {+ let mut referenced = FFVideo::empty();+ let status = unsafe { ffmpeg::ffi::av_frame_ref(referenced.as_mut_ptr(), frame.as_ptr()) };+ if status < 0 {+ return frame.clone();}+ referenced}fn copy_packed(@@ -104,18 +165,18 @@ fn copy_packed(let source_stride = format.stride.max(row_width);require_len(bytes, source_stride * height, format)?;- let mut frame = FFVideo::new(pixel, format.width, format.height);- let dest_stride = frame.stride(0);- let dest = frame.data_mut(0);-- for y in 0..height {- let source_start = y * source_stride;- let source_end = source_start + row_width;- let dest_start = y * dest_stride;- dest[dest_start..dest_start + row_width].copy_from_slice(&bytes[source_start..source_end]);- }-- Ok(frame)+ with_pooled_frame(pixel, format.width, format.height, |frame| {+ let dest_stride = frame.stride(0);+ let dest = frame.data_mut(0);+ for y in 0..height {+ let source_start = y * source_stride;+ let source_end = source_start + row_width;+ let dest_start = y * dest_stride;+ dest[dest_start..dest_start + row_width]+ .copy_from_slice(&bytes[source_start..source_end]);+ }+ Ok(())+ })}fn copy_nv12(bytes: &[u8], format: NativeFrameFormat) -> Result<FFVideo, AsFFmpegError> {@@ -126,29 +187,29 @@ fn copy_nv12(bytes: &[u8], format: NativeFrameFormat) -> Result<FFVideo, AsFFmpelet uv_height = height / 2;require_len(bytes, y_size + source_stride * uv_height, format)?;- let mut frame = FFVideo::new(Pixel::NV12, format.width, format.height);- let y_dest_stride = frame.stride(0);- copy_plane(- bytes,- 0,- source_stride,- frame.data_mut(0),- y_dest_stride,- width,- height,- );- let uv_dest_stride = frame.stride(1);- copy_plane(- bytes,- y_size,- source_stride,- frame.data_mut(1),- uv_dest_stride,- width,- uv_height,- );-- Ok(frame)+ with_pooled_frame(Pixel::NV12, format.width, format.height, |frame| {+ let y_dest_stride = frame.stride(0);+ copy_plane(+ bytes,+ 0,+ source_stride,+ frame.data_mut(0),+ y_dest_stride,+ width,+ height,+ );+ let uv_dest_stride = frame.stride(1);+ copy_plane(+ bytes,+ y_size,+ source_stride,+ frame.data_mut(1),+ uv_dest_stride,+ width,+ uv_height,+ );+ Ok(())+ })}fn copy_yuv420(@@ -172,43 +233,42 @@ fn copy_yuv420(let chroma_size = chroma_stride * chroma_height;require_len(bytes, y_size + chroma_size * 2, format)?;- let mut frame = FFVideo::new(Pixel::YUV420P, format.width, format.height);- let y_dest_stride = frame.stride(0);- copy_plane(- bytes,- 0,- y_stride,- frame.data_mut(0),- y_dest_stride,- width,- height,- );-let first_chroma_plane = if v_before_u { 2 } else { 1 };let second_chroma_plane = if v_before_u { 1 } else { 2 };- let first_chroma_stride = frame.stride(first_chroma_plane);- copy_plane(- bytes,- y_size,- chroma_stride,- frame.data_mut(first_chroma_plane),- first_chroma_stride,- chroma_width,- chroma_height,- );- let second_chroma_stride = frame.stride(second_chroma_plane);- copy_plane(- bytes,- y_size + chroma_size,- chroma_stride,- frame.data_mut(second_chroma_plane),- second_chroma_stride,- chroma_width,- chroma_height,- );-- Ok(frame)+ with_pooled_frame(Pixel::YUV420P, format.width, format.height, |frame| {+ let y_dest_stride = frame.stride(0);+ copy_plane(+ bytes,+ 0,+ y_stride,+ frame.data_mut(0),+ y_dest_stride,+ width,+ height,+ );+ let first_chroma_stride = frame.stride(first_chroma_plane);+ copy_plane(+ bytes,+ y_size,+ chroma_stride,+ frame.data_mut(first_chroma_plane),+ first_chroma_stride,+ chroma_width,+ chroma_height,+ );+ let second_chroma_stride = frame.stride(second_chroma_plane);+ copy_plane(+ bytes,+ y_size + chroma_size,+ chroma_stride,+ frame.data_mut(second_chroma_plane),+ second_chroma_stride,+ chroma_width,+ chroma_height,+ );+ Ok(())+ })}fn copy_plane(@@ -247,3 +307,99 @@ fn require_len(fn fourcc_str(fourcc: [u8; 4]) -> String {String::from_utf8_lossy(&fourcc).to_string()}++#[cfg(test)]+mod tests {+ use super::{AsFFmpegError, copy_nv12, copy_packed, frame_from_bytes};+ use cap_camera::NativeFrameFormat;+ use ffmpeg::format::Pixel;++ fn format(fourcc: &[u8; 4], width: u32, height: u32, stride: usize) -> NativeFrameFormat {+ NativeFrameFormat {+ fourcc: *fourcc,+ width,+ height,+ stride,+ }+ }++ fn active_rows(frame: &ffmpeg::frame::Video, plane: usize, row_width: usize) -> Vec<u8> {+ let stride = frame.stride(plane);+ let height = frame.plane_height(plane) as usize;+ let data = frame.data(plane);+ let mut rows = Vec::with_capacity(height * row_width);+ for y in 0..height {+ let start = y * stride;+ rows.extend_from_slice(&data[start..start + row_width]);+ }+ rows+ }++ #[test]+ fn packed_yuyv_keeps_rows_and_drops_source_padding() {+ let _ = ffmpeg::init();+ let source = [1, 2, 3, 4, 9, 9, 5, 6, 7, 8, 8, 8];+ let frame = copy_packed(&source, format(b"YUYV", 2, 2, 6), Pixel::YUYV422, 2).unwrap();+ assert_eq!(frame.format(), Pixel::YUYV422);+ assert_eq!(frame.width(), 2);+ assert_eq!(frame.height(), 2);+ assert_eq!(active_rows(&frame, 0, 4), vec![1, 2, 3, 4, 5, 6, 7, 8]);+ }++ #[test]+ fn nv12_copies_y_and_interleaved_chroma() {+ let _ = ffmpeg::init();+ let source = [1, 2, 3, 4, 5, 6];+ let frame = copy_nv12(&source, format(b"NV12", 2, 2, 2)).unwrap();+ assert_eq!(active_rows(&frame, 0, 2), vec![1, 2, 3, 4]);+ assert_eq!(active_rows(&frame, 1, 2), vec![5, 6]);+ }++ #[test]+ fn yv12_stores_v_before_u() {+ let _ = ffmpeg::init();+ let source = [1, 2, 3, 4, 9, 8];+ let yv12 = frame_from_bytes(&source, format(b"YV12", 2, 2, 2)).unwrap();+ let yu12 = frame_from_bytes(&source, format(b"YU12", 2, 2, 2)).unwrap();+ assert_eq!(active_rows(&yv12, 0, 2), vec![1, 2, 3, 4]);+ assert_eq!(active_rows(&yv12, 1, 1), vec![8]);+ assert_eq!(active_rows(&yv12, 2, 1), vec![9]);+ assert_eq!(active_rows(&yu12, 1, 1), vec![9]);+ assert_eq!(active_rows(&yu12, 2, 1), vec![8]);+ }++ #[test]+ fn a_live_frame_keeps_its_pixels_when_the_next_one_is_filled() {+ let _ = ffmpeg::init();+ let first =+ copy_packed(&[1, 2, 3, 4], format(b"YUYV", 2, 1, 4), Pixel::YUYV422, 2).unwrap();+ let second =+ copy_packed(&[5, 6, 7, 8], format(b"YUYV", 2, 1, 4), Pixel::YUYV422, 2).unwrap();+ assert_eq!(active_rows(&first, 0, 4), vec![1, 2, 3, 4]);+ assert_eq!(active_rows(&second, 0, 4), vec![5, 6, 7, 8]);+ drop(first);+ let third = copy_packed(+ &[9, 10, 11, 12],+ format(b"YUYV", 2, 1, 4),+ Pixel::YUYV422,+ 2,+ )+ .unwrap();+ assert_eq!(active_rows(&third, 0, 4), vec![9, 10, 11, 12]);+ assert_eq!(active_rows(&second, 0, 4), vec![5, 6, 7, 8]);+ }++ #[test]+ fn short_and_odd_frames_keep_their_errors() {+ let _ = ffmpeg::init();+ let short = copy_packed(&[1], format(b"YUYV", 2, 1, 4), Pixel::YUYV422, 2);+ assert!(matches!(short, Err(AsFFmpegError::InvalidFrameData { .. })));+ let odd = frame_from_bytes(&[1, 2, 3, 4, 5, 6], format(b"YU12", 2, 2, 3));+ assert!(matches!(odd, Err(AsFFmpegError::InvalidFrameStride { .. })));+ let unsupported = frame_from_bytes(&[], format(b"XXXX", 1, 1, 1));+ assert!(matches!(+ unsupported,+ Err(AsFFmpegError::UnsupportedFormat(_))+ ));+ }+}diff --git a/crates/camera/examples/cli.rs b/crates/camera/examples/cli.rsindex 8e7bc1385..eb80293d9 100644--- a/crates/camera/examples/cli.rs+++ b/crates/camera/examples/cli.rs@@ -27,7 +27,7 @@ fn main() {let _handle = selected_camera.start_capturing(selected_format.0, |frame| {- drop(frame);+ let _ = frame.native().bytes.len();}).unwrap();diff --git a/crates/camera/examples/diagnose.rs b/crates/camera/examples/diagnose.rsindex 8abf734a3..67866b3eb 100644--- a/crates/camera/examples/diagnose.rs+++ b/crates/camera/examples/diagnose.rs@@ -196,6 +196,13 @@ fn capture_test(camera: &CameraInfo, mode: CaptureMode) {}}+#[cfg(target_os = "linux")]+fn describe_frame(frame: &cap_camera::CapturedFrame<'_>) -> String {+ let _ = frame;+ "<frame>".to_string()+}++#[cfg(not(target_os = "linux"))]fn describe_frame(frame: &cap_camera::CapturedFrame) -> String {#[cfg(target_os = "macos")]{diff --git a/crates/camera/src/lib.rs b/crates/camera/src/lib.rsindex 98b287506..dd1602777 100644--- a/crates/camera/src/lib.rs+++ b/crates/camera/src/lib.rs@@ -218,6 +218,7 @@ pub enum CaptureMode {Compatibility,}+#[cfg(not(target_os = "linux"))]#[derive(Debug)]pub struct CapturedFrame {native: NativeCapturedFrame,@@ -226,17 +227,35 @@ pub struct CapturedFrame {// pub capture_begin_time: Option<Instant>,}+#[cfg(target_os = "linux")]+#[derive(Debug)]+pub struct CapturedFrame<'a> {+ native: NativeCapturedFrame<'a>,+ // pub reference_time: Instant,+ pub timestamp: Duration,+ // pub capture_begin_time: Option<Instant>,+}++#[cfg(not(target_os = "linux"))]impl CapturedFrame {pub fn native(&self) -> &NativeCapturedFrame {&self.native}}+#[cfg(target_os = "linux")]+impl<'a> CapturedFrame<'a> {+ pub fn native(&self) -> &NativeCapturedFrame<'a> {+ &self.native+ }+}+impl CameraInfo {pub fn formats(&self) -> Option<Vec<Format>> {self.formats_impl()}+ #[cfg(not(target_os = "linux"))]pub fn start_capturing(&self,format: Format,@@ -245,6 +264,16 @@ impl CameraInfo {self.start_capturing_with_mode(format, CaptureMode::default(), callback)}+ #[cfg(target_os = "linux")]+ pub fn start_capturing(+ &self,+ format: Format,+ callback: impl for<'a> FnMut(CapturedFrame<'a>) + Send + 'static,+ ) -> Result<CaptureHandle, StartCapturingError> {+ self.start_capturing_with_mode(format, CaptureMode::default(), callback)+ }++ #[cfg(not(target_os = "linux"))]#[cfg_attr(not(target_os = "macos"), allow(unused_variables))]pub fn start_capturing_with_mode(&self,@@ -262,7 +291,17 @@ impl CameraInfo {)?),#[cfg(windows)]native: Some(start_capturing_impl(self, format, Box::new(callback))?),- #[cfg(target_os = "linux")]+ })+ }++ #[cfg(target_os = "linux")]+ pub fn start_capturing_with_mode(+ &self,+ format: Format,+ _mode: CaptureMode,+ callback: impl for<'a> FnMut(CapturedFrame<'a>) + Send + 'static,+ ) -> Result<CaptureHandle, StartCapturingError> {+ Ok(CaptureHandle {native: Some(start_capturing_impl(self, format, Box::new(callback))?),})}diff --git a/crates/camera/src/linux.rs b/crates/camera/src/linux.rsindex f12147c10..16b3bba23 100644--- a/crates/camera/src/linux.rs+++ b/crates/camera/src/linux.rs@@ -63,8 +63,8 @@ pub struct NativeFrameFormat {}#[derive(Debug)]-pub struct NativeCapturedFrame {- pub bytes: Vec<u8>,+pub struct NativeCapturedFrame<'a> {+ pub bytes: &'a [u8],pub format: NativeFrameFormat,}@@ -183,7 +183,7 @@ impl CameraInfo {pub fn start_capturing_impl(camera: &CameraInfo,format: Format,- mut callback: Box<dyn FnMut(CapturedFrame) + Send>,+ mut callback: Box<dyn for<'a> FnMut(CapturedFrame<'a>) + Send>,) -> Result<NativeCaptureHandle, StartCapturingError> {let device = open_device(camera)?;let requested = V4lFormat::new(@@ -252,7 +252,7 @@ pub fn start_capturing_impl(let bytes = bytes.get(..used).unwrap_or(bytes);callback(CapturedFrame {native: NativeCapturedFrame {- bytes: bytes.to_vec(),+ bytes,format: frame_format,},timestamp: started.elapsed(),