Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

improve: copy Linux camera frames once

Kyzer39016e9635fe
diff --git a/crates/camera-ffmpeg/src/linux.rs b/crates/camera-ffmpeg/src/linux.rs
index 89e42e2a8..ad28a4be3 100644
--- a/crates/camera-ffmpeg/src/linux.rs
+++ b/crates/camera-ffmpeg/src/linux.rs
@@ -72,24 +72,85 @@ fn decode_mjpeg(bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {
})
}
-impl CapturedFrameExt for CapturedFrame {
+impl CapturedFrameExt for CapturedFrame<'_> {
fn as_ffmpeg(&self) -> Result<FFVideo, AsFFmpegError> {
let native = self.native();
- let format = native.format;
- let bytes = native.bytes.as_slice();
-
- match fourcc_str(format.fourcc).as_str() {
- "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2),
- "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2),
- "RGB3" => copy_packed(bytes, format, Pixel::RGB24, 3),
- "BGR3" => copy_packed(bytes, format, Pixel::BGR24, 3),
- "NV12" => copy_nv12(bytes, format),
- "YU12" => copy_yuv420(bytes, format, false),
- "YV12" => copy_yuv420(bytes, format, true),
- "MJPG" | "JPEG" => decode_mjpeg(bytes),
- other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())),
- }
+ frame_from_bytes(native.bytes, native.format)
+ }
+}
+
+fn frame_from_bytes(bytes: &[u8], format: NativeFrameFormat) -> Result<FFVideo, AsFFmpegError> {
+ match fourcc_str(format.fourcc).as_str() {
+ "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2),
+ "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2),
+ "RGB3" => copy_packed(bytes, format, Pixel::RGB24, 3),
+ "BGR3" => copy_packed(bytes, format, Pixel::BGR24, 3),
+ "NV12" => copy_nv12(bytes, format),
+ "YU12" => copy_yuv420(bytes, format, false),
+ "YV12" => copy_yuv420(bytes, format, true),
+ "MJPG" | "JPEG" => decode_mjpeg(bytes),
+ other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())),
+ }
+}
+
+const FRAME_POOL_CAPACITY: usize = 4;
+
+struct FramePool {
+ frames: Vec<FFVideo>,
+}
+
+thread_local! {
+ static FRAME_POOL: RefCell<FramePool> = const { RefCell::new(FramePool { frames: Vec::new() }) };
+}
+
+fn with_pooled_frame(
+ pixel: Pixel,
+ width: u32,
+ height: u32,
+ fill: impl FnOnce(&mut FFVideo) -> Result<(), AsFFmpegError>,
+) -> Result<FFVideo, AsFFmpegError> {
+ FRAME_POOL.with(|pool| {
+ let mut pool = pool.borrow_mut();
+ let index = if let Some(index) = matching_reusable(&pool.frames, pixel, width, height) {
+ index
+ } else if let Some(index) = pool.frames.iter().position(frame_is_reusable) {
+ pool.frames[index] = FFVideo::new(pixel, width, height);
+ index
+ } else if pool.frames.len() < FRAME_POOL_CAPACITY {
+ pool.frames.push(FFVideo::new(pixel, width, height));
+ pool.frames.len() - 1
+ } else {
+ let mut frame = FFVideo::new(pixel, width, height);
+ fill(&mut frame)?;
+ return Ok(frame);
+ };
+ fill(&mut pool.frames[index])?;
+ Ok(reference_frame(&pool.frames[index]))
+ })
+}
+
+fn matching_reusable(frames: &[FFVideo], pixel: Pixel, width: u32, height: u32) -> Option<usize> {
+ frames.iter().position(|frame| {
+ frame.format() == pixel
+ && frame.width() == width
+ && frame.height() == height
+ && frame_is_reusable(frame)
+ })
+}
+
+fn frame_is_reusable(frame: &FFVideo) -> bool {
+ // av_frame_ref hands this buffer to the feed. It can be filled again only
+ // after those references are gone, or a live frame would change underneath.
+ unsafe { ffmpeg::ffi::av_frame_is_writable(frame.as_ptr().cast_mut()) != 0 }
+}
+
+fn reference_frame(frame: &FFVideo) -> FFVideo {
+ let mut referenced = FFVideo::empty();
+ let status = unsafe { ffmpeg::ffi::av_frame_ref(referenced.as_mut_ptr(), frame.as_ptr()) };
+ if status < 0 {
+ return frame.clone();
}
+ referenced
}
fn copy_packed(
@@ -104,18 +165,18 @@ fn copy_packed(
let source_stride = format.stride.max(row_width);
require_len(bytes, source_stride * height, format)?;
- let mut frame = FFVideo::new(pixel, format.width, format.height);
- let dest_stride = frame.stride(0);
- let dest = frame.data_mut(0);
-
- for y in 0..height {
- let source_start = y * source_stride;
- let source_end = source_start + row_width;
- let dest_start = y * dest_stride;
- dest[dest_start..dest_start + row_width].copy_from_slice(&bytes[source_start..source_end]);
- }
-
- Ok(frame)
+ with_pooled_frame(pixel, format.width, format.height, |frame| {
+ let dest_stride = frame.stride(0);
+ let dest = frame.data_mut(0);
+ for y in 0..height {
+ let source_start = y * source_stride;
+ let source_end = source_start + row_width;
+ let dest_start = y * dest_stride;
+ dest[dest_start..dest_start + row_width]
+ .copy_from_slice(&bytes[source_start..source_end]);
+ }
+ Ok(())
+ })
}
fn copy_nv12(bytes: &[u8], format: NativeFrameFormat) -> Result<FFVideo, AsFFmpegError> {
@@ -126,29 +187,29 @@ fn copy_nv12(bytes: &[u8], format: NativeFrameFormat) -> Result<FFVideo, AsFFmpe
let uv_height = height / 2;
require_len(bytes, y_size + source_stride * uv_height, format)?;
- let mut frame = FFVideo::new(Pixel::NV12, format.width, format.height);
- let y_dest_stride = frame.stride(0);
- copy_plane(
- bytes,
- 0,
- source_stride,
- frame.data_mut(0),
- y_dest_stride,
- width,
- height,
- );
- let uv_dest_stride = frame.stride(1);
- copy_plane(
- bytes,
- y_size,
- source_stride,
- frame.data_mut(1),
- uv_dest_stride,
- width,
- uv_height,
- );
-
- Ok(frame)
+ with_pooled_frame(Pixel::NV12, format.width, format.height, |frame| {
+ let y_dest_stride = frame.stride(0);
+ copy_plane(
+ bytes,
+ 0,
+ source_stride,
+ frame.data_mut(0),
+ y_dest_stride,
+ width,
+ height,
+ );
+ let uv_dest_stride = frame.stride(1);
+ copy_plane(
+ bytes,
+ y_size,
+ source_stride,
+ frame.data_mut(1),
+ uv_dest_stride,
+ width,
+ uv_height,
+ );
+ Ok(())
+ })
}
fn copy_yuv420(
@@ -172,43 +233,42 @@ fn copy_yuv420(
let chroma_size = chroma_stride * chroma_height;
require_len(bytes, y_size + chroma_size * 2, format)?;
- let mut frame = FFVideo::new(Pixel::YUV420P, format.width, format.height);
- let y_dest_stride = frame.stride(0);
- copy_plane(
- bytes,
- 0,
- y_stride,
- frame.data_mut(0),
- y_dest_stride,
- width,
- height,
- );
-
let first_chroma_plane = if v_before_u { 2 } else { 1 };
let second_chroma_plane = if v_before_u { 1 } else { 2 };
- let first_chroma_stride = frame.stride(first_chroma_plane);
- copy_plane(
- bytes,
- y_size,
- chroma_stride,
- frame.data_mut(first_chroma_plane),
- first_chroma_stride,
- chroma_width,
- chroma_height,
- );
- let second_chroma_stride = frame.stride(second_chroma_plane);
- copy_plane(
- bytes,
- y_size + chroma_size,
- chroma_stride,
- frame.data_mut(second_chroma_plane),
- second_chroma_stride,
- chroma_width,
- chroma_height,
- );
-
- Ok(frame)
+ with_pooled_frame(Pixel::YUV420P, format.width, format.height, |frame| {
+ let y_dest_stride = frame.stride(0);
+ copy_plane(
+ bytes,
+ 0,
+ y_stride,
+ frame.data_mut(0),
+ y_dest_stride,
+ width,
+ height,
+ );
+ let first_chroma_stride = frame.stride(first_chroma_plane);
+ copy_plane(
+ bytes,
+ y_size,
+ chroma_stride,
+ frame.data_mut(first_chroma_plane),
+ first_chroma_stride,
+ chroma_width,
+ chroma_height,
+ );
+ let second_chroma_stride = frame.stride(second_chroma_plane);
+ copy_plane(
+ bytes,
+ y_size + chroma_size,
+ chroma_stride,
+ frame.data_mut(second_chroma_plane),
+ second_chroma_stride,
+ chroma_width,
+ chroma_height,
+ );
+ Ok(())
+ })
}
fn copy_plane(
@@ -247,3 +307,99 @@ fn require_len(
fn fourcc_str(fourcc: [u8; 4]) -> String {
String::from_utf8_lossy(&fourcc).to_string()
}
+
+#[cfg(test)]
+mod tests {
+ use super::{AsFFmpegError, copy_nv12, copy_packed, frame_from_bytes};
+ use cap_camera::NativeFrameFormat;
+ use ffmpeg::format::Pixel;
+
+ fn format(fourcc: &[u8; 4], width: u32, height: u32, stride: usize) -> NativeFrameFormat {
+ NativeFrameFormat {
+ fourcc: *fourcc,
+ width,
+ height,
+ stride,
+ }
+ }
+
+ fn active_rows(frame: &ffmpeg::frame::Video, plane: usize, row_width: usize) -> Vec<u8> {
+ let stride = frame.stride(plane);
+ let height = frame.plane_height(plane) as usize;
+ let data = frame.data(plane);
+ let mut rows = Vec::with_capacity(height * row_width);
+ for y in 0..height {
+ let start = y * stride;
+ rows.extend_from_slice(&data[start..start + row_width]);
+ }
+ rows
+ }
+
+ #[test]
+ fn packed_yuyv_keeps_rows_and_drops_source_padding() {
+ let _ = ffmpeg::init();
+ let source = [1, 2, 3, 4, 9, 9, 5, 6, 7, 8, 8, 8];
+ let frame = copy_packed(&source, format(b"YUYV", 2, 2, 6), Pixel::YUYV422, 2).unwrap();
+ assert_eq!(frame.format(), Pixel::YUYV422);
+ assert_eq!(frame.width(), 2);
+ assert_eq!(frame.height(), 2);
+ assert_eq!(active_rows(&frame, 0, 4), vec![1, 2, 3, 4, 5, 6, 7, 8]);
+ }
+
+ #[test]
+ fn nv12_copies_y_and_interleaved_chroma() {
+ let _ = ffmpeg::init();
+ let source = [1, 2, 3, 4, 5, 6];
+ let frame = copy_nv12(&source, format(b"NV12", 2, 2, 2)).unwrap();
+ assert_eq!(active_rows(&frame, 0, 2), vec![1, 2, 3, 4]);
+ assert_eq!(active_rows(&frame, 1, 2), vec![5, 6]);
+ }
+
+ #[test]
+ fn yv12_stores_v_before_u() {
+ let _ = ffmpeg::init();
+ let source = [1, 2, 3, 4, 9, 8];
+ let yv12 = frame_from_bytes(&source, format(b"YV12", 2, 2, 2)).unwrap();
+ let yu12 = frame_from_bytes(&source, format(b"YU12", 2, 2, 2)).unwrap();
+ assert_eq!(active_rows(&yv12, 0, 2), vec![1, 2, 3, 4]);
+ assert_eq!(active_rows(&yv12, 1, 1), vec![8]);
+ assert_eq!(active_rows(&yv12, 2, 1), vec![9]);
+ assert_eq!(active_rows(&yu12, 1, 1), vec![9]);
+ assert_eq!(active_rows(&yu12, 2, 1), vec![8]);
+ }
+
+ #[test]
+ fn a_live_frame_keeps_its_pixels_when_the_next_one_is_filled() {
+ let _ = ffmpeg::init();
+ let first =
+ copy_packed(&[1, 2, 3, 4], format(b"YUYV", 2, 1, 4), Pixel::YUYV422, 2).unwrap();
+ let second =
+ copy_packed(&[5, 6, 7, 8], format(b"YUYV", 2, 1, 4), Pixel::YUYV422, 2).unwrap();
+ assert_eq!(active_rows(&first, 0, 4), vec![1, 2, 3, 4]);
+ assert_eq!(active_rows(&second, 0, 4), vec![5, 6, 7, 8]);
+ drop(first);
+ let third = copy_packed(
+ &[9, 10, 11, 12],
+ format(b"YUYV", 2, 1, 4),
+ Pixel::YUYV422,
+ 2,
+ )
+ .unwrap();
+ assert_eq!(active_rows(&third, 0, 4), vec![9, 10, 11, 12]);
+ assert_eq!(active_rows(&second, 0, 4), vec![5, 6, 7, 8]);
+ }
+
+ #[test]
+ fn short_and_odd_frames_keep_their_errors() {
+ let _ = ffmpeg::init();
+ let short = copy_packed(&[1], format(b"YUYV", 2, 1, 4), Pixel::YUYV422, 2);
+ assert!(matches!(short, Err(AsFFmpegError::InvalidFrameData { .. })));
+ let odd = frame_from_bytes(&[1, 2, 3, 4, 5, 6], format(b"YU12", 2, 2, 3));
+ assert!(matches!(odd, Err(AsFFmpegError::InvalidFrameStride { .. })));
+ let unsupported = frame_from_bytes(&[], format(b"XXXX", 1, 1, 1));
+ assert!(matches!(
+ unsupported,
+ Err(AsFFmpegError::UnsupportedFormat(_))
+ ));
+ }
+}
diff --git a/crates/camera/examples/cli.rs b/crates/camera/examples/cli.rs
index 8e7bc1385..eb80293d9 100644
--- a/crates/camera/examples/cli.rs
+++ b/crates/camera/examples/cli.rs
@@ -27,7 +27,7 @@ fn main() {
let _handle = selected_camera
.start_capturing(selected_format.0, |frame| {
- drop(frame);
+ let _ = frame.native().bytes.len();
})
.unwrap();
diff --git a/crates/camera/examples/diagnose.rs b/crates/camera/examples/diagnose.rs
index 8abf734a3..67866b3eb 100644
--- a/crates/camera/examples/diagnose.rs
+++ b/crates/camera/examples/diagnose.rs
@@ -196,6 +196,13 @@ fn capture_test(camera: &CameraInfo, mode: CaptureMode) {
}
}
+#[cfg(target_os = "linux")]
+fn describe_frame(frame: &cap_camera::CapturedFrame<'_>) -> String {
+ let _ = frame;
+ "<frame>".to_string()
+}
+
+#[cfg(not(target_os = "linux"))]
fn describe_frame(frame: &cap_camera::CapturedFrame) -> String {
#[cfg(target_os = "macos")]
{
diff --git a/crates/camera/src/lib.rs b/crates/camera/src/lib.rs
index 98b287506..dd1602777 100644
--- a/crates/camera/src/lib.rs
+++ b/crates/camera/src/lib.rs
@@ -218,6 +218,7 @@ pub enum CaptureMode {
Compatibility,
}
+#[cfg(not(target_os = "linux"))]
#[derive(Debug)]
pub struct CapturedFrame {
native: NativeCapturedFrame,
@@ -226,17 +227,35 @@ pub struct CapturedFrame {
// pub capture_begin_time: Option<Instant>,
}
+#[cfg(target_os = "linux")]
+#[derive(Debug)]
+pub struct CapturedFrame<'a> {
+ native: NativeCapturedFrame<'a>,
+ // pub reference_time: Instant,
+ pub timestamp: Duration,
+ // pub capture_begin_time: Option<Instant>,
+}
+
+#[cfg(not(target_os = "linux"))]
impl CapturedFrame {
pub fn native(&self) -> &NativeCapturedFrame {
&self.native
}
}
+#[cfg(target_os = "linux")]
+impl<'a> CapturedFrame<'a> {
+ pub fn native(&self) -> &NativeCapturedFrame<'a> {
+ &self.native
+ }
+}
+
impl CameraInfo {
pub fn formats(&self) -> Option<Vec<Format>> {
self.formats_impl()
}
+ #[cfg(not(target_os = "linux"))]
pub fn start_capturing(
&self,
format: Format,
@@ -245,6 +264,16 @@ impl CameraInfo {
self.start_capturing_with_mode(format, CaptureMode::default(), callback)
}
+ #[cfg(target_os = "linux")]
+ pub fn start_capturing(
+ &self,
+ format: Format,
+ callback: impl for<'a> FnMut(CapturedFrame<'a>) + Send + 'static,
+ ) -> Result<CaptureHandle, StartCapturingError> {
+ self.start_capturing_with_mode(format, CaptureMode::default(), callback)
+ }
+
+ #[cfg(not(target_os = "linux"))]
#[cfg_attr(not(target_os = "macos"), allow(unused_variables))]
pub fn start_capturing_with_mode(
&self,
@@ -262,7 +291,17 @@ impl CameraInfo {
)?),
#[cfg(windows)]
native: Some(start_capturing_impl(self, format, Box::new(callback))?),
- #[cfg(target_os = "linux")]
+ })
+ }
+
+ #[cfg(target_os = "linux")]
+ pub fn start_capturing_with_mode(
+ &self,
+ format: Format,
+ _mode: CaptureMode,
+ callback: impl for<'a> FnMut(CapturedFrame<'a>) + Send + 'static,
+ ) -> Result<CaptureHandle, StartCapturingError> {
+ Ok(CaptureHandle {
native: Some(start_capturing_impl(self, format, Box::new(callback))?),
})
}
diff --git a/crates/camera/src/linux.rs b/crates/camera/src/linux.rs
index f12147c10..16b3bba23 100644
--- a/crates/camera/src/linux.rs
+++ b/crates/camera/src/linux.rs
@@ -63,8 +63,8 @@ pub struct NativeFrameFormat {
}
#[derive(Debug)]
-pub struct NativeCapturedFrame {
- pub bytes: Vec<u8>,
+pub struct NativeCapturedFrame<'a> {
+ pub bytes: &'a [u8],
pub format: NativeFrameFormat,
}
@@ -183,7 +183,7 @@ impl CameraInfo {
pub fn start_capturing_impl(
camera: &CameraInfo,
format: Format,
- mut callback: Box<dyn FnMut(CapturedFrame) + Send>,
+ mut callback: Box<dyn for<'a> FnMut(CapturedFrame<'a>) + Send>,
) -> Result<NativeCaptureHandle, StartCapturingError> {
let device = open_device(camera)?;
let requested = V4lFormat::new(
@@ -252,7 +252,7 @@ pub fn start_capturing_impl(
let bytes = bytes.get(..used).unwrap_or(bytes);
callback(CapturedFrame {
native: NativeCapturedFrame {
- bytes: bytes.to_vec(),
+ bytes,
format: frame_format,
},
timestamp: started.elapsed(),