Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(recording): share camera frame pixels across fan-out senders and Linux processed camera

Richie McIlroy06121fbb6742
diff --git a/apps/desktop-gpui/src/feeds.rs b/apps/desktop-gpui/src/feeds.rs
index 17f195422..9635846c0 100644
--- a/apps/desktop-gpui/src/feeds.rs
+++ b/apps/desktop-gpui/src/feeds.rs
@@ -313,7 +313,7 @@ impl CameraProcessingFactory {
impl RecordingCameraWorker {
fn publish(
&self,
- image: &gpui::RenderImage,
+ image: &Arc<gpui::RenderImage>,
dims: (usize, usize),
timestamp: cap_timestamp::Timestamp,
mask: Option<cap_recording::instant_recording::LinuxCameraMaskReceipt>,
@@ -334,14 +334,16 @@ impl RecordingCameraWorker {
return;
}
}
- let Some(bytes) = image.as_bytes(0) else {
+ if image.as_bytes(0).is_none() {
self.publisher
.fail("Processed camera pixels unavailable".to_string());
return;
- };
+ }
self.publisher.publish(
cap_recording::instant_recording::LinuxProcessedCameraFrame {
- bgra: Arc::from(bytes),
+ bgra: cap_recording::instant_recording::LinuxCameraPixels::new(
+ RenderImagePixels(image.clone()),
+ ),
dimensions: (dims.0 as u32, dims.1 as u32),
stride: dims.0 * 4,
timestamp,
@@ -353,6 +355,16 @@ impl RecordingCameraWorker {
}
}
+#[cfg(target_os = "linux")]
+struct RenderImagePixels(Arc<gpui::RenderImage>);
+
+#[cfg(target_os = "linux")]
+impl AsRef<[u8]> for RenderImagePixels {
+ fn as_ref(&self) -> &[u8] {
+ self.0.as_bytes(0).unwrap_or_default()
+ }
+}
+
#[cfg(target_os = "linux")]
fn validate_recording_camera_frame(
timestamp: cap_timestamp::Timestamp,
@@ -2135,6 +2147,22 @@ mod tests {
assert!(scaler.is_some());
}
+ #[cfg(target_os = "linux")]
+ #[test]
+ fn recording_camera_pixels_share_the_preview_image() {
+ let mut frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::BGRA, 2, 1);
+ frame.data_mut(0)[..8].copy_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]);
+ let (image, dims) = camera_preview_image(&frame, &mut None, false, None).unwrap();
+
+ let shared = RenderImagePixels(image.clone());
+ let pixels = cap_recording::instant_recording::LinuxCameraPixels::new(shared);
+
+ let preview = image.as_bytes(0).unwrap();
+ assert_eq!(pixels.as_ptr(), preview.as_ptr());
+ assert_eq!(&*pixels, preview);
+ assert_eq!(pixels.len(), dims.0 * dims.1 * 4);
+ }
+
#[test]
fn camera_preview_downscales_only_when_blur_requires_it() {
let frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::BGRA, 64, 32);
diff --git a/apps/desktop/src-tauri/src/linux_instant_camera.rs b/apps/desktop/src-tauri/src/linux_instant_camera.rs
index 56d0aaa7a..ff9fa79d1 100644
--- a/apps/desktop/src-tauri/src/linux_instant_camera.rs
+++ b/apps/desktop/src-tauri/src/linux_instant_camera.rs
@@ -462,7 +462,7 @@ mod producer {
FFmpegVideoFrame,
feeds::camera::{self, CameraFeed, CameraFeedLock},
instant_recording::{
- LINUX_CAMERA_MAX_MASK_AGE, LinuxCameraBlur, LinuxCameraMaskReceipt,
+ LINUX_CAMERA_MAX_MASK_AGE, LinuxCameraBlur, LinuxCameraMaskReceipt, LinuxCameraPixels,
LinuxCameraProcessing, LinuxCameraPublisher, LinuxProcessedCameraFrame,
LinuxProcessedCameraSource,
},
@@ -765,7 +765,7 @@ mod producer {
}
}
Ok(Some(LinuxProcessedCameraFrame {
- bgra: Arc::from(bgra),
+ bgra: LinuxCameraPixels::new(bgra),
dimensions: receipt.dimensions,
stride: row,
timestamp: receipt.timestamp,
diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs
index 47e354584..e70151f06 100644
--- a/crates/recording/src/feeds/camera.rs
+++ b/crates/recording/src/feeds/camera.rs
@@ -25,7 +25,7 @@ use tokio::{sync::oneshot, task::LocalSet};
use tracing::{debug, error, trace, warn};
use crate::ffmpeg::FFmpegVideoFrame;
-use crate::output_pipeline::NativeCameraFrame;
+use crate::output_pipeline::{NativeCameraFrame, VideoFrame};
const CAMERA_INIT_TIMEOUT: Duration = Duration::from_secs(4);
/// Outer deadline for camera readiness. Must cover both capture attempts on
@@ -114,6 +114,22 @@ impl<T> FanoutTarget<T> for CountedSender<T> {
}
}
+trait FanoutFrame {
+ fn share(&self) -> Self;
+}
+
+impl FanoutFrame for FFmpegVideoFrame {
+ fn share(&self) -> Self {
+ self.duplicate().unwrap_or_else(|| self.clone())
+ }
+}
+
+impl FanoutFrame for NativeCameraFrame {
+ fn share(&self) -> Self {
+ self.clone()
+ }
+}
+
/// Rate limit for drop warnings: the first drop is reported at once, later
/// drops at most once per interval with the count since the last report, so
/// an isolated loss is never hidden and a sustained loss cannot flood the log.
@@ -1619,7 +1635,7 @@ impl FanoutDrops<'_> {
}
}
-fn send_frame_to_camera_senders<T: Clone, S: FanoutTarget<T>>(
+fn send_frame_to_camera_senders<T: FanoutFrame, S: FanoutTarget<T>>(
senders: &mut Vec<S>,
frame: T,
frame_num: u64,
@@ -1639,7 +1655,8 @@ fn send_frame_to_camera_senders<T: Clone, S: FanoutTarget<T>>(
);
}
- // The last sender with room takes the frame itself instead of a clone.
+ // The last sender with room takes the frame itself; the others share its
+ // pixel storage.
let last_ready_sender = senders
.iter()
.rposition(|sender| !sender.channel().is_full());
@@ -1665,7 +1682,7 @@ fn send_frame_to_camera_senders<T: Clone, S: FanoutTarget<T>>(
let Some(frame) = frame.as_ref() else {
break;
};
- sender.channel().try_send(frame.clone())
+ sender.channel().try_send(frame.share())
};
match send_result {
@@ -2759,6 +2776,12 @@ mod ingress_tests {
type Output = (flume::Receiver<u32>, Arc<AtomicU64>);
+ impl FanoutFrame for u32 {
+ fn share(&self) -> Self {
+ *self
+ }
+ }
+
struct Fanout {
senders: Vec<CountedSender<u32>>,
total: u64,
@@ -2861,6 +2884,58 @@ mod ingress_tests {
assert_eq!(fanout.total, 14);
}
+ #[test]
+ fn camera_fanout_delivers_identical_pixels_without_copying() {
+ let mut inner = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::NV12, 64, 48);
+ for plane in 0..inner.planes() {
+ for (index, byte) in inner.data_mut(plane).iter_mut().enumerate() {
+ *byte = (index * 7 + plane * 31) as u8;
+ }
+ }
+ inner.set_pts(Some(1234));
+ let expected: Vec<_> = (0..inner.planes())
+ .map(|plane| (inner.stride(plane), inner.data(plane).to_vec()))
+ .collect();
+ let pixels = inner.data(0).as_ptr();
+ let captured_at = std::time::Instant::now();
+ let (mut senders, receivers): (Vec<_>, Vec<_>) = (0..3).map(|_| flume::bounded(1)).unzip();
+ let mut total = 0;
+ let mut log = DropLog::default();
+
+ send_frame_to_camera_senders(
+ &mut senders,
+ FFmpegVideoFrame {
+ inner,
+ timestamp: Timestamp::Instant(captured_at),
+ },
+ 0,
+ "Camera",
+ FanoutDrops {
+ total: &mut total,
+ log: &mut log,
+ },
+ );
+
+ let frames: Vec<FFmpegVideoFrame> = receivers
+ .iter()
+ .map(|rx| rx.try_recv().expect("fanned-out frame"))
+ .collect();
+ for frame in &frames {
+ assert_eq!(frame.inner.data(0).as_ptr(), pixels);
+ assert_eq!(frame.inner.format(), ffmpeg::format::Pixel::NV12);
+ assert_eq!((frame.inner.width(), frame.inner.height()), (64, 48));
+ assert_eq!(frame.inner.pts(), Some(1234));
+ assert!(matches!(frame.timestamp, Timestamp::Instant(value) if value == captured_at));
+ for (plane, (stride, bytes)) in expected.iter().enumerate() {
+ assert_eq!(frame.inner.stride(plane), *stride);
+ assert_eq!(frame.inner.data(plane), bytes.as_slice());
+ }
+ }
+ let buffer = unsafe { (*frames[0].inner.as_ptr()).buf[0] };
+ assert_eq!(unsafe { ffmpeg::ffi::av_buffer_get_ref_count(buffer) }, 3);
+ assert_eq!(total, 0);
+ }
+
/// Consumers draining concurrently with the fan-out: every frame is
/// either received or counted as dropped for each queue, never both.
#[test]
diff --git a/crates/recording/src/instant_recording.rs b/crates/recording/src/instant_recording.rs
index bae7bf008..d38bf6404 100644
--- a/crates/recording/src/instant_recording.rs
+++ b/crates/recording/src/instant_recording.rs
@@ -6,7 +6,7 @@ mod linux_camera;
#[cfg(target_os = "linux")]
pub use linux_camera::{
LINUX_CAMERA_MAX_MASK_AGE, LinuxCameraBlur, LinuxCameraEffect, LinuxCameraMaskReceipt,
- LinuxCameraPresentation, LinuxCameraPresentationError, LinuxCameraProcessing,
+ LinuxCameraPixels, LinuxCameraPresentation, LinuxCameraPresentationError, LinuxCameraProcessing,
LinuxCameraPublisher, LinuxCameraRect, LinuxCameraShape, LinuxProcessedCameraFrame,
LinuxProcessedCameraSource,
};
diff --git a/crates/recording/src/instant_recording/linux_camera.rs b/crates/recording/src/instant_recording/linux_camera.rs
index e87fc756e..fa92bf956 100644
--- a/crates/recording/src/instant_recording/linux_camera.rs
+++ b/crates/recording/src/instant_recording/linux_camera.rs
@@ -202,9 +202,34 @@ pub struct LinuxCameraMaskReceipt {
pub completed_at: Instant,
}
+#[derive(Clone)]
+pub struct LinuxCameraPixels(Arc<dyn AsRef<[u8]> + Send + Sync>);
+
+impl LinuxCameraPixels {
+ pub fn new(pixels: impl AsRef<[u8]> + Send + Sync + 'static) -> Self {
+ Self(Arc::new(pixels))
+ }
+}
+
+impl std::ops::Deref for LinuxCameraPixels {
+ type Target = [u8];
+
+ fn deref(&self) -> &[u8] {
+ (*self.0).as_ref()
+ }
+}
+
+impl std::fmt::Debug for LinuxCameraPixels {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("LinuxCameraPixels")
+ .field("len", &self.len())
+ .finish()
+ }
+}
+
#[derive(Debug, Clone)]
pub struct LinuxProcessedCameraFrame {
- pub bgra: Arc<[u8]>,
+ pub bgra: LinuxCameraPixels,
pub dimensions: (u32, u32),
pub stride: usize,
pub timestamp: Timestamp,
@@ -1446,9 +1471,9 @@ mod tests {
fn processed_frame(now: Instant, generation: u64) -> LinuxProcessedCameraFrame {
LinuxProcessedCameraFrame {
- bgra: Arc::from([
+ bgra: LinuxCameraPixels::new([
10, 20, 30, 255, 40, 50, 60, 255, 99, 99, 99, 99, 70, 80, 90, 255, 100, 110, 120,
- 255, 99, 99, 99, 99,
+ 255, 99, 99, 99, 99u8,
]),
dimensions: (2, 2),
stride: 12,
@@ -1531,7 +1556,7 @@ mod tests {
invalid.stride = 7;
assert!(validate_processed_frame(&invalid, frame.processing, 7, now, now).is_err());
invalid = frame.clone();
- invalid.bgra = Arc::from([0u8; 23]);
+ invalid.bgra = LinuxCameraPixels::new([0u8; 23]);
assert!(validate_processed_frame(&invalid, frame.processing, 7, now, now).is_err());
invalid = frame.clone();
invalid.timestamp = Timestamp::SystemTime(std::time::SystemTime::now());