Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(recording): size camera ingress by time and count every lost frame
Richie McIlroy9b4aa5edb9be
diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rsindex 28d822b4b..bbd025587 100644--- a/crates/recording/src/feeds/camera.rs+++ b/crates/recording/src/feeds/camera.rs@@ -54,11 +54,86 @@ pub struct CameraFeed {senders: Vec<flume::Sender<FFmpegVideoFrame>>,ffmpeg_sender_count: Arc<std::sync::atomic::AtomicUsize>,native_senders: Vec<flume::Sender<NativeCameraFrame>>,- native_sender_count: Arc<std::sync::atomic::AtomicUsize>,+ native_ingress: Arc<NativeCameraIngress>,+ native_fanout_log: DropLog,+ ffmpeg_fanout_full: std::sync::atomic::AtomicU64,+ ffmpeg_fanout_log: DropLog,on_ready: Vec<oneshot::Sender<()>>,on_disconnect: Vec<Box<dyn Fn() + Send>>,}+/// Shared between the camera feed, its capture callback and the native+/// sources of the recordings attached to it.+#[derive(Debug, Default)]+pub struct NativeCameraIngress {+ senders: std::sync::atomic::AtomicUsize,+ mailbox_full: std::sync::atomic::AtomicU64,+ fanout_full: std::sync::atomic::AtomicU64,+}++/// Native camera frames lost inside the feed, before a recording's source+/// received them. Monotonic over the feed's lifetime; recordings report the+/// difference between their start and the current value.+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub struct NativeCameraIngressDrops {+ /// The capture callback found the feed's mailbox full (the feed actor+ /// was not running, for example during a runtime stall).+ pub mailbox_full: u64,+ /// A recording's ingress queue was full when the feed fanned a frame out.+ pub fanout_full: u64,+}++impl NativeCameraIngressDrops {+ pub fn since(self, start: Self) -> Self {+ Self {+ mailbox_full: self.mailbox_full.saturating_sub(start.mailbox_full),+ fanout_full: self.fanout_full.saturating_sub(start.fanout_full),+ }+ }+}++impl NativeCameraIngress {+ pub fn drops(&self) -> NativeCameraIngressDrops {+ NativeCameraIngressDrops {+ mailbox_full: self.mailbox_full.load(std::sync::atomic::Ordering::Relaxed),+ fanout_full: self.fanout_full.load(std::sync::atomic::Ordering::Relaxed),+ }+ }++ /// Called from capture callbacks: a relaxed increment, never a wait.+ pub fn record_mailbox_full(&self) {+ self.mailbox_full+ .fetch_add(1, std::sync::atomic::Ordering::Relaxed);+ }+}++/// Rate limit for drop warnings: the first drop is reported at once, later+/// drops at most once per interval with the count since the last report, so+/// an isolated loss is never hidden and a sustained loss cannot flood the log.+#[derive(Debug, Default)]+pub(crate) struct DropLog {+ last_report: Option<std::time::Instant>,+ unreported: u64,+}++impl DropLog {+ pub(crate) const INTERVAL: Duration = Duration::from_secs(5);++ /// Adds `count` drops. Returns the drops to report now, if a report is due.+ pub(crate) fn record(&mut self, count: u64, now: std::time::Instant) -> Option<u64> {+ self.unreported = self.unreported.saturating_add(count);+ if self.unreported == 0+ || self+ .last_report+ .is_some_and(|last| now.duration_since(last) < Self::INTERVAL)+ {+ return None;+ }+ self.last_report = Some(now);+ Some(std::mem::take(&mut self.unreported))+ }+}+enum State {Open(OpenState),Locked {@@ -210,7 +285,10 @@ impl Default for CameraFeed {senders: Vec::new(),ffmpeg_sender_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)),native_senders: Vec::new(),- native_sender_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)),+ native_ingress: Arc::new(NativeCameraIngress::default()),+ native_fanout_log: DropLog::default(),+ ffmpeg_fanout_full: std::sync::atomic::AtomicU64::new(0),+ ffmpeg_fanout_log: DropLog::default(),on_ready: Vec::new(),on_disconnect: Vec::new(),}@@ -229,11 +307,16 @@ pub struct CameraFeedLock {actor: ActorRef<CameraFeed>,camera_info: cap_camera::CameraInfo,video_info: VideoInfo,+ native_ingress: Arc<NativeCameraIngress>,drop_tx: Option<oneshot::Sender<()>>,_token: Arc<()>,}impl CameraFeedLock {+ pub fn native_ingress(&self) -> &Arc<NativeCameraIngress> {+ &self.native_ingress+ }+pub fn camera_info(&self) -> &cap_camera::CameraInfo {&self.camera_info}@@ -369,7 +452,7 @@ struct CameraSetupArgs {new_frame_recipient: Recipient<NewFrame>,native_frame_recipient: Recipient<NewNativeFrame>,ffmpeg_sender_count: Arc<std::sync::atomic::AtomicUsize>,- native_sender_count: Arc<std::sync::atomic::AtomicUsize>,+ native_ingress: Arc<NativeCameraIngress>,flow: CameraSetupFlow,}@@ -432,7 +515,7 @@ fn spawn_camera_setup(args: CameraSetupArgs) -> (ReadyFuture, SyncSender<()>) {new_frame_recipient,native_frame_recipient,ffmpeg_sender_count,- native_sender_count,+ native_ingress,flow,} = args;@@ -498,7 +581,7 @@ fn spawn_camera_setup(args: CameraSetupArgs) -> (ReadyFuture, SyncSender<()>) {new_frame_recipient,native_frame_recipient,ffmpeg_sender_count,- native_sender_count,+ native_ingress,&done_rx_thread,).await;@@ -857,7 +940,7 @@ async fn setup_camera(recipient: Recipient<NewFrame>,native_recipient: Recipient<NewNativeFrame>,ffmpeg_sender_count: Arc<std::sync::atomic::AtomicUsize>,- native_sender_count: Arc<std::sync::atomic::AtomicUsize>,+ native_ingress: Arc<NativeCameraIngress>,cancel_rx: &mpsc::Receiver<()>,) -> Result<SetupCameraResult, SetInputError> {let camera = find_camera(id).ok_or(SetInputError::DeviceNotFound)?;@@ -882,7 +965,7 @@ async fn setup_camera(recipient.clone(),native_recipient.clone(),ffmpeg_sender_count.clone(),- native_sender_count.clone(),+ native_ingress.clone(),).await;@@ -910,7 +993,7 @@ async fn setup_camera(recipient,native_recipient,ffmpeg_sender_count,- native_sender_count,+ native_ingress,).await.inspect(|_| {@@ -921,6 +1004,21 @@ async fn setup_camera(}}+/// The capture callback's hand-over to the feed: never waits for mailbox+/// capacity; a full mailbox loses the frame and counts it.+#[cfg(any(target_os = "macos", windows))]+fn hand_native_frame_to_feed(+ recipient: &Recipient<NewNativeFrame>,+ ingress: &NativeCameraIngress,+ frame: NativeCameraFrame,+) {+ if let Err(kameo::error::SendError::MailboxFull(_)) =+ recipient.tell(NewNativeFrame(frame)).try_send()+ {+ ingress.record_mailbox_full();+ }+}+#[cfg(target_os = "macos")]async fn start_camera_capture_attempt(camera: &cap_camera::CameraInfo,@@ -929,7 +1027,7 @@ async fn start_camera_capture_attempt(recipient: Recipient<NewFrame>,native_recipient: Recipient<NewNativeFrame>,ffmpeg_sender_count: Arc<std::sync::atomic::AtomicUsize>,- native_sender_count: Arc<std::sync::atomic::AtomicUsize>,+ native_ingress: Arc<NativeCameraIngress>,) -> Result<SetupCameraResult, SetInputError> {let frame_rate = format.frame_rate().round().max(1.0) as u32;@@ -945,13 +1043,19 @@ async fn start_camera_capture_attempt(cidre::cm::Clock::convert_host_time_to_sys_units(frame.native().sample_buf().pts()),));- if native_sender_count.load(std::sync::atomic::Ordering::Relaxed) > 0 {- let _ = native_recipient- .tell(NewNativeFrame(NativeCameraFrame {+ if native_ingress+ .senders+ .load(std::sync::atomic::Ordering::Relaxed)+ > 0+ {+ hand_native_frame_to_feed(+ &native_recipient,+ &native_ingress,+ NativeCameraFrame {sample_buf: frame.native().sample_buf().clone(),timestamp,- }))- .try_send();+ },+ );}// Until the ready signal fires the first frame must still be@@ -1015,7 +1119,7 @@ async fn setup_camera(recipient: Recipient<NewFrame>,native_recipient: Recipient<NewNativeFrame>,ffmpeg_sender_count: Arc<std::sync::atomic::AtomicUsize>,- native_sender_count: Arc<std::sync::atomic::AtomicUsize>,+ native_ingress: Arc<NativeCameraIngress>,_cancel_rx: &mpsc::Receiver<()>,) -> Result<SetupCameraResult, SetInputError> {let camera = find_camera(id).ok_or(SetInputError::DeviceNotFound)?;@@ -1034,7 +1138,10 @@ async fn setup_camera(cap_timestamp::PerformanceCounterTimestamp::new(frame.native().perf_counter),);- if native_sender_count.load(std::sync::atomic::Ordering::Relaxed) > 0+ if native_ingress+ .senders+ .load(std::sync::atomic::Ordering::Relaxed)+ > 0&& let Ok(bytes) = frame.native().bytes(){use cap_mediafoundation_utils::IMFMediaBufferExt;@@ -1056,16 +1163,18 @@ async fn setup_camera(#[allow(clippy::arc_with_non_send_sync)]let buffer = std::sync::Arc::new(std::sync::Mutex::new(buffer));- let _ = native_recipient- .tell(NewNativeFrame(NativeCameraFrame {+ hand_native_frame_to_feed(+ &native_recipient,+ &native_ingress,+ NativeCameraFrame {buffer,pixel_format: frame.native().pixel_format,width: frame.native().width as u32,height: frame.native().height as u32,is_bottom_up: frame.native().is_bottom_up,timestamp,- }))- .try_send();+ },+ );}}}@@ -1131,7 +1240,7 @@ async fn setup_camera(recipient: Recipient<NewFrame>,_native_recipient: Recipient<NewNativeFrame>,_ffmpeg_sender_count: Arc<std::sync::atomic::AtomicUsize>,- _native_sender_count: Arc<std::sync::atomic::AtomicUsize>,+ _native_ingress: Arc<NativeCameraIngress>,_cancel_rx: &mpsc::Receiver<()>,) -> Result<SetupCameraResult, SetInputError> {let camera = find_camera(id).ok_or(SetInputError::DeviceNotFound)?;@@ -1267,7 +1376,7 @@ impl Message<SetInput> for CameraFeed {new_frame_recipient,native_frame_recipient,ffmpeg_sender_count: self.ffmpeg_sender_count.clone(),- native_sender_count: self.native_sender_count.clone(),+ native_ingress: self.native_ingress.clone(),flow: CameraSetupFlow::Open,});@@ -1295,7 +1404,7 @@ impl Message<SetInput> for CameraFeed {new_frame_recipient,native_frame_recipient,ffmpeg_sender_count: self.ffmpeg_sender_count.clone(),- native_sender_count: self.native_sender_count.clone(),+ native_ingress: self.native_ingress.clone(),flow: CameraSetupFlow::Locked {lock_generation: self.lock_generation,},@@ -1329,7 +1438,8 @@ impl Message<RemoveInput> for CameraFeed {self.ffmpeg_sender_count.store(0, std::sync::atomic::Ordering::Release);self.native_senders.clear();- self.native_sender_count+ self.native_ingress+ .senders.store(0, std::sync::atomic::Ordering::Release);for cb in &self.on_disconnect {@@ -1377,7 +1487,7 @@ impl Message<AddNativeSender> for CameraFeed {debug!("CameraFeed: Adding new native sender");self.native_senders.push(msg.0);- self.native_sender_count.store(+ self.native_ingress.senders.store(self.native_senders.len(),std::sync::atomic::Ordering::Release,);@@ -1408,7 +1518,7 @@ impl Message<RemoveNativeSender> for CameraFeed {) -> Self::Reply {self.native_senders.retain(|sender| !sender.same_channel(&msg.0));- self.native_sender_count.store(+ self.native_ingress.senders.store(self.native_senders.len(),std::sync::atomic::Ordering::Release,);@@ -1451,11 +1561,36 @@ impl Message<OnFeedDisconnect> for CameraFeed {static CAMERA_FRAME_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);+/// Frames a fan-out could not deliver: one per sender whose queue was full.+struct FanoutDrops<'a> {+ total: &'a std::sync::atomic::AtomicU64,+ log: &'a mut DropLog,+}++impl FanoutDrops<'_> {+ fn record(&mut self, count: u64, frame_num: u64, sender_label: &str) {+ if count == 0 {+ return;+ }+ let total = self+ .total+ .fetch_add(count, std::sync::atomic::Ordering::Relaxed)+ + count;+ if let Some(dropped) = self.log.record(count, std::time::Instant::now()) {+ warn!(+ dropped,+ total, frame_num, "{sender_label} ingress queue full, frames dropped"+ );+ }+ }+}+fn send_frame_to_camera_senders<T: Clone>(senders: &mut Vec<flume::Sender<T>>,frame: T,frame_num: u64,sender_label: &str,+ mut drops: FanoutDrops<'_>,) -> bool {// A disconnected sender whose queue is still full would otherwise never be// try_send'd again, leaving it (and its queued frames) retained forever.@@ -1471,21 +1606,18 @@ fn send_frame_to_camera_senders<T: Clone>(}let mut last_ready_sender = None;+ let mut full = 0;for (i, sender) in senders.iter().enumerate() {if sender.is_full() {- if frame_num.is_multiple_of(30) {- warn!(- "{} sender {} channel full at frame {}, dropping frame",- sender_label, i, frame_num- );- }+ full += 1;} else {last_ready_sender = Some(i);}}let Some(last_ready_sender) = last_ready_sender else {+ drops.record(full, frame_num, sender_label);return removed_disconnected;};@@ -1512,12 +1644,7 @@ fn send_frame_to_camera_senders<T: Clone>(match send_result {Ok(()) => {}Err(flume::TrySendError::Full(_)) => {- if frame_num.is_multiple_of(30) {- warn!(- "{} sender {} channel full at frame {}, dropping frame",- sender_label, i, frame_num- );- }+ full += 1;}Err(flume::TrySendError::Disconnected(_)) => {debug!(@@ -1529,6 +1656,8 @@ fn send_frame_to_camera_senders<T: Clone>(}}+ drops.record(full, frame_num, sender_label);+if to_remove.is_empty() {return removed_disconnected;}@@ -1557,7 +1686,16 @@ impl Message<NewFrame> for CameraFeed {}let frame_num = CAMERA_FRAME_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);- if send_frame_to_camera_senders(&mut self.senders, msg.0, frame_num, "Camera") {+ if send_frame_to_camera_senders(+ &mut self.senders,+ msg.0,+ frame_num,+ "Camera",+ FanoutDrops {+ total: &self.ffmpeg_fanout_full,+ log: &mut self.ffmpeg_fanout_log,+ },+ ) {self.ffmpeg_sender_count.store(self.senders.len(), std::sync::atomic::Ordering::Release);}@@ -1585,9 +1723,17 @@ impl Message<NewNativeFrame> for CameraFeed {let frame_num =NATIVE_CAMERA_FRAME_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);- if send_frame_to_camera_senders(&mut self.native_senders, msg.0, frame_num, "Native camera")- {- self.native_sender_count.store(+ if send_frame_to_camera_senders(+ &mut self.native_senders,+ msg.0,+ frame_num,+ "Native camera",+ FanoutDrops {+ total: &self.native_ingress.fanout_full,+ log: &mut self.native_fanout_log,+ },+ ) {+ self.native_ingress.senders.store(self.native_senders.len(),std::sync::atomic::Ordering::Release,);@@ -1666,6 +1812,7 @@ impl Message<Lock> for CameraFeed {Ok(CameraFeedLock {camera_info,video_info,+ native_ingress: self.native_ingress.clone(),actor: ctx.actor_ref(),drop_tx: Some(drop_tx),_token: token,@@ -2483,3 +2630,210 @@ mod tests {);}}++#[cfg(test)]+mod ingress_tests {+ use super::*;+ use crate::sources::native_camera::native_ingress_capacity;+ use cap_media_info::RawVideoFormat;++ fn fanout(+ senders: &mut Vec<flume::Sender<u32>>,+ frame: u32,+ total: &std::sync::atomic::AtomicU64,+ log: &mut DropLog,+ ) {+ send_frame_to_camera_senders(+ senders,+ frame,+ u64::from(frame),+ "Native camera",+ FanoutDrops { total, log },+ );+ }++ fn hd30() -> VideoInfo {+ VideoInfo::from_raw(RawVideoFormat::Nv12, 1280, 720, 30)+ }++ #[test]+ fn burst_after_a_two_second_stall_fits_the_ingress_queue() {+ let capacity = native_ingress_capacity(&hd30());+ let (tx, rx) = flume::bounded(capacity);+ let mut senders = vec![tx];+ let total = std::sync::atomic::AtomicU64::new(0);+ let mut log = DropLog::default();+ // The consumer is stalled for 2 s at 30 fps; the feed then hands the+ // whole backlog over at once.+ for frame in 0..60 {+ fanout(&mut senders, frame, &total, &mut log);+ }+ assert_eq!(total.load(std::sync::atomic::Ordering::Relaxed), 0);+ assert_eq!(rx.drain().collect::<Vec<_>>(), (0..60).collect::<Vec<_>>());+ }++ #[test]+ fn frames_past_the_ingress_bound_are_counted_and_reported() {+ let capacity = native_ingress_capacity(&hd30());+ let (tx, rx) = flume::bounded(capacity);+ let mut senders = vec![tx];+ let total = std::sync::atomic::AtomicU64::new(0);+ let mut log = DropLog::default();+ for frame in 0..capacity as u32 + 5 {+ fanout(&mut senders, frame, &total, &mut log);+ }+ assert_eq!(total.load(std::sync::atomic::Ordering::Relaxed), 5);+ assert_eq!(rx.len(), capacity);+ // The first drop was reported at once; the other four wait for the+ // next report window.+ assert_eq!(log.record(0, std::time::Instant::now()), None);+ assert_eq!(+ log.record(0, std::time::Instant::now() + DropLog::INTERVAL),+ Some(4)+ );+ }++ #[test]+ fn previous_eight_frame_queue_lost_most_of_the_same_burst() {+ let (tx, rx) = flume::bounded(8);+ let mut senders = vec![tx];+ let total = std::sync::atomic::AtomicU64::new(0);+ let mut log = DropLog::default();+ for frame in 0..60 {+ fanout(&mut senders, frame, &total, &mut log);+ }+ assert_eq!(total.load(std::sync::atomic::Ordering::Relaxed), 52);+ assert_eq!(rx.len(), 8);+ }++ #[test]+ fn drop_log_reports_the_first_drop_then_rate_limits() {+ let mut log = DropLog::default();+ let start = std::time::Instant::now();+ assert_eq!(log.record(0, start), None);+ assert_eq!(log.record(1, start), Some(1));+ assert_eq!(log.record(3, start + Duration::from_secs(1)), None);+ assert_eq!(log.record(1, start + Duration::from_secs(4)), None);+ assert_eq!(log.record(2, start + DropLog::INTERVAL), Some(6));+ assert_eq!(log.record(0, start + DropLog::INTERVAL * 3), None);+ }++ #[cfg(target_os = "macos")]+ mod actor {+ use super::*;++ /// Scripted runtime stall: the feed actor handles nothing else.+ struct Hold(Duration, oneshot::Sender<()>);++ impl Message<Hold> for CameraFeed {+ type Reply = ();++ async fn handle(&mut self, msg: Hold, _: &mut Context<Self, Self::Reply>) {+ let _ = msg.1.send(());+ tokio::time::sleep(msg.0).await;+ }+ }++ fn frame(index: i64) -> NativeCameraFrame {+ use cidre::{cm, cv};+ let buf = cv::PixelBuf::new(16, 16, cv::PixelFormat::_420V, None).unwrap();+ let desc = cm::VideoFormatDesc::with_image_buf(&buf).unwrap();+ let timing = cm::SampleTimingInfo {+ duration: cm::Time::new(1, 30),+ pts: cm::Time::new(index, 30),+ dts: cm::Time::invalid(),+ };+ let sample_buf =+ cm::SampleBuf::with_image_buf(&buf, true, None, std::ptr::null(), &desc, &timing)+ .unwrap();+ NativeCameraFrame {+ sample_buf,+ timestamp: Timestamp::MachAbsoluteTime(cap_timestamp::MachAbsoluteTimestamp::new(+ index as u64,+ )),+ }+ }++ async fn eventually(what: &str, check: impl Fn() -> bool) {+ let deadline = std::time::Instant::now() + Duration::from_secs(10);+ while !check() {+ assert!(std::time::Instant::now() < deadline, "timed out: {what}");+ tokio::time::sleep(Duration::from_millis(5)).await;+ }+ }++ async fn stalled_feed(+ capacity: usize,+ hold: Duration,+ ) -> (+ ActorRef<CameraFeed>,+ Arc<NativeCameraIngress>,+ flume::Receiver<NativeCameraFrame>,+ ) {+ let ingress = Arc::new(NativeCameraIngress::default());+ let feed = CameraFeed::spawn(CameraFeed {+ native_ingress: ingress.clone(),+ ..CameraFeed::default()+ });+ let (tx, rx) = flume::bounded(capacity);+ feed.ask(AddNativeSender(tx)).await.unwrap();+ let (held_tx, held_rx) = oneshot::channel();+ feed.tell(Hold(hold, held_tx)).await.unwrap();+ held_rx.await.unwrap();+ (feed, ingress, rx)+ }++ /// The capture callback keeps delivering while the feed is stalled;+ /// after the stall the feed fans the backlog out in one burst and the+ /// recording receives every frame, in order.+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn stall_then_burst_keeps_every_frame() {+ let capacity = native_ingress_capacity(&hd30());+ let (feed, ingress, rx) = stalled_feed(capacity, Duration::from_millis(1500)).await;+ let recipient = feed.clone().recipient::<NewNativeFrame>();+ let frames: Vec<_> = (0..45).map(frame).collect();+ for frame in frames {+ hand_native_frame_to_feed(&recipient, &ingress, frame);+ }+ eventually("burst delivered", || rx.len() == 45).await;+ assert_eq!(ingress.drops(), NativeCameraIngressDrops::default());+ let pts: Vec<_> = rx+ .drain()+ .map(|frame| frame.sample_buf.pts().value)+ .collect();+ assert_eq!(pts, (0..45).collect::<Vec<_>>());+ }++ /// Past the mailbox and the ingress queue every lost frame is+ /// counted, and the callback's hand-over never waits for the stalled+ /// feed.+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn losses_past_the_bound_are_counted_without_blocking_the_callback() {+ let capacity = native_ingress_capacity(&hd30());+ let (feed, ingress, rx) = stalled_feed(capacity, Duration::from_millis(1000)).await;+ let recipient = feed.clone().recipient::<NewNativeFrame>();+ let frames: Vec<_> = (0..70).map(frame).collect();+ let mut slowest = Duration::ZERO;+ for frame in frames {+ let start = std::time::Instant::now();+ hand_native_frame_to_feed(&recipient, &ingress, frame);+ slowest = slowest.max(start.elapsed());+ }+ assert!(+ slowest < Duration::from_millis(50),+ "callback hand-over waited {slowest:?}"+ );+ // The mailbox holds 64 frames; the other six are lost and counted.+ assert_eq!(ingress.drops().mailbox_full, 6);+ eventually("backlog delivered", || rx.len() == capacity).await;+ assert_eq!(ingress.drops().fanout_full, 0);+ // The recording still does not drain: the next frames overflow+ // its ingress queue and are counted.+ for frame in (70..73).map(frame) {+ hand_native_frame_to_feed(&recipient, &ingress, frame);+ }+ eventually("overflow counted", || ingress.drops().fanout_full == 3).await;+ assert_eq!(rx.len(), capacity);+ }+ }+}diff --git a/crates/recording/src/sources/native_camera.rs b/crates/recording/src/sources/native_camera.rsindex abb50eff2..8e8036804 100644--- a/crates/recording/src/sources/native_camera.rs+++ b/crates/recording/src/sources/native_camera.rs@@ -1,36 +1,206 @@use crate::{- feeds::camera::{self, CameraFeedLock},- output_pipeline::{NativeCameraFrame, SetupCtx, VideoSource},+ feeds::camera::{self, CameraFeedLock, DropLog, NativeCameraIngress, NativeCameraIngressDrops},+ output_pipeline::{+ HealthSender, NativeCameraFrame, PipelineHealthEvent, SetupCtx, VideoSource, emit_health,+ },};use anyhow::anyhow;-use cap_media_info::VideoInfo;+use cap_media_info::{Pixel, VideoInfo};use futures::{FutureExt, channel::mpsc, future::BoxFuture};use std::sync::{Arc,atomic::{AtomicBool, Ordering},};-use std::time::Duration;+use std::time::{Duration, Instant};use tokio::sync::oneshot;const CAMERA_SENDER_ATTACH_TIMEOUT: Duration = Duration::from_millis(1500);+/// How long the recording may stop draining camera frames (a runtime stall)+/// before the ingress queue fills.+const INGRESS_TIME_BUDGET_SECS: usize = 2;+/// The camera feed's actor mailbox (kameo's default bounded mailbox). After a+/// stall the feed hands its whole backlog over in one burst, so the ingress+/// queue must take that many frames at once.+const CAMERA_FEED_MAILBOX_FRAMES: usize = 64;+/// Upper bound on the pixel data the ingress queue can keep alive. On macOS a+/// queued frame retains a capture pool surface (the pool grows per retained+/// buffer); on Windows it is a heap copy of the frame.+const INGRESS_BYTE_BUDGET: usize = 256 * 1024 * 1024;+/// The previous fixed capacity; never go below it.+const MIN_INGRESS_FRAMES: usize = 8;++fn frame_bytes(info: &VideoInfo) -> usize {+ let pixels = info.width as usize * info.height as usize;+ match info.pixel_format {+ Pixel::NV12 | Pixel::NV21 | Pixel::YUV420P | Pixel::YUVJ420P => pixels * 3 / 2,+ Pixel::UYVY422 | Pixel::YUYV422 | Pixel::YUV422P | Pixel::YUVJ422P | Pixel::NV16 => {+ pixels * 2+ }+ _ => pixels * 4,+ }+}++/// Frames the native ingress queue holds: the time budget at the camera's+/// frame rate (at least one full feed mailbox), capped by the byte budget.+pub(crate) fn native_ingress_capacity(info: &VideoInfo) -> usize {+ let by_time = (info.fps().max(1) as usize).saturating_mul(INGRESS_TIME_BUDGET_SECS);+ let by_bytes = INGRESS_BYTE_BUDGET / frame_bytes(info).max(1);+ by_time+ .max(CAMERA_FEED_MAILBOX_FRAMES)+ .min(by_bytes)+ .max(MIN_INGRESS_FRAMES)+}+pub struct NativeCamera {feed_lock: Arc<CameraFeedLock>,stop_tx: Option<oneshot::Sender<()>>,stopped: Arc<AtomicBool>,}+/// Camera frames this recording lost, by boundary.+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub(crate) struct IngressSummary {+ pub received: u64,+ pub sent: u64,+ pub feed: NativeCameraIngressDrops,+ pub forwarder_full: u64,+}++impl IngressSummary {+ fn lost(&self) -> u64 {+ self.feed.mailbox_full + self.feed.fanout_full + self.forwarder_full+ }+}++/// Reports every camera frame this recording loses: in the feed (its mailbox+/// or this recording's ingress queue) and between this source and the+/// encoder. The first loss at each boundary is logged at warn and sent as a+/// health event at once; later losses at most every five seconds; the totals+/// at the end.+pub(crate) struct IngressReporter {+ feed: Option<Arc<NativeCameraIngress>>,+ start: NativeCameraIngressDrops,+ seen: NativeCameraIngressDrops,+ health_tx: Option<HealthSender>,+ feed_log: DropLog,+ forwarder_log: DropLog,+ summary: IngressSummary,+}++impl IngressReporter {+ pub(crate) fn new(+ feed: Option<Arc<NativeCameraIngress>>,+ health_tx: Option<HealthSender>,+ ) -> Self {+ let start = feed.as_ref().map(|feed| feed.drops()).unwrap_or_default();+ Self {+ feed,+ start,+ seen: start,+ health_tx,+ feed_log: DropLog::default(),+ forwarder_log: DropLog::default(),+ summary: IngressSummary::default(),+ }+ }++ fn received(&mut self) {+ self.summary.received += 1;+ self.poll_feed(Instant::now());+ }++ fn sent(&mut self) {+ self.summary.sent += 1;+ }++ fn poll_feed(&mut self, now: Instant) {+ let Some(feed) = &self.feed else {+ return;+ };+ let current = feed.drops();+ let new = current.since(self.seen);+ if new == NativeCameraIngressDrops::default() {+ return;+ }+ self.seen = current;+ self.summary.feed = current.since(self.start);+ if let Some(dropped) = self+ .feed_log+ .record(new.mailbox_full + new.fanout_full, now)+ {+ tracing::warn!(+ dropped,+ mailbox_full = self.summary.feed.mailbox_full,+ ingress_full = self.summary.feed.fanout_full,+ "Native camera frames lost before the recording received them"+ );+ self.emit("camera-ingress:feed");+ }+ }++ fn forwarder_full(&mut self, now: Instant) {+ self.summary.forwarder_full += 1;+ if let Some(dropped) = self.forwarder_log.record(1, now) {+ tracing::warn!(+ dropped,+ total = self.summary.forwarder_full,+ "Native camera encoder queue full, frames dropped"+ );+ self.emit("camera-ingress:encoder-queue");+ }+ }++ fn emit(&self, source: &str) {+ let Some(health_tx) = &self.health_tx else {+ return;+ };+ let lost = self.summary.lost();+ let offered =+ self.summary.received + self.summary.feed.mailbox_full + self.summary.feed.fanout_full;+ emit_health(+ health_tx,+ PipelineHealthEvent::FrameDropRateHigh {+ source: source.to_string(),+ rate_pct: 100.0 * lost as f64 / offered.max(1) as f64,+ },+ );+ }++ fn finish(mut self, elapsed: Duration) -> IngressSummary {+ self.poll_feed(Instant::now());+ let summary = self.summary;+ if summary.lost() > 0 {+ tracing::warn!(+ received = summary.received,+ sent = summary.sent,+ mailbox_full = summary.feed.mailbox_full,+ ingress_full = summary.feed.fanout_full,+ encoder_queue_full = summary.forwarder_full,+ ?elapsed,+ "Native camera source finished with lost frames"+ );+ } else {+ tracing::info!(+ "Native camera source finished: {} received, {} sent, 0 dropped in {:?}",+ summary.received,+ summary.sent,+ elapsed+ );+ }+ summary+ }+}+async fn forward_native_frames<T: Send + 'static>(rx: flume::Receiver<T>,video_tx: mpsc::Sender<T>,stop_rx: oneshot::Receiver<()>,stopped: Arc<AtomicBool>,required: bool,-) -> anyhow::Result<()> {+ mut report: IngressReporter,+) -> anyhow::Result<IngressSummary> {tracing::debug!("Native camera source task started");- let mut frame_count: u64 = 0;- let mut sent_count: u64 = 0;- let mut dropped_count: u64 = 0;let start = std::time::Instant::now();let mut video_tx = video_tx;let mut stop_rx = stop_rx.fuse();@@ -50,33 +220,16 @@ async fn forward_native_frames<T: Send + 'static>(result = rx.recv_async() => {match result {Ok(frame) => {- frame_count += 1;+ report.received();match video_tx.try_send(frame) {- Ok(()) => {- sent_count += 1;- if sent_count.is_multiple_of(30) {- tracing::debug!(- "Native camera source: sent {} frames, dropped {} in {:?}",- sent_count,- dropped_count,- start.elapsed()- );- }- }+ Ok(()) => report.sent(),Err(e) => {if e.is_full() {- dropped_count += 1;- if dropped_count.is_multiple_of(30) {- tracing::warn!(- "Native camera source: encoder can't keep up, dropped {} frames so far",- dropped_count- );- }+ report.forwarder_full(Instant::now());} else if e.is_disconnected() {tracing::debug!(- "Native camera source: pipeline closed after {} sent, {} dropped",- sent_count,- dropped_count+ "Native camera source: pipeline closed after {:?}",+ report.summary);if required && !stopped.load(Ordering::Acquire) {return Err(anyhow!("Native camera encoder disconnected before Stop"));@@ -89,7 +242,7 @@ async fn forward_native_frames<T: Send + 'static>(Err(e) => {tracing::debug!("Native camera feed disconnected (rx closed) after {} frames in {:?}: {e}",- frame_count,+ report.summary.received,start.elapsed());if required && !stopped.load(Ordering::Acquire) {@@ -104,14 +257,7 @@ async fn forward_native_frames<T: Send + 'static>(drop(video_tx);- tracing::info!(- "Native camera source finished: {} received, {} sent, {} dropped in {:?}",- frame_count,- sent_count,- dropped_count,- start.elapsed()- );- Ok(())+ Ok(report.finish(start.elapsed()))}impl VideoSource for NativeCamera {@@ -121,12 +267,14 @@ impl VideoSource for NativeCamera {async fn setup(feed_lock: Self::Config,video_tx: mpsc::Sender<Self::Frame>,- _ctx: &mut SetupCtx,+ ctx: &mut SetupCtx,) -> anyhow::Result<Self>whereSelf: Sized,{- let (tx, rx) = flume::bounded(8);+ let capacity = native_ingress_capacity(feed_lock.video_info());+ tracing::debug!(capacity, "Native camera ingress queue");+ let (tx, rx) = flume::bounded(capacity);tokio::time::timeout(CAMERA_SENDER_ATTACH_TIMEOUT,@@ -140,9 +288,15 @@ impl VideoSource for NativeCamera {let stopped = Arc::new(AtomicBool::new(false));let stopped_clone = stopped.clone();- let task = forward_native_frames(rx, video_tx, stop_rx, stopped_clone, cfg!(windows));+ let report = IngressReporter::new(+ Some(feed_lock.native_ingress().clone()),+ Some(ctx.health_tx().clone()),+ );+ let task =+ forward_native_frames(rx, video_tx, stop_rx, stopped_clone, cfg!(windows), report)+ .map(|result| result.map(drop));#[cfg(windows)]- _ctx.tasks().spawn("native-camera", task);+ ctx.tasks().spawn("native-camera", task);#[cfg(not(windows))]drop(tokio::spawn(task));@@ -187,6 +341,7 @@ mod forwarding_tests {stopped,Arc::new(AtomicBool::new(false)),true,+ IngressReporter::new(None, None),).await;assert!(@@ -210,6 +365,7 @@ mod forwarding_tests {stopped,Arc::new(AtomicBool::new(false)),true,+ IngressReporter::new(None, None),).await.unwrap();@@ -227,6 +383,7 @@ mod forwarding_tests {stopped,Arc::new(AtomicBool::new(false)),true,+ IngressReporter::new(None, None),));sender.send_async(17_u8).await.unwrap();assert_eq!(frames.next().await, Some(17));@@ -249,6 +406,7 @@ mod forwarding_tests {stopped,Arc::new(AtomicBool::new(false)),true,+ IngressReporter::new(None, None),).await;assert!(@@ -271,8 +429,97 @@ mod forwarding_tests {stopped,Arc::new(AtomicBool::new(false)),false,+ IngressReporter::new(None, None),+ )+ .await+ .unwrap();+ }++ fn info(format: cap_media_info::RawVideoFormat, w: u32, h: u32, fps: u32) -> VideoInfo {+ VideoInfo::from_raw(format, w, h, fps)+ }++ #[test]+ fn ingress_capacity_is_sized_by_time_and_bounded_by_bytes() {+ use cap_media_info::RawVideoFormat::{Bgra, Nv12, Uyvy};+ let cases = [+ (info(Nv12, 1280, 720, 30), 64),+ (info(Nv12, 1920, 1080, 30), 64),+ (info(Nv12, 1280, 720, 60), 120),+ (info(Nv12, 1920, 1080, 60), 86),+ (info(Uyvy, 1920, 1080, 30), 64),+ (info(Nv12, 3840, 2160, 30), 21),+ (info(Bgra, 3840, 2160, 30), MIN_INGRESS_FRAMES),+ ];+ for (info, expected) in cases {+ let capacity = native_ingress_capacity(&info);+ assert_eq!(+ capacity,+ expected,+ "{}x{}@{} {:?}",+ info.width,+ info.height,+ info.fps(),+ info.pixel_format+ );+ assert!(+ capacity == MIN_INGRESS_FRAMES+ || capacity * frame_bytes(&info) <= INGRESS_BYTE_BUDGET+ );+ }+ }++ #[tokio::test]+ async fn encoder_queue_overflow_is_counted() {+ let (sender, receiver) = flume::bounded(16);+ let (video, frames) = mpsc::channel(1);+ let (_stop, stopped) = oneshot::channel();+ for frame in 0..10_u8 {+ sender.send_async(frame).await.unwrap();+ }+ drop(sender);+ let summary = forward_native_frames(+ receiver,+ video,+ stopped,+ Arc::new(AtomicBool::new(false)),+ false,+ IngressReporter::new(None, None),+ )+ .await+ .unwrap();+ // futures mpsc holds its buffer plus one slot per sender.+ assert_eq!(summary.received, 10);+ assert_eq!(summary.sent, 2);+ assert_eq!(summary.forwarder_full, 8);+ drop(frames);+ }++ #[tokio::test]+ async fn feed_losses_count_from_the_recording_start() {+ let ingress = Arc::new(NativeCameraIngress::default());+ ingress.record_mailbox_full();+ let report = IngressReporter::new(Some(ingress.clone()), None);+ ingress.record_mailbox_full();+ ingress.record_mailbox_full();+ let (sender, receiver) = flume::bounded(4);+ let (video, mut frames) = mpsc::channel(4);+ let (_stop, stopped) = oneshot::channel();+ sender.send_async(1_u8).await.unwrap();+ drop(sender);+ let summary = forward_native_frames(+ receiver,+ video,+ stopped,+ Arc::new(AtomicBool::new(false)),+ false,+ report,).await.unwrap();+ assert_eq!(summary.feed.mailbox_full, 2);+ assert_eq!(summary.feed.fanout_full, 0);+ assert_eq!(summary.sent, 1);+ assert_eq!(futures::StreamExt::next(&mut frames).await, Some(1));}}