Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(recording): size camera ingress by time and count every lost frame

Richie McIlroy9b4aa5edb9be
diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs
index 28d822b4b..bbd025587 100644
--- a/crates/recording/src/feeds/camera.rs
+++ b/crates/recording/src/feeds/camera.rs
@@ -54,11 +54,86 @@ pub struct CameraFeed {
senders: Vec<flume::Sender<FFmpegVideoFrame>>,
ffmpeg_sender_count: Arc<std::sync::atomic::AtomicUsize>,
native_senders: Vec<flume::Sender<NativeCameraFrame>>,
- native_sender_count: Arc<std::sync::atomic::AtomicUsize>,
+ native_ingress: Arc<NativeCameraIngress>,
+ native_fanout_log: DropLog,
+ ffmpeg_fanout_full: std::sync::atomic::AtomicU64,
+ ffmpeg_fanout_log: DropLog,
on_ready: Vec<oneshot::Sender<()>>,
on_disconnect: Vec<Box<dyn Fn() + Send>>,
}
+/// Shared between the camera feed, its capture callback and the native
+/// sources of the recordings attached to it.
+#[derive(Debug, Default)]
+pub struct NativeCameraIngress {
+ senders: std::sync::atomic::AtomicUsize,
+ mailbox_full: std::sync::atomic::AtomicU64,
+ fanout_full: std::sync::atomic::AtomicU64,
+}
+
+/// Native camera frames lost inside the feed, before a recording's source
+/// received them. Monotonic over the feed's lifetime; recordings report the
+/// difference between their start and the current value.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct NativeCameraIngressDrops {
+ /// The capture callback found the feed's mailbox full (the feed actor
+ /// was not running, for example during a runtime stall).
+ pub mailbox_full: u64,
+ /// A recording's ingress queue was full when the feed fanned a frame out.
+ pub fanout_full: u64,
+}
+
+impl NativeCameraIngressDrops {
+ pub fn since(self, start: Self) -> Self {
+ Self {
+ mailbox_full: self.mailbox_full.saturating_sub(start.mailbox_full),
+ fanout_full: self.fanout_full.saturating_sub(start.fanout_full),
+ }
+ }
+}
+
+impl NativeCameraIngress {
+ pub fn drops(&self) -> NativeCameraIngressDrops {
+ NativeCameraIngressDrops {
+ mailbox_full: self.mailbox_full.load(std::sync::atomic::Ordering::Relaxed),
+ fanout_full: self.fanout_full.load(std::sync::atomic::Ordering::Relaxed),
+ }
+ }
+
+ /// Called from capture callbacks: a relaxed increment, never a wait.
+ pub fn record_mailbox_full(&self) {
+ self.mailbox_full
+ .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ }
+}
+
+/// Rate limit for drop warnings: the first drop is reported at once, later
+/// drops at most once per interval with the count since the last report, so
+/// an isolated loss is never hidden and a sustained loss cannot flood the log.
+#[derive(Debug, Default)]
+pub(crate) struct DropLog {
+ last_report: Option<std::time::Instant>,
+ unreported: u64,
+}
+
+impl DropLog {
+ pub(crate) const INTERVAL: Duration = Duration::from_secs(5);
+
+ /// Adds `count` drops. Returns the drops to report now, if a report is due.
+ pub(crate) fn record(&mut self, count: u64, now: std::time::Instant) -> Option<u64> {
+ self.unreported = self.unreported.saturating_add(count);
+ if self.unreported == 0
+ || self
+ .last_report
+ .is_some_and(|last| now.duration_since(last) < Self::INTERVAL)
+ {
+ return None;
+ }
+ self.last_report = Some(now);
+ Some(std::mem::take(&mut self.unreported))
+ }
+}
+
enum State {
Open(OpenState),
Locked {
@@ -210,7 +285,10 @@ impl Default for CameraFeed {
senders: Vec::new(),
ffmpeg_sender_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
native_senders: Vec::new(),
- native_sender_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
+ native_ingress: Arc::new(NativeCameraIngress::default()),
+ native_fanout_log: DropLog::default(),
+ ffmpeg_fanout_full: std::sync::atomic::AtomicU64::new(0),
+ ffmpeg_fanout_log: DropLog::default(),
on_ready: Vec::new(),
on_disconnect: Vec::new(),
}
@@ -229,11 +307,16 @@ pub struct CameraFeedLock {
actor: ActorRef<CameraFeed>,
camera_info: cap_camera::CameraInfo,
video_info: VideoInfo,
+ native_ingress: Arc<NativeCameraIngress>,
drop_tx: Option<oneshot::Sender<()>>,
_token: Arc<()>,
}
impl CameraFeedLock {
+ pub fn native_ingress(&self) -> &Arc<NativeCameraIngress> {
+ &self.native_ingress
+ }
+
pub fn camera_info(&self) -> &cap_camera::CameraInfo {
&self.camera_info
}
@@ -369,7 +452,7 @@ struct CameraSetupArgs {
new_frame_recipient: Recipient<NewFrame>,
native_frame_recipient: Recipient<NewNativeFrame>,
ffmpeg_sender_count: Arc<std::sync::atomic::AtomicUsize>,
- native_sender_count: Arc<std::sync::atomic::AtomicUsize>,
+ native_ingress: Arc<NativeCameraIngress>,
flow: CameraSetupFlow,
}
@@ -432,7 +515,7 @@ fn spawn_camera_setup(args: CameraSetupArgs) -> (ReadyFuture, SyncSender<()>) {
new_frame_recipient,
native_frame_recipient,
ffmpeg_sender_count,
- native_sender_count,
+ native_ingress,
flow,
} = args;
@@ -498,7 +581,7 @@ fn spawn_camera_setup(args: CameraSetupArgs) -> (ReadyFuture, SyncSender<()>) {
new_frame_recipient,
native_frame_recipient,
ffmpeg_sender_count,
- native_sender_count,
+ native_ingress,
&done_rx_thread,
)
.await;
@@ -857,7 +940,7 @@ async fn setup_camera(
recipient: Recipient<NewFrame>,
native_recipient: Recipient<NewNativeFrame>,
ffmpeg_sender_count: Arc<std::sync::atomic::AtomicUsize>,
- native_sender_count: Arc<std::sync::atomic::AtomicUsize>,
+ native_ingress: Arc<NativeCameraIngress>,
cancel_rx: &mpsc::Receiver<()>,
) -> Result<SetupCameraResult, SetInputError> {
let camera = find_camera(id).ok_or(SetInputError::DeviceNotFound)?;
@@ -882,7 +965,7 @@ async fn setup_camera(
recipient.clone(),
native_recipient.clone(),
ffmpeg_sender_count.clone(),
- native_sender_count.clone(),
+ native_ingress.clone(),
)
.await;
@@ -910,7 +993,7 @@ async fn setup_camera(
recipient,
native_recipient,
ffmpeg_sender_count,
- native_sender_count,
+ native_ingress,
)
.await
.inspect(|_| {
@@ -921,6 +1004,21 @@ async fn setup_camera(
}
}
+/// The capture callback's hand-over to the feed: never waits for mailbox
+/// capacity; a full mailbox loses the frame and counts it.
+#[cfg(any(target_os = "macos", windows))]
+fn hand_native_frame_to_feed(
+ recipient: &Recipient<NewNativeFrame>,
+ ingress: &NativeCameraIngress,
+ frame: NativeCameraFrame,
+) {
+ if let Err(kameo::error::SendError::MailboxFull(_)) =
+ recipient.tell(NewNativeFrame(frame)).try_send()
+ {
+ ingress.record_mailbox_full();
+ }
+}
+
#[cfg(target_os = "macos")]
async fn start_camera_capture_attempt(
camera: &cap_camera::CameraInfo,
@@ -929,7 +1027,7 @@ async fn start_camera_capture_attempt(
recipient: Recipient<NewFrame>,
native_recipient: Recipient<NewNativeFrame>,
ffmpeg_sender_count: Arc<std::sync::atomic::AtomicUsize>,
- native_sender_count: Arc<std::sync::atomic::AtomicUsize>,
+ native_ingress: Arc<NativeCameraIngress>,
) -> Result<SetupCameraResult, SetInputError> {
let frame_rate = format.frame_rate().round().max(1.0) as u32;
@@ -945,13 +1043,19 @@ async fn start_camera_capture_attempt(
cidre::cm::Clock::convert_host_time_to_sys_units(frame.native().sample_buf().pts()),
));
- if native_sender_count.load(std::sync::atomic::Ordering::Relaxed) > 0 {
- let _ = native_recipient
- .tell(NewNativeFrame(NativeCameraFrame {
+ if native_ingress
+ .senders
+ .load(std::sync::atomic::Ordering::Relaxed)
+ > 0
+ {
+ hand_native_frame_to_feed(
+ &native_recipient,
+ &native_ingress,
+ NativeCameraFrame {
sample_buf: frame.native().sample_buf().clone(),
timestamp,
- }))
- .try_send();
+ },
+ );
}
// Until the ready signal fires the first frame must still be
@@ -1015,7 +1119,7 @@ async fn setup_camera(
recipient: Recipient<NewFrame>,
native_recipient: Recipient<NewNativeFrame>,
ffmpeg_sender_count: Arc<std::sync::atomic::AtomicUsize>,
- native_sender_count: Arc<std::sync::atomic::AtomicUsize>,
+ native_ingress: Arc<NativeCameraIngress>,
_cancel_rx: &mpsc::Receiver<()>,
) -> Result<SetupCameraResult, SetInputError> {
let camera = find_camera(id).ok_or(SetInputError::DeviceNotFound)?;
@@ -1034,7 +1138,10 @@ async fn setup_camera(
cap_timestamp::PerformanceCounterTimestamp::new(frame.native().perf_counter),
);
- if native_sender_count.load(std::sync::atomic::Ordering::Relaxed) > 0
+ if native_ingress
+ .senders
+ .load(std::sync::atomic::Ordering::Relaxed)
+ > 0
&& let Ok(bytes) = frame.native().bytes()
{
use cap_mediafoundation_utils::IMFMediaBufferExt;
@@ -1056,16 +1163,18 @@ async fn setup_camera(
#[allow(clippy::arc_with_non_send_sync)]
let buffer = std::sync::Arc::new(std::sync::Mutex::new(buffer));
- let _ = native_recipient
- .tell(NewNativeFrame(NativeCameraFrame {
+ hand_native_frame_to_feed(
+ &native_recipient,
+ &native_ingress,
+ NativeCameraFrame {
buffer,
pixel_format: frame.native().pixel_format,
width: frame.native().width as u32,
height: frame.native().height as u32,
is_bottom_up: frame.native().is_bottom_up,
timestamp,
- }))
- .try_send();
+ },
+ );
}
}
}
@@ -1131,7 +1240,7 @@ async fn setup_camera(
recipient: Recipient<NewFrame>,
_native_recipient: Recipient<NewNativeFrame>,
_ffmpeg_sender_count: Arc<std::sync::atomic::AtomicUsize>,
- _native_sender_count: Arc<std::sync::atomic::AtomicUsize>,
+ _native_ingress: Arc<NativeCameraIngress>,
_cancel_rx: &mpsc::Receiver<()>,
) -> Result<SetupCameraResult, SetInputError> {
let camera = find_camera(id).ok_or(SetInputError::DeviceNotFound)?;
@@ -1267,7 +1376,7 @@ impl Message<SetInput> for CameraFeed {
new_frame_recipient,
native_frame_recipient,
ffmpeg_sender_count: self.ffmpeg_sender_count.clone(),
- native_sender_count: self.native_sender_count.clone(),
+ native_ingress: self.native_ingress.clone(),
flow: CameraSetupFlow::Open,
});
@@ -1295,7 +1404,7 @@ impl Message<SetInput> for CameraFeed {
new_frame_recipient,
native_frame_recipient,
ffmpeg_sender_count: self.ffmpeg_sender_count.clone(),
- native_sender_count: self.native_sender_count.clone(),
+ native_ingress: self.native_ingress.clone(),
flow: CameraSetupFlow::Locked {
lock_generation: self.lock_generation,
},
@@ -1329,7 +1438,8 @@ impl Message<RemoveInput> for CameraFeed {
self.ffmpeg_sender_count
.store(0, std::sync::atomic::Ordering::Release);
self.native_senders.clear();
- self.native_sender_count
+ self.native_ingress
+ .senders
.store(0, std::sync::atomic::Ordering::Release);
for cb in &self.on_disconnect {
@@ -1377,7 +1487,7 @@ impl Message<AddNativeSender> for CameraFeed {
debug!("CameraFeed: Adding new native sender");
self.native_senders.push(msg.0);
- self.native_sender_count.store(
+ self.native_ingress.senders.store(
self.native_senders.len(),
std::sync::atomic::Ordering::Release,
);
@@ -1408,7 +1518,7 @@ impl Message<RemoveNativeSender> for CameraFeed {
) -> Self::Reply {
self.native_senders
.retain(|sender| !sender.same_channel(&msg.0));
- self.native_sender_count.store(
+ self.native_ingress.senders.store(
self.native_senders.len(),
std::sync::atomic::Ordering::Release,
);
@@ -1451,11 +1561,36 @@ impl Message<OnFeedDisconnect> for CameraFeed {
static CAMERA_FRAME_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+/// Frames a fan-out could not deliver: one per sender whose queue was full.
+struct FanoutDrops<'a> {
+ total: &'a std::sync::atomic::AtomicU64,
+ log: &'a mut DropLog,
+}
+
+impl FanoutDrops<'_> {
+ fn record(&mut self, count: u64, frame_num: u64, sender_label: &str) {
+ if count == 0 {
+ return;
+ }
+ let total = self
+ .total
+ .fetch_add(count, std::sync::atomic::Ordering::Relaxed)
+ + count;
+ if let Some(dropped) = self.log.record(count, std::time::Instant::now()) {
+ warn!(
+ dropped,
+ total, frame_num, "{sender_label} ingress queue full, frames dropped"
+ );
+ }
+ }
+}
+
fn send_frame_to_camera_senders<T: Clone>(
senders: &mut Vec<flume::Sender<T>>,
frame: T,
frame_num: u64,
sender_label: &str,
+ mut drops: FanoutDrops<'_>,
) -> bool {
// A disconnected sender whose queue is still full would otherwise never be
// try_send'd again, leaving it (and its queued frames) retained forever.
@@ -1471,21 +1606,18 @@ fn send_frame_to_camera_senders<T: Clone>(
}
let mut last_ready_sender = None;
+ let mut full = 0;
for (i, sender) in senders.iter().enumerate() {
if sender.is_full() {
- if frame_num.is_multiple_of(30) {
- warn!(
- "{} sender {} channel full at frame {}, dropping frame",
- sender_label, i, frame_num
- );
- }
+ full += 1;
} else {
last_ready_sender = Some(i);
}
}
let Some(last_ready_sender) = last_ready_sender else {
+ drops.record(full, frame_num, sender_label);
return removed_disconnected;
};
@@ -1512,12 +1644,7 @@ fn send_frame_to_camera_senders<T: Clone>(
match send_result {
Ok(()) => {}
Err(flume::TrySendError::Full(_)) => {
- if frame_num.is_multiple_of(30) {
- warn!(
- "{} sender {} channel full at frame {}, dropping frame",
- sender_label, i, frame_num
- );
- }
+ full += 1;
}
Err(flume::TrySendError::Disconnected(_)) => {
debug!(
@@ -1529,6 +1656,8 @@ fn send_frame_to_camera_senders<T: Clone>(
}
}
+ drops.record(full, frame_num, sender_label);
+
if to_remove.is_empty() {
return removed_disconnected;
}
@@ -1557,7 +1686,16 @@ impl Message<NewFrame> for CameraFeed {
}
let frame_num = CAMERA_FRAME_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
- if send_frame_to_camera_senders(&mut self.senders, msg.0, frame_num, "Camera") {
+ if send_frame_to_camera_senders(
+ &mut self.senders,
+ msg.0,
+ frame_num,
+ "Camera",
+ FanoutDrops {
+ total: &self.ffmpeg_fanout_full,
+ log: &mut self.ffmpeg_fanout_log,
+ },
+ ) {
self.ffmpeg_sender_count
.store(self.senders.len(), std::sync::atomic::Ordering::Release);
}
@@ -1585,9 +1723,17 @@ impl Message<NewNativeFrame> for CameraFeed {
let frame_num =
NATIVE_CAMERA_FRAME_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
- if send_frame_to_camera_senders(&mut self.native_senders, msg.0, frame_num, "Native camera")
- {
- self.native_sender_count.store(
+ if send_frame_to_camera_senders(
+ &mut self.native_senders,
+ msg.0,
+ frame_num,
+ "Native camera",
+ FanoutDrops {
+ total: &self.native_ingress.fanout_full,
+ log: &mut self.native_fanout_log,
+ },
+ ) {
+ self.native_ingress.senders.store(
self.native_senders.len(),
std::sync::atomic::Ordering::Release,
);
@@ -1666,6 +1812,7 @@ impl Message<Lock> for CameraFeed {
Ok(CameraFeedLock {
camera_info,
video_info,
+ native_ingress: self.native_ingress.clone(),
actor: ctx.actor_ref(),
drop_tx: Some(drop_tx),
_token: token,
@@ -2483,3 +2630,210 @@ mod tests {
);
}
}
+
+#[cfg(test)]
+mod ingress_tests {
+ use super::*;
+ use crate::sources::native_camera::native_ingress_capacity;
+ use cap_media_info::RawVideoFormat;
+
+ fn fanout(
+ senders: &mut Vec<flume::Sender<u32>>,
+ frame: u32,
+ total: &std::sync::atomic::AtomicU64,
+ log: &mut DropLog,
+ ) {
+ send_frame_to_camera_senders(
+ senders,
+ frame,
+ u64::from(frame),
+ "Native camera",
+ FanoutDrops { total, log },
+ );
+ }
+
+ fn hd30() -> VideoInfo {
+ VideoInfo::from_raw(RawVideoFormat::Nv12, 1280, 720, 30)
+ }
+
+ #[test]
+ fn burst_after_a_two_second_stall_fits_the_ingress_queue() {
+ let capacity = native_ingress_capacity(&hd30());
+ let (tx, rx) = flume::bounded(capacity);
+ let mut senders = vec![tx];
+ let total = std::sync::atomic::AtomicU64::new(0);
+ let mut log = DropLog::default();
+ // The consumer is stalled for 2 s at 30 fps; the feed then hands the
+ // whole backlog over at once.
+ for frame in 0..60 {
+ fanout(&mut senders, frame, &total, &mut log);
+ }
+ assert_eq!(total.load(std::sync::atomic::Ordering::Relaxed), 0);
+ assert_eq!(rx.drain().collect::<Vec<_>>(), (0..60).collect::<Vec<_>>());
+ }
+
+ #[test]
+ fn frames_past_the_ingress_bound_are_counted_and_reported() {
+ let capacity = native_ingress_capacity(&hd30());
+ let (tx, rx) = flume::bounded(capacity);
+ let mut senders = vec![tx];
+ let total = std::sync::atomic::AtomicU64::new(0);
+ let mut log = DropLog::default();
+ for frame in 0..capacity as u32 + 5 {
+ fanout(&mut senders, frame, &total, &mut log);
+ }
+ assert_eq!(total.load(std::sync::atomic::Ordering::Relaxed), 5);
+ assert_eq!(rx.len(), capacity);
+ // The first drop was reported at once; the other four wait for the
+ // next report window.
+ assert_eq!(log.record(0, std::time::Instant::now()), None);
+ assert_eq!(
+ log.record(0, std::time::Instant::now() + DropLog::INTERVAL),
+ Some(4)
+ );
+ }
+
+ #[test]
+ fn previous_eight_frame_queue_lost_most_of_the_same_burst() {
+ let (tx, rx) = flume::bounded(8);
+ let mut senders = vec![tx];
+ let total = std::sync::atomic::AtomicU64::new(0);
+ let mut log = DropLog::default();
+ for frame in 0..60 {
+ fanout(&mut senders, frame, &total, &mut log);
+ }
+ assert_eq!(total.load(std::sync::atomic::Ordering::Relaxed), 52);
+ assert_eq!(rx.len(), 8);
+ }
+
+ #[test]
+ fn drop_log_reports_the_first_drop_then_rate_limits() {
+ let mut log = DropLog::default();
+ let start = std::time::Instant::now();
+ assert_eq!(log.record(0, start), None);
+ assert_eq!(log.record(1, start), Some(1));
+ assert_eq!(log.record(3, start + Duration::from_secs(1)), None);
+ assert_eq!(log.record(1, start + Duration::from_secs(4)), None);
+ assert_eq!(log.record(2, start + DropLog::INTERVAL), Some(6));
+ assert_eq!(log.record(0, start + DropLog::INTERVAL * 3), None);
+ }
+
+ #[cfg(target_os = "macos")]
+ mod actor {
+ use super::*;
+
+ /// Scripted runtime stall: the feed actor handles nothing else.
+ struct Hold(Duration, oneshot::Sender<()>);
+
+ impl Message<Hold> for CameraFeed {
+ type Reply = ();
+
+ async fn handle(&mut self, msg: Hold, _: &mut Context<Self, Self::Reply>) {
+ let _ = msg.1.send(());
+ tokio::time::sleep(msg.0).await;
+ }
+ }
+
+ fn frame(index: i64) -> NativeCameraFrame {
+ use cidre::{cm, cv};
+ let buf = cv::PixelBuf::new(16, 16, cv::PixelFormat::_420V, None).unwrap();
+ let desc = cm::VideoFormatDesc::with_image_buf(&buf).unwrap();
+ let timing = cm::SampleTimingInfo {
+ duration: cm::Time::new(1, 30),
+ pts: cm::Time::new(index, 30),
+ dts: cm::Time::invalid(),
+ };
+ let sample_buf =
+ cm::SampleBuf::with_image_buf(&buf, true, None, std::ptr::null(), &desc, &timing)
+ .unwrap();
+ NativeCameraFrame {
+ sample_buf,
+ timestamp: Timestamp::MachAbsoluteTime(cap_timestamp::MachAbsoluteTimestamp::new(
+ index as u64,
+ )),
+ }
+ }
+
+ async fn eventually(what: &str, check: impl Fn() -> bool) {
+ let deadline = std::time::Instant::now() + Duration::from_secs(10);
+ while !check() {
+ assert!(std::time::Instant::now() < deadline, "timed out: {what}");
+ tokio::time::sleep(Duration::from_millis(5)).await;
+ }
+ }
+
+ async fn stalled_feed(
+ capacity: usize,
+ hold: Duration,
+ ) -> (
+ ActorRef<CameraFeed>,
+ Arc<NativeCameraIngress>,
+ flume::Receiver<NativeCameraFrame>,
+ ) {
+ let ingress = Arc::new(NativeCameraIngress::default());
+ let feed = CameraFeed::spawn(CameraFeed {
+ native_ingress: ingress.clone(),
+ ..CameraFeed::default()
+ });
+ let (tx, rx) = flume::bounded(capacity);
+ feed.ask(AddNativeSender(tx)).await.unwrap();
+ let (held_tx, held_rx) = oneshot::channel();
+ feed.tell(Hold(hold, held_tx)).await.unwrap();
+ held_rx.await.unwrap();
+ (feed, ingress, rx)
+ }
+
+ /// The capture callback keeps delivering while the feed is stalled;
+ /// after the stall the feed fans the backlog out in one burst and the
+ /// recording receives every frame, in order.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn stall_then_burst_keeps_every_frame() {
+ let capacity = native_ingress_capacity(&hd30());
+ let (feed, ingress, rx) = stalled_feed(capacity, Duration::from_millis(1500)).await;
+ let recipient = feed.clone().recipient::<NewNativeFrame>();
+ let frames: Vec<_> = (0..45).map(frame).collect();
+ for frame in frames {
+ hand_native_frame_to_feed(&recipient, &ingress, frame);
+ }
+ eventually("burst delivered", || rx.len() == 45).await;
+ assert_eq!(ingress.drops(), NativeCameraIngressDrops::default());
+ let pts: Vec<_> = rx
+ .drain()
+ .map(|frame| frame.sample_buf.pts().value)
+ .collect();
+ assert_eq!(pts, (0..45).collect::<Vec<_>>());
+ }
+
+ /// Past the mailbox and the ingress queue every lost frame is
+ /// counted, and the callback's hand-over never waits for the stalled
+ /// feed.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn losses_past_the_bound_are_counted_without_blocking_the_callback() {
+ let capacity = native_ingress_capacity(&hd30());
+ let (feed, ingress, rx) = stalled_feed(capacity, Duration::from_millis(1000)).await;
+ let recipient = feed.clone().recipient::<NewNativeFrame>();
+ let frames: Vec<_> = (0..70).map(frame).collect();
+ let mut slowest = Duration::ZERO;
+ for frame in frames {
+ let start = std::time::Instant::now();
+ hand_native_frame_to_feed(&recipient, &ingress, frame);
+ slowest = slowest.max(start.elapsed());
+ }
+ assert!(
+ slowest < Duration::from_millis(50),
+ "callback hand-over waited {slowest:?}"
+ );
+ // The mailbox holds 64 frames; the other six are lost and counted.
+ assert_eq!(ingress.drops().mailbox_full, 6);
+ eventually("backlog delivered", || rx.len() == capacity).await;
+ assert_eq!(ingress.drops().fanout_full, 0);
+ // The recording still does not drain: the next frames overflow
+ // its ingress queue and are counted.
+ for frame in (70..73).map(frame) {
+ hand_native_frame_to_feed(&recipient, &ingress, frame);
+ }
+ eventually("overflow counted", || ingress.drops().fanout_full == 3).await;
+ assert_eq!(rx.len(), capacity);
+ }
+ }
+}
diff --git a/crates/recording/src/sources/native_camera.rs b/crates/recording/src/sources/native_camera.rs
index abb50eff2..8e8036804 100644
--- a/crates/recording/src/sources/native_camera.rs
+++ b/crates/recording/src/sources/native_camera.rs
@@ -1,36 +1,206 @@
use crate::{
- feeds::camera::{self, CameraFeedLock},
- output_pipeline::{NativeCameraFrame, SetupCtx, VideoSource},
+ feeds::camera::{self, CameraFeedLock, DropLog, NativeCameraIngress, NativeCameraIngressDrops},
+ output_pipeline::{
+ HealthSender, NativeCameraFrame, PipelineHealthEvent, SetupCtx, VideoSource, emit_health,
+ },
};
use anyhow::anyhow;
-use cap_media_info::VideoInfo;
+use cap_media_info::{Pixel, VideoInfo};
use futures::{FutureExt, channel::mpsc, future::BoxFuture};
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
-use std::time::Duration;
+use std::time::{Duration, Instant};
use tokio::sync::oneshot;
const CAMERA_SENDER_ATTACH_TIMEOUT: Duration = Duration::from_millis(1500);
+/// How long the recording may stop draining camera frames (a runtime stall)
+/// before the ingress queue fills.
+const INGRESS_TIME_BUDGET_SECS: usize = 2;
+/// The camera feed's actor mailbox (kameo's default bounded mailbox). After a
+/// stall the feed hands its whole backlog over in one burst, so the ingress
+/// queue must take that many frames at once.
+const CAMERA_FEED_MAILBOX_FRAMES: usize = 64;
+/// Upper bound on the pixel data the ingress queue can keep alive. On macOS a
+/// queued frame retains a capture pool surface (the pool grows per retained
+/// buffer); on Windows it is a heap copy of the frame.
+const INGRESS_BYTE_BUDGET: usize = 256 * 1024 * 1024;
+/// The previous fixed capacity; never go below it.
+const MIN_INGRESS_FRAMES: usize = 8;
+
+fn frame_bytes(info: &VideoInfo) -> usize {
+ let pixels = info.width as usize * info.height as usize;
+ match info.pixel_format {
+ Pixel::NV12 | Pixel::NV21 | Pixel::YUV420P | Pixel::YUVJ420P => pixels * 3 / 2,
+ Pixel::UYVY422 | Pixel::YUYV422 | Pixel::YUV422P | Pixel::YUVJ422P | Pixel::NV16 => {
+ pixels * 2
+ }
+ _ => pixels * 4,
+ }
+}
+
+/// Frames the native ingress queue holds: the time budget at the camera's
+/// frame rate (at least one full feed mailbox), capped by the byte budget.
+pub(crate) fn native_ingress_capacity(info: &VideoInfo) -> usize {
+ let by_time = (info.fps().max(1) as usize).saturating_mul(INGRESS_TIME_BUDGET_SECS);
+ let by_bytes = INGRESS_BYTE_BUDGET / frame_bytes(info).max(1);
+ by_time
+ .max(CAMERA_FEED_MAILBOX_FRAMES)
+ .min(by_bytes)
+ .max(MIN_INGRESS_FRAMES)
+}
+
pub struct NativeCamera {
feed_lock: Arc<CameraFeedLock>,
stop_tx: Option<oneshot::Sender<()>>,
stopped: Arc<AtomicBool>,
}
+/// Camera frames this recording lost, by boundary.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub(crate) struct IngressSummary {
+ pub received: u64,
+ pub sent: u64,
+ pub feed: NativeCameraIngressDrops,
+ pub forwarder_full: u64,
+}
+
+impl IngressSummary {
+ fn lost(&self) -> u64 {
+ self.feed.mailbox_full + self.feed.fanout_full + self.forwarder_full
+ }
+}
+
+/// Reports every camera frame this recording loses: in the feed (its mailbox
+/// or this recording's ingress queue) and between this source and the
+/// encoder. The first loss at each boundary is logged at warn and sent as a
+/// health event at once; later losses at most every five seconds; the totals
+/// at the end.
+pub(crate) struct IngressReporter {
+ feed: Option<Arc<NativeCameraIngress>>,
+ start: NativeCameraIngressDrops,
+ seen: NativeCameraIngressDrops,
+ health_tx: Option<HealthSender>,
+ feed_log: DropLog,
+ forwarder_log: DropLog,
+ summary: IngressSummary,
+}
+
+impl IngressReporter {
+ pub(crate) fn new(
+ feed: Option<Arc<NativeCameraIngress>>,
+ health_tx: Option<HealthSender>,
+ ) -> Self {
+ let start = feed.as_ref().map(|feed| feed.drops()).unwrap_or_default();
+ Self {
+ feed,
+ start,
+ seen: start,
+ health_tx,
+ feed_log: DropLog::default(),
+ forwarder_log: DropLog::default(),
+ summary: IngressSummary::default(),
+ }
+ }
+
+ fn received(&mut self) {
+ self.summary.received += 1;
+ self.poll_feed(Instant::now());
+ }
+
+ fn sent(&mut self) {
+ self.summary.sent += 1;
+ }
+
+ fn poll_feed(&mut self, now: Instant) {
+ let Some(feed) = &self.feed else {
+ return;
+ };
+ let current = feed.drops();
+ let new = current.since(self.seen);
+ if new == NativeCameraIngressDrops::default() {
+ return;
+ }
+ self.seen = current;
+ self.summary.feed = current.since(self.start);
+ if let Some(dropped) = self
+ .feed_log
+ .record(new.mailbox_full + new.fanout_full, now)
+ {
+ tracing::warn!(
+ dropped,
+ mailbox_full = self.summary.feed.mailbox_full,
+ ingress_full = self.summary.feed.fanout_full,
+ "Native camera frames lost before the recording received them"
+ );
+ self.emit("camera-ingress:feed");
+ }
+ }
+
+ fn forwarder_full(&mut self, now: Instant) {
+ self.summary.forwarder_full += 1;
+ if let Some(dropped) = self.forwarder_log.record(1, now) {
+ tracing::warn!(
+ dropped,
+ total = self.summary.forwarder_full,
+ "Native camera encoder queue full, frames dropped"
+ );
+ self.emit("camera-ingress:encoder-queue");
+ }
+ }
+
+ fn emit(&self, source: &str) {
+ let Some(health_tx) = &self.health_tx else {
+ return;
+ };
+ let lost = self.summary.lost();
+ let offered =
+ self.summary.received + self.summary.feed.mailbox_full + self.summary.feed.fanout_full;
+ emit_health(
+ health_tx,
+ PipelineHealthEvent::FrameDropRateHigh {
+ source: source.to_string(),
+ rate_pct: 100.0 * lost as f64 / offered.max(1) as f64,
+ },
+ );
+ }
+
+ fn finish(mut self, elapsed: Duration) -> IngressSummary {
+ self.poll_feed(Instant::now());
+ let summary = self.summary;
+ if summary.lost() > 0 {
+ tracing::warn!(
+ received = summary.received,
+ sent = summary.sent,
+ mailbox_full = summary.feed.mailbox_full,
+ ingress_full = summary.feed.fanout_full,
+ encoder_queue_full = summary.forwarder_full,
+ ?elapsed,
+ "Native camera source finished with lost frames"
+ );
+ } else {
+ tracing::info!(
+ "Native camera source finished: {} received, {} sent, 0 dropped in {:?}",
+ summary.received,
+ summary.sent,
+ elapsed
+ );
+ }
+ summary
+ }
+}
+
async fn forward_native_frames<T: Send + 'static>(
rx: flume::Receiver<T>,
video_tx: mpsc::Sender<T>,
stop_rx: oneshot::Receiver<()>,
stopped: Arc<AtomicBool>,
required: bool,
-) -> anyhow::Result<()> {
+ mut report: IngressReporter,
+) -> anyhow::Result<IngressSummary> {
tracing::debug!("Native camera source task started");
- let mut frame_count: u64 = 0;
- let mut sent_count: u64 = 0;
- let mut dropped_count: u64 = 0;
let start = std::time::Instant::now();
let mut video_tx = video_tx;
let mut stop_rx = stop_rx.fuse();
@@ -50,33 +220,16 @@ async fn forward_native_frames<T: Send + 'static>(
result = rx.recv_async() => {
match result {
Ok(frame) => {
- frame_count += 1;
+ report.received();
match video_tx.try_send(frame) {
- Ok(()) => {
- sent_count += 1;
- if sent_count.is_multiple_of(30) {
- tracing::debug!(
- "Native camera source: sent {} frames, dropped {} in {:?}",
- sent_count,
- dropped_count,
- start.elapsed()
- );
- }
- }
+ Ok(()) => report.sent(),
Err(e) => {
if e.is_full() {
- dropped_count += 1;
- if dropped_count.is_multiple_of(30) {
- tracing::warn!(
- "Native camera source: encoder can't keep up, dropped {} frames so far",
- dropped_count
- );
- }
+ report.forwarder_full(Instant::now());
} else if e.is_disconnected() {
tracing::debug!(
- "Native camera source: pipeline closed after {} sent, {} dropped",
- sent_count,
- dropped_count
+ "Native camera source: pipeline closed after {:?}",
+ report.summary
);
if required && !stopped.load(Ordering::Acquire) {
return Err(anyhow!("Native camera encoder disconnected before Stop"));
@@ -89,7 +242,7 @@ async fn forward_native_frames<T: Send + 'static>(
Err(e) => {
tracing::debug!(
"Native camera feed disconnected (rx closed) after {} frames in {:?}: {e}",
- frame_count,
+ report.summary.received,
start.elapsed()
);
if required && !stopped.load(Ordering::Acquire) {
@@ -104,14 +257,7 @@ async fn forward_native_frames<T: Send + 'static>(
drop(video_tx);
- tracing::info!(
- "Native camera source finished: {} received, {} sent, {} dropped in {:?}",
- frame_count,
- sent_count,
- dropped_count,
- start.elapsed()
- );
- Ok(())
+ Ok(report.finish(start.elapsed()))
}
impl VideoSource for NativeCamera {
@@ -121,12 +267,14 @@ impl VideoSource for NativeCamera {
async fn setup(
feed_lock: Self::Config,
video_tx: mpsc::Sender<Self::Frame>,
- _ctx: &mut SetupCtx,
+ ctx: &mut SetupCtx,
) -> anyhow::Result<Self>
where
Self: Sized,
{
- let (tx, rx) = flume::bounded(8);
+ let capacity = native_ingress_capacity(feed_lock.video_info());
+ tracing::debug!(capacity, "Native camera ingress queue");
+ let (tx, rx) = flume::bounded(capacity);
tokio::time::timeout(
CAMERA_SENDER_ATTACH_TIMEOUT,
@@ -140,9 +288,15 @@ impl VideoSource for NativeCamera {
let stopped = Arc::new(AtomicBool::new(false));
let stopped_clone = stopped.clone();
- let task = forward_native_frames(rx, video_tx, stop_rx, stopped_clone, cfg!(windows));
+ let report = IngressReporter::new(
+ Some(feed_lock.native_ingress().clone()),
+ Some(ctx.health_tx().clone()),
+ );
+ let task =
+ forward_native_frames(rx, video_tx, stop_rx, stopped_clone, cfg!(windows), report)
+ .map(|result| result.map(drop));
#[cfg(windows)]
- _ctx.tasks().spawn("native-camera", task);
+ ctx.tasks().spawn("native-camera", task);
#[cfg(not(windows))]
drop(tokio::spawn(task));
@@ -187,6 +341,7 @@ mod forwarding_tests {
stopped,
Arc::new(AtomicBool::new(false)),
true,
+ IngressReporter::new(None, None),
)
.await;
assert!(
@@ -210,6 +365,7 @@ mod forwarding_tests {
stopped,
Arc::new(AtomicBool::new(false)),
true,
+ IngressReporter::new(None, None),
)
.await
.unwrap();
@@ -227,6 +383,7 @@ mod forwarding_tests {
stopped,
Arc::new(AtomicBool::new(false)),
true,
+ IngressReporter::new(None, None),
));
sender.send_async(17_u8).await.unwrap();
assert_eq!(frames.next().await, Some(17));
@@ -249,6 +406,7 @@ mod forwarding_tests {
stopped,
Arc::new(AtomicBool::new(false)),
true,
+ IngressReporter::new(None, None),
)
.await;
assert!(
@@ -271,8 +429,97 @@ mod forwarding_tests {
stopped,
Arc::new(AtomicBool::new(false)),
false,
+ IngressReporter::new(None, None),
+ )
+ .await
+ .unwrap();
+ }
+
+ fn info(format: cap_media_info::RawVideoFormat, w: u32, h: u32, fps: u32) -> VideoInfo {
+ VideoInfo::from_raw(format, w, h, fps)
+ }
+
+ #[test]
+ fn ingress_capacity_is_sized_by_time_and_bounded_by_bytes() {
+ use cap_media_info::RawVideoFormat::{Bgra, Nv12, Uyvy};
+ let cases = [
+ (info(Nv12, 1280, 720, 30), 64),
+ (info(Nv12, 1920, 1080, 30), 64),
+ (info(Nv12, 1280, 720, 60), 120),
+ (info(Nv12, 1920, 1080, 60), 86),
+ (info(Uyvy, 1920, 1080, 30), 64),
+ (info(Nv12, 3840, 2160, 30), 21),
+ (info(Bgra, 3840, 2160, 30), MIN_INGRESS_FRAMES),
+ ];
+ for (info, expected) in cases {
+ let capacity = native_ingress_capacity(&info);
+ assert_eq!(
+ capacity,
+ expected,
+ "{}x{}@{} {:?}",
+ info.width,
+ info.height,
+ info.fps(),
+ info.pixel_format
+ );
+ assert!(
+ capacity == MIN_INGRESS_FRAMES
+ || capacity * frame_bytes(&info) <= INGRESS_BYTE_BUDGET
+ );
+ }
+ }
+
+ #[tokio::test]
+ async fn encoder_queue_overflow_is_counted() {
+ let (sender, receiver) = flume::bounded(16);
+ let (video, frames) = mpsc::channel(1);
+ let (_stop, stopped) = oneshot::channel();
+ for frame in 0..10_u8 {
+ sender.send_async(frame).await.unwrap();
+ }
+ drop(sender);
+ let summary = forward_native_frames(
+ receiver,
+ video,
+ stopped,
+ Arc::new(AtomicBool::new(false)),
+ false,
+ IngressReporter::new(None, None),
+ )
+ .await
+ .unwrap();
+ // futures mpsc holds its buffer plus one slot per sender.
+ assert_eq!(summary.received, 10);
+ assert_eq!(summary.sent, 2);
+ assert_eq!(summary.forwarder_full, 8);
+ drop(frames);
+ }
+
+ #[tokio::test]
+ async fn feed_losses_count_from_the_recording_start() {
+ let ingress = Arc::new(NativeCameraIngress::default());
+ ingress.record_mailbox_full();
+ let report = IngressReporter::new(Some(ingress.clone()), None);
+ ingress.record_mailbox_full();
+ ingress.record_mailbox_full();
+ let (sender, receiver) = flume::bounded(4);
+ let (video, mut frames) = mpsc::channel(4);
+ let (_stop, stopped) = oneshot::channel();
+ sender.send_async(1_u8).await.unwrap();
+ drop(sender);
+ let summary = forward_native_frames(
+ receiver,
+ video,
+ stopped,
+ Arc::new(AtomicBool::new(false)),
+ false,
+ report,
)
.await
.unwrap();
+ assert_eq!(summary.feed.mailbox_full, 2);
+ assert_eq!(summary.feed.fanout_full, 0);
+ assert_eq!(summary.sent, 1);
+ assert_eq!(futures::StreamExt::next(&mut frames).await, Some(1));
}
}