Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(recording): count camera fan-out losses per attached queue

Richie McIlroy28b6732e63dc
diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs
index bbd025587..d9417d61c 100644
--- a/crates/recording/src/feeds/camera.rs
+++ b/crates/recording/src/feeds/camera.rs
@@ -53,57 +53,64 @@ pub struct CameraFeed {
state: State,
senders: Vec<flume::Sender<FFmpegVideoFrame>>,
ffmpeg_sender_count: Arc<std::sync::atomic::AtomicUsize>,
- native_senders: Vec<flume::Sender<NativeCameraFrame>>,
+ native_senders: Vec<CountedSender<NativeCameraFrame>>,
native_ingress: Arc<NativeCameraIngress>,
+ native_fanout_full: u64,
native_fanout_log: DropLog,
- ffmpeg_fanout_full: std::sync::atomic::AtomicU64,
+ ffmpeg_fanout_full: u64,
ffmpeg_fanout_log: DropLog,
on_ready: Vec<oneshot::Sender<()>>,
on_disconnect: Vec<Box<dyn Fn() + Send>>,
}
-/// Shared between the camera feed, its capture callback and the native
-/// sources of the recordings attached to it.
+/// Shared between the camera feed and its capture callback, which runs on a
+/// capture thread and must never wait for the feed.
#[derive(Debug, Default)]
pub struct NativeCameraIngress {
senders: std::sync::atomic::AtomicUsize,
mailbox_full: std::sync::atomic::AtomicU64,
- fanout_full: std::sync::atomic::AtomicU64,
}
-/// Native camera frames lost inside the feed, before a recording's source
-/// received them. Monotonic over the feed's lifetime; recordings report the
-/// difference between their start and the current value.
-#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
-pub struct NativeCameraIngressDrops {
- /// The capture callback found the feed's mailbox full (the feed actor
- /// was not running, for example during a runtime stall).
- pub mailbox_full: u64,
- /// A recording's ingress queue was full when the feed fanned a frame out.
- pub fanout_full: u64,
+impl NativeCameraIngress {
+ /// Frames the capture callback could not hand to the feed because its
+ /// mailbox was full. Monotonic over the feed's lifetime.
+ pub fn mailbox_full(&self) -> u64 {
+ self.mailbox_full.load(std::sync::atomic::Ordering::Relaxed)
+ }
+
+ pub fn record_mailbox_full(&self) {
+ self.mailbox_full
+ .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ }
}
-impl NativeCameraIngressDrops {
- pub fn since(self, start: Self) -> Self {
- Self {
- mailbox_full: self.mailbox_full.saturating_sub(start.mailbox_full),
- fanout_full: self.fanout_full.saturating_sub(start.fanout_full),
- }
+/// A fan-out target. Senders attached with a counter (recordings) count the
+/// frames the feed could not deliver to them; previews attach without one.
+struct CountedSender<T> {
+ tx: flume::Sender<T>,
+ dropped: Option<Arc<std::sync::atomic::AtomicU64>>,
+}
+
+trait FanoutTarget<T> {
+ fn channel(&self) -> &flume::Sender<T>;
+ fn record_drop(&self) {}
+}
+
+impl<T> FanoutTarget<T> for flume::Sender<T> {
+ fn channel(&self) -> &flume::Sender<T> {
+ self
}
}
-impl NativeCameraIngress {
- pub fn drops(&self) -> NativeCameraIngressDrops {
- NativeCameraIngressDrops {
- mailbox_full: self.mailbox_full.load(std::sync::atomic::Ordering::Relaxed),
- fanout_full: self.fanout_full.load(std::sync::atomic::Ordering::Relaxed),
- }
+impl<T> FanoutTarget<T> for CountedSender<T> {
+ fn channel(&self) -> &flume::Sender<T> {
+ &self.tx
}
- /// Called from capture callbacks: a relaxed increment, never a wait.
- pub fn record_mailbox_full(&self) {
- self.mailbox_full
- .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ fn record_drop(&self) {
+ if let Some(dropped) = &self.dropped {
+ dropped.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+ }
}
}
@@ -119,7 +126,6 @@ pub(crate) struct DropLog {
impl DropLog {
pub(crate) const INTERVAL: Duration = Duration::from_secs(5);
- /// Adds `count` drops. Returns the drops to report now, if a report is due.
pub(crate) fn record(&mut self, count: u64, now: std::time::Instant) -> Option<u64> {
self.unreported = self.unreported.saturating_add(count);
if self.unreported == 0
@@ -286,8 +292,9 @@ impl Default for CameraFeed {
ffmpeg_sender_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
native_senders: Vec::new(),
native_ingress: Arc::new(NativeCameraIngress::default()),
+ native_fanout_full: 0,
native_fanout_log: DropLog::default(),
- ffmpeg_fanout_full: std::sync::atomic::AtomicU64::new(0),
+ ffmpeg_fanout_full: 0,
ffmpeg_fanout_log: DropLog::default(),
on_ready: Vec::new(),
on_disconnect: Vec::new(),
@@ -384,6 +391,13 @@ pub struct AddSender(pub flume::Sender<FFmpegVideoFrame>);
pub struct AddNativeSender(pub flume::Sender<NativeCameraFrame>);
+/// Attaches a recording's native queue; every frame the feed cannot deliver
+/// to it is added to the counter.
+pub struct AddCountedNativeSender(
+ pub flume::Sender<NativeCameraFrame>,
+ pub Arc<std::sync::atomic::AtomicU64>,
+);
+
pub struct RemoveSender(pub flume::Sender<FFmpegVideoFrame>);
pub struct RemoveNativeSender(pub flume::Sender<NativeCameraFrame>);
@@ -1469,24 +1483,22 @@ impl Message<AddSender> for CameraFeed {
}
}
-impl Message<AddNativeSender> for CameraFeed {
- type Reply = ();
-
- async fn handle(
+impl CameraFeed {
+ fn add_native_sender(
&mut self,
- msg: AddNativeSender,
- _: &mut Context<Self, Self::Reply>,
- ) -> Self::Reply {
+ tx: flume::Sender<NativeCameraFrame>,
+ dropped: Option<Arc<std::sync::atomic::AtomicU64>>,
+ ) {
if self
.native_senders
.iter()
- .any(|sender| sender.same_channel(&msg.0))
+ .any(|sender| sender.tx.same_channel(&tx))
{
return;
}
debug!("CameraFeed: Adding new native sender");
- self.native_senders.push(msg.0);
+ self.native_senders.push(CountedSender { tx, dropped });
self.native_ingress.senders.store(
self.native_senders.len(),
std::sync::atomic::Ordering::Release,
@@ -1494,6 +1506,30 @@ impl Message<AddNativeSender> for CameraFeed {
}
}
+impl Message<AddNativeSender> for CameraFeed {
+ type Reply = ();
+
+ async fn handle(
+ &mut self,
+ msg: AddNativeSender,
+ _: &mut Context<Self, Self::Reply>,
+ ) -> Self::Reply {
+ self.add_native_sender(msg.0, None);
+ }
+}
+
+impl Message<AddCountedNativeSender> for CameraFeed {
+ type Reply = ();
+
+ async fn handle(
+ &mut self,
+ msg: AddCountedNativeSender,
+ _: &mut Context<Self, Self::Reply>,
+ ) -> Self::Reply {
+ self.add_native_sender(msg.0, Some(msg.1));
+ }
+}
+
impl Message<RemoveSender> for CameraFeed {
type Reply = ();
@@ -1517,7 +1553,7 @@ impl Message<RemoveNativeSender> for CameraFeed {
_: &mut Context<Self, Self::Reply>,
) -> Self::Reply {
self.native_senders
- .retain(|sender| !sender.same_channel(&msg.0));
+ .retain(|sender| !sender.tx.same_channel(&msg.0));
self.native_ingress.senders.store(
self.native_senders.len(),
std::sync::atomic::Ordering::Release,
@@ -1563,7 +1599,7 @@ static CAMERA_FRAME_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::A
/// Frames a fan-out could not deliver: one per sender whose queue was full.
struct FanoutDrops<'a> {
- total: &'a std::sync::atomic::AtomicU64,
+ total: &'a mut u64,
log: &'a mut DropLog,
}
@@ -1572,10 +1608,8 @@ impl FanoutDrops<'_> {
if count == 0 {
return;
}
- let total = self
- .total
- .fetch_add(count, std::sync::atomic::Ordering::Relaxed)
- + count;
+ *self.total = self.total.saturating_add(count);
+ let total = *self.total;
if let Some(dropped) = self.log.record(count, std::time::Instant::now()) {
warn!(
dropped,
@@ -1585,8 +1619,8 @@ impl FanoutDrops<'_> {
}
}
-fn send_frame_to_camera_senders<T: Clone>(
- senders: &mut Vec<flume::Sender<T>>,
+fn send_frame_to_camera_senders<T: Clone, S: FanoutTarget<T>>(
+ senders: &mut Vec<S>,
frame: T,
frame_num: u64,
sender_label: &str,
@@ -1595,7 +1629,7 @@ fn send_frame_to_camera_senders<T: Clone>(
// A disconnected sender whose queue is still full would otherwise never be
// try_send'd again, leaving it (and its queued frames) retained forever.
let len_before_retain = senders.len();
- senders.retain(|sender| !sender.is_disconnected());
+ senders.retain(|sender| !sender.channel().is_disconnected());
let removed_disconnected = senders.len() != len_before_retain;
if removed_disconnected {
debug!(
@@ -1605,46 +1639,40 @@ fn send_frame_to_camera_senders<T: Clone>(
);
}
- let mut last_ready_sender = None;
- let mut full = 0;
-
- for (i, sender) in senders.iter().enumerate() {
- if sender.is_full() {
- full += 1;
- } else {
- last_ready_sender = Some(i);
- }
- }
-
- let Some(last_ready_sender) = last_ready_sender else {
- drops.record(full, frame_num, sender_label);
- return removed_disconnected;
- };
+ // The last sender with room takes the frame itself instead of a clone.
+ let last_ready_sender = senders
+ .iter()
+ .rposition(|sender| !sender.channel().is_full());
let mut frame = Some(frame);
+ let mut skipped = 0;
let mut to_remove = vec![];
- for (i, sender) in senders.iter().enumerate().take(last_ready_sender + 1) {
- if sender.is_full() {
+ for (i, sender) in senders.iter().enumerate() {
+ let ready = last_ready_sender.is_some_and(|last| i <= last);
+ if !ready || sender.channel().is_full() {
+ sender.record_drop();
+ skipped += 1;
continue;
}
- let send_result = if i == last_ready_sender {
+ let send_result = if Some(i) == last_ready_sender {
let Some(frame) = frame.take() else {
break;
};
- sender.try_send(frame)
+ sender.channel().try_send(frame)
} else {
let Some(frame) = frame.as_ref() else {
break;
};
- sender.try_send((*frame).clone())
+ sender.channel().try_send(frame.clone())
};
match send_result {
Ok(()) => {}
Err(flume::TrySendError::Full(_)) => {
- full += 1;
+ sender.record_drop();
+ skipped += 1;
}
Err(flume::TrySendError::Disconnected(_)) => {
debug!(
@@ -1656,7 +1684,7 @@ fn send_frame_to_camera_senders<T: Clone>(
}
}
- drops.record(full, frame_num, sender_label);
+ drops.record(skipped, frame_num, sender_label);
if to_remove.is_empty() {
return removed_disconnected;
@@ -1692,7 +1720,7 @@ impl Message<NewFrame> for CameraFeed {
frame_num,
"Camera",
FanoutDrops {
- total: &self.ffmpeg_fanout_full,
+ total: &mut self.ffmpeg_fanout_full,
log: &mut self.ffmpeg_fanout_log,
},
) {
@@ -1729,7 +1757,7 @@ impl Message<NewNativeFrame> for CameraFeed {
frame_num,
"Native camera",
FanoutDrops {
- total: &self.native_ingress.fanout_full,
+ total: &mut self.native_fanout_full,
log: &mut self.native_fanout_log,
},
) {
@@ -2636,20 +2664,49 @@ mod ingress_tests {
use super::*;
use crate::sources::native_camera::native_ingress_capacity;
use cap_media_info::RawVideoFormat;
+ use std::sync::atomic::{AtomicU64, Ordering::Relaxed};
+
+ type Output = (flume::Receiver<u32>, Arc<AtomicU64>);
+
+ struct Fanout {
+ senders: Vec<CountedSender<u32>>,
+ total: u64,
+ log: DropLog,
+ }
+
+ impl Fanout {
+ fn new(capacities: &[usize]) -> (Self, Vec<Output>) {
+ let mut senders = vec![];
+ let mut outputs = vec![];
+ for &capacity in capacities {
+ let (tx, rx) = flume::bounded(capacity);
+ let dropped = Arc::new(AtomicU64::new(0));
+ senders.push(CountedSender {
+ tx,
+ dropped: Some(dropped.clone()),
+ });
+ outputs.push((rx, dropped));
+ }
+ let fanout = Self {
+ senders,
+ total: 0,
+ log: DropLog::default(),
+ };
+ (fanout, outputs)
+ }
- fn fanout(
- senders: &mut Vec<flume::Sender<u32>>,
- frame: u32,
- total: &std::sync::atomic::AtomicU64,
- log: &mut DropLog,
- ) {
- send_frame_to_camera_senders(
- senders,
- frame,
- u64::from(frame),
- "Native camera",
- FanoutDrops { total, log },
- );
+ fn send(&mut self, frame: u32) {
+ send_frame_to_camera_senders(
+ &mut self.senders,
+ frame,
+ u64::from(frame),
+ "Native camera",
+ FanoutDrops {
+ total: &mut self.total,
+ log: &mut self.log,
+ },
+ );
+ }
}
fn hd30() -> VideoInfo {
@@ -2658,52 +2715,100 @@ mod ingress_tests {
#[test]
fn burst_after_a_two_second_stall_fits_the_ingress_queue() {
- let capacity = native_ingress_capacity(&hd30());
- let (tx, rx) = flume::bounded(capacity);
- let mut senders = vec![tx];
- let total = std::sync::atomic::AtomicU64::new(0);
- let mut log = DropLog::default();
+ let (mut fanout, outputs) = Fanout::new(&[native_ingress_capacity(&hd30())]);
// The consumer is stalled for 2 s at 30 fps; the feed then hands the
// whole backlog over at once.
for frame in 0..60 {
- fanout(&mut senders, frame, &total, &mut log);
+ fanout.send(frame);
}
- assert_eq!(total.load(std::sync::atomic::Ordering::Relaxed), 0);
+ let (rx, dropped) = &outputs[0];
+ assert_eq!(dropped.load(Relaxed), 0);
+ assert_eq!(fanout.total, 0);
assert_eq!(rx.drain().collect::<Vec<_>>(), (0..60).collect::<Vec<_>>());
}
#[test]
fn frames_past_the_ingress_bound_are_counted_and_reported() {
let capacity = native_ingress_capacity(&hd30());
- let (tx, rx) = flume::bounded(capacity);
- let mut senders = vec![tx];
- let total = std::sync::atomic::AtomicU64::new(0);
- let mut log = DropLog::default();
+ let (mut fanout, outputs) = Fanout::new(&[capacity]);
for frame in 0..capacity as u32 + 5 {
- fanout(&mut senders, frame, &total, &mut log);
+ fanout.send(frame);
}
- assert_eq!(total.load(std::sync::atomic::Ordering::Relaxed), 5);
- assert_eq!(rx.len(), capacity);
+ assert_eq!(outputs[0].1.load(Relaxed), 5);
+ assert_eq!(outputs[0].0.len(), capacity);
+ assert_eq!(fanout.total, 5);
// The first drop was reported at once; the other four wait for the
// next report window.
- assert_eq!(log.record(0, std::time::Instant::now()), None);
- assert_eq!(
- log.record(0, std::time::Instant::now() + DropLog::INTERVAL),
- Some(4)
- );
+ let now = std::time::Instant::now();
+ assert_eq!(fanout.log.record(0, now), None);
+ assert_eq!(fanout.log.record(0, now + DropLog::INTERVAL), Some(4));
}
#[test]
fn previous_eight_frame_queue_lost_most_of_the_same_burst() {
- let (tx, rx) = flume::bounded(8);
- let mut senders = vec![tx];
- let total = std::sync::atomic::AtomicU64::new(0);
- let mut log = DropLog::default();
+ let (mut fanout, outputs) = Fanout::new(&[8]);
for frame in 0..60 {
- fanout(&mut senders, frame, &total, &mut log);
+ fanout.send(frame);
}
- assert_eq!(total.load(std::sync::atomic::Ordering::Relaxed), 52);
- assert_eq!(rx.len(), 8);
+ assert_eq!(outputs[0].1.load(Relaxed), 52);
+ assert_eq!(outputs[0].0.len(), 8);
+ }
+
+ /// A full preview queue is not a recording loss: each attached queue
+ /// counts only the frames it did not receive.
+ #[test]
+ fn losses_are_counted_per_attached_queue() {
+ let (mut fanout, outputs) = Fanout::new(&[4, 64, 2]);
+ for frame in 0..10 {
+ fanout.send(frame);
+ }
+ let counts: Vec<_> = outputs
+ .iter()
+ .map(|(rx, dropped)| (rx.len(), dropped.load(Relaxed)))
+ .collect();
+ assert_eq!(counts, [(4, 6), (10, 0), (2, 8)]);
+ assert_eq!(fanout.total, 14);
+ }
+
+ /// Consumers draining concurrently with the fan-out: every frame is
+ /// either received or counted as dropped for each queue, never both.
+ #[test]
+ fn received_plus_dropped_is_every_frame_per_queue() {
+ const FRAMES: u32 = 20_000;
+ let (mut fanout, outputs) = Fanout::new(&[1, 2, 1]);
+ let stop = Arc::new(std::sync::atomic::AtomicBool::new(false));
+ let drains: Vec<_> = outputs
+ .iter()
+ .enumerate()
+ .map(|(i, (rx, _))| {
+ let rx = rx.clone();
+ let stop = stop.clone();
+ std::thread::spawn(move || {
+ let mut received = 0_u64;
+ while !stop.load(Relaxed) || !rx.is_empty() {
+ if rx.try_recv().is_ok() {
+ received += 1;
+ }
+ if i == 1 {
+ std::thread::yield_now();
+ }
+ }
+ received
+ })
+ })
+ .collect();
+ for frame in 0..FRAMES {
+ fanout.send(frame);
+ }
+ stop.store(true, Relaxed);
+ let mut total_dropped = 0;
+ for ((_, dropped), drain) in outputs.iter().zip(drains) {
+ let received = drain.join().unwrap();
+ let dropped = dropped.load(Relaxed);
+ assert_eq!(received + dropped, u64::from(FRAMES));
+ total_dropped += dropped;
+ }
+ assert_eq!(fanout.total, total_dropped);
}
#[test]
@@ -2762,25 +2867,33 @@ mod ingress_tests {
}
}
- async fn stalled_feed(
- capacity: usize,
- hold: Duration,
- ) -> (
- ActorRef<CameraFeed>,
- Arc<NativeCameraIngress>,
- flume::Receiver<NativeCameraFrame>,
- ) {
+ struct StalledFeed {
+ feed: ActorRef<CameraFeed>,
+ ingress: Arc<NativeCameraIngress>,
+ dropped: Arc<AtomicU64>,
+ rx: flume::Receiver<NativeCameraFrame>,
+ }
+
+ async fn stalled_feed(capacity: usize, hold: Duration) -> StalledFeed {
let ingress = Arc::new(NativeCameraIngress::default());
let feed = CameraFeed::spawn(CameraFeed {
native_ingress: ingress.clone(),
..CameraFeed::default()
});
let (tx, rx) = flume::bounded(capacity);
- feed.ask(AddNativeSender(tx)).await.unwrap();
+ let dropped = Arc::new(AtomicU64::new(0));
+ feed.ask(AddCountedNativeSender(tx, dropped.clone()))
+ .await
+ .unwrap();
let (held_tx, held_rx) = oneshot::channel();
feed.tell(Hold(hold, held_tx)).await.unwrap();
held_rx.await.unwrap();
- (feed, ingress, rx)
+ StalledFeed {
+ feed,
+ ingress,
+ dropped,
+ rx,
+ }
}
/// The capture callback keeps delivering while the feed is stalled;
@@ -2789,18 +2902,19 @@ mod ingress_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stall_then_burst_keeps_every_frame() {
let capacity = native_ingress_capacity(&hd30());
- let (feed, ingress, rx) = stalled_feed(capacity, Duration::from_millis(1500)).await;
- let recipient = feed.clone().recipient::<NewNativeFrame>();
+ let f = stalled_feed(capacity, Duration::from_millis(1500)).await;
+ let recipient = f.feed.clone().recipient::<NewNativeFrame>();
let frames: Vec<_> = (0..45).map(frame).collect();
for frame in frames {
- hand_native_frame_to_feed(&recipient, &ingress, frame);
+ hand_native_frame_to_feed(&recipient, &f.ingress, frame);
}
- eventually("burst delivered", || rx.len() == 45).await;
- assert_eq!(ingress.drops(), NativeCameraIngressDrops::default());
- let pts: Vec<_> = rx
- .drain()
- .map(|frame| frame.sample_buf.pts().value)
- .collect();
+ eventually("burst delivered", || f.rx.len() == 45).await;
+ assert_eq!(f.ingress.mailbox_full(), 0);
+ assert_eq!(f.dropped.load(Relaxed), 0);
+ let pts: Vec<_> =
+ f.rx.drain()
+ .map(|frame| frame.sample_buf.pts().value)
+ .collect();
assert_eq!(pts, (0..45).collect::<Vec<_>>());
}
@@ -2810,13 +2924,13 @@ mod ingress_tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn losses_past_the_bound_are_counted_without_blocking_the_callback() {
let capacity = native_ingress_capacity(&hd30());
- let (feed, ingress, rx) = stalled_feed(capacity, Duration::from_millis(1000)).await;
- let recipient = feed.clone().recipient::<NewNativeFrame>();
+ let f = stalled_feed(capacity, Duration::from_millis(1000)).await;
+ let recipient = f.feed.clone().recipient::<NewNativeFrame>();
let frames: Vec<_> = (0..70).map(frame).collect();
let mut slowest = Duration::ZERO;
for frame in frames {
let start = std::time::Instant::now();
- hand_native_frame_to_feed(&recipient, &ingress, frame);
+ hand_native_frame_to_feed(&recipient, &f.ingress, frame);
slowest = slowest.max(start.elapsed());
}
assert!(
@@ -2824,16 +2938,16 @@ mod ingress_tests {
"callback hand-over waited {slowest:?}"
);
// The mailbox holds 64 frames; the other six are lost and counted.
- assert_eq!(ingress.drops().mailbox_full, 6);
- eventually("backlog delivered", || rx.len() == capacity).await;
- assert_eq!(ingress.drops().fanout_full, 0);
+ assert_eq!(f.ingress.mailbox_full(), 6);
+ eventually("backlog delivered", || f.rx.len() == capacity).await;
+ assert_eq!(f.dropped.load(Relaxed), 0);
// The recording still does not drain: the next frames overflow
// its ingress queue and are counted.
for frame in (70..73).map(frame) {
- hand_native_frame_to_feed(&recipient, &ingress, frame);
+ hand_native_frame_to_feed(&recipient, &f.ingress, frame);
}
- eventually("overflow counted", || ingress.drops().fanout_full == 3).await;
- assert_eq!(rx.len(), capacity);
+ eventually("overflow counted", || f.dropped.load(Relaxed) == 3).await;
+ assert_eq!(f.rx.len(), capacity);
}
}
}
diff --git a/crates/recording/src/sources/native_camera.rs b/crates/recording/src/sources/native_camera.rs
index 8e8036804..d82d2cfe6 100644
--- a/crates/recording/src/sources/native_camera.rs
+++ b/crates/recording/src/sources/native_camera.rs
@@ -1,5 +1,5 @@
use crate::{
- feeds::camera::{self, CameraFeedLock, DropLog, NativeCameraIngress, NativeCameraIngressDrops},
+ feeds::camera::{self, CameraFeedLock, DropLog, NativeCameraIngress},
output_pipeline::{
HealthSender, NativeCameraFrame, PipelineHealthEvent, SetupCtx, VideoSource, emit_health,
},
@@ -9,7 +9,7 @@ use cap_media_info::{Pixel, VideoInfo};
use futures::{FutureExt, channel::mpsc, future::BoxFuture};
use std::sync::{
Arc,
- atomic::{AtomicBool, Ordering},
+ atomic::{AtomicBool, AtomicU64, Ordering},
};
use std::time::{Duration, Instant};
use tokio::sync::oneshot;
@@ -58,30 +58,32 @@ pub struct NativeCamera {
stopped: Arc<AtomicBool>,
}
-/// Camera frames this recording lost, by boundary.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub(crate) struct IngressSummary {
pub received: u64,
pub sent: u64,
- pub feed: NativeCameraIngressDrops,
+ pub mailbox_full: u64,
+ pub ingress_full: u64,
pub forwarder_full: u64,
}
impl IngressSummary {
fn lost(&self) -> u64 {
- self.feed.mailbox_full + self.feed.fanout_full + self.forwarder_full
+ self.mailbox_full + self.ingress_full + self.forwarder_full
}
}
-/// Reports every camera frame this recording loses: in the feed (its mailbox
-/// or this recording's ingress queue) and between this source and the
-/// encoder. The first loss at each boundary is logged at warn and sent as a
-/// health event at once; later losses at most every five seconds; the totals
-/// at the end.
+/// Reports every camera frame this recording loses: at the feed's mailbox
+/// (shared by everything attached to the feed), at this recording's ingress
+/// queue, and between this source and the encoder. The first loss at each
+/// boundary is logged at warn and sent as a health event at once; later
+/// losses at most every five seconds; the totals at the end.
pub(crate) struct IngressReporter {
feed: Option<Arc<NativeCameraIngress>>,
- start: NativeCameraIngressDrops,
- seen: NativeCameraIngressDrops,
+ mailbox_start: u64,
+ mailbox_seen: u64,
+ ingress_dropped: Arc<AtomicU64>,
+ ingress_seen: u64,
health_tx: Option<HealthSender>,
feed_log: DropLog,
forwarder_log: DropLog,
@@ -89,15 +91,20 @@ pub(crate) struct IngressReporter {
}
impl IngressReporter {
+ /// Create before attaching the ingress queue: mailbox losses while it
+ /// attaches already affect this recording.
pub(crate) fn new(
feed: Option<Arc<NativeCameraIngress>>,
+ ingress_dropped: Arc<AtomicU64>,
health_tx: Option<HealthSender>,
) -> Self {
- let start = feed.as_ref().map(|feed| feed.drops()).unwrap_or_default();
+ let mailbox_start = feed.as_ref().map_or(0, |feed| feed.mailbox_full());
Self {
feed,
- start,
- seen: start,
+ mailbox_start,
+ mailbox_seen: mailbox_start,
+ ingress_seen: ingress_dropped.load(Ordering::Relaxed),
+ ingress_dropped,
health_tx,
feed_log: DropLog::default(),
forwarder_log: DropLog::default(),
@@ -115,24 +122,25 @@ impl IngressReporter {
}
fn poll_feed(&mut self, now: Instant) {
- let Some(feed) = &self.feed else {
- return;
- };
- let current = feed.drops();
- let new = current.since(self.seen);
- if new == NativeCameraIngressDrops::default() {
+ let mailbox = self
+ .feed
+ .as_ref()
+ .map_or(self.mailbox_seen, |feed| feed.mailbox_full());
+ let ingress = self.ingress_dropped.load(Ordering::Relaxed);
+ let new =
+ mailbox.saturating_sub(self.mailbox_seen) + ingress.saturating_sub(self.ingress_seen);
+ if new == 0 {
return;
}
- self.seen = current;
- self.summary.feed = current.since(self.start);
- if let Some(dropped) = self
- .feed_log
- .record(new.mailbox_full + new.fanout_full, now)
- {
+ self.mailbox_seen = mailbox;
+ self.ingress_seen = ingress;
+ self.summary.mailbox_full = mailbox.saturating_sub(self.mailbox_start);
+ self.summary.ingress_full = ingress;
+ if let Some(dropped) = self.feed_log.record(new, now) {
tracing::warn!(
dropped,
- mailbox_full = self.summary.feed.mailbox_full,
- ingress_full = self.summary.feed.fanout_full,
+ mailbox_full = self.summary.mailbox_full,
+ ingress_full = self.summary.ingress_full,
"Native camera frames lost before the recording received them"
);
self.emit("camera-ingress:feed");
@@ -156,8 +164,7 @@ impl IngressReporter {
return;
};
let lost = self.summary.lost();
- let offered =
- self.summary.received + self.summary.feed.mailbox_full + self.summary.feed.fanout_full;
+ let offered = self.summary.received + self.summary.mailbox_full + self.summary.ingress_full;
emit_health(
health_tx,
PipelineHealthEvent::FrameDropRateHigh {
@@ -174,8 +181,8 @@ impl IngressReporter {
tracing::warn!(
received = summary.received,
sent = summary.sent,
- mailbox_full = summary.feed.mailbox_full,
- ingress_full = summary.feed.fanout_full,
+ mailbox_full = summary.mailbox_full,
+ ingress_full = summary.ingress_full,
encoder_queue_full = summary.forwarder_full,
?elapsed,
"Native camera source finished with lost frames"
@@ -275,10 +282,16 @@ impl VideoSource for NativeCamera {
let capacity = native_ingress_capacity(feed_lock.video_info());
tracing::debug!(capacity, "Native camera ingress queue");
let (tx, rx) = flume::bounded(capacity);
+ let ingress_dropped = Arc::new(AtomicU64::new(0));
+ let report = IngressReporter::new(
+ Some(feed_lock.native_ingress().clone()),
+ ingress_dropped.clone(),
+ Some(ctx.health_tx().clone()),
+ );
tokio::time::timeout(
CAMERA_SENDER_ATTACH_TIMEOUT,
- feed_lock.ask(camera::AddNativeSender(tx)),
+ feed_lock.ask(camera::AddCountedNativeSender(tx, ingress_dropped)),
)
.await
.map_err(|_| anyhow!("Timed out adding native camera sender"))?
@@ -288,10 +301,6 @@ impl VideoSource for NativeCamera {
let stopped = Arc::new(AtomicBool::new(false));
let stopped_clone = stopped.clone();
- let report = IngressReporter::new(
- Some(feed_lock.native_ingress().clone()),
- Some(ctx.health_tx().clone()),
- );
let task =
forward_native_frames(rx, video_tx, stop_rx, stopped_clone, cfg!(windows), report)
.map(|result| result.map(drop));
@@ -341,7 +350,7 @@ mod forwarding_tests {
stopped,
Arc::new(AtomicBool::new(false)),
true,
- IngressReporter::new(None, None),
+ IngressReporter::new(None, Default::default(), None),
)
.await;
assert!(
@@ -365,7 +374,7 @@ mod forwarding_tests {
stopped,
Arc::new(AtomicBool::new(false)),
true,
- IngressReporter::new(None, None),
+ IngressReporter::new(None, Default::default(), None),
)
.await
.unwrap();
@@ -383,7 +392,7 @@ mod forwarding_tests {
stopped,
Arc::new(AtomicBool::new(false)),
true,
- IngressReporter::new(None, None),
+ IngressReporter::new(None, Default::default(), None),
));
sender.send_async(17_u8).await.unwrap();
assert_eq!(frames.next().await, Some(17));
@@ -406,7 +415,7 @@ mod forwarding_tests {
stopped,
Arc::new(AtomicBool::new(false)),
true,
- IngressReporter::new(None, None),
+ IngressReporter::new(None, Default::default(), None),
)
.await;
assert!(
@@ -429,7 +438,7 @@ mod forwarding_tests {
stopped,
Arc::new(AtomicBool::new(false)),
false,
- IngressReporter::new(None, None),
+ IngressReporter::new(None, Default::default(), None),
)
.await
.unwrap();
@@ -484,7 +493,7 @@ mod forwarding_tests {
stopped,
Arc::new(AtomicBool::new(false)),
false,
- IngressReporter::new(None, None),
+ IngressReporter::new(None, Default::default(), None),
)
.await
.unwrap();
@@ -499,9 +508,11 @@ mod forwarding_tests {
async fn feed_losses_count_from_the_recording_start() {
let ingress = Arc::new(NativeCameraIngress::default());
ingress.record_mailbox_full();
- let report = IngressReporter::new(Some(ingress.clone()), None);
+ let ingress_dropped = Arc::new(AtomicU64::new(0));
+ let report = IngressReporter::new(Some(ingress.clone()), ingress_dropped.clone(), None);
ingress.record_mailbox_full();
ingress.record_mailbox_full();
+ ingress_dropped.fetch_add(3, Ordering::Relaxed);
let (sender, receiver) = flume::bounded(4);
let (video, mut frames) = mpsc::channel(4);
let (_stop, stopped) = oneshot::channel();
@@ -517,8 +528,8 @@ mod forwarding_tests {
)
.await
.unwrap();
- assert_eq!(summary.feed.mailbox_full, 2);
- assert_eq!(summary.feed.fanout_full, 0);
+ assert_eq!(summary.mailbox_full, 2);
+ assert_eq!(summary.ingress_full, 3);
assert_eq!(summary.sent, 1);
assert_eq!(futures::StreamExt::next(&mut frames).await, Some(1));
}