Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(recording): Windows encoders repeat a stalled frame only after three intervals, behind real time
Richie McIlroy8a03f9e952c5
diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rsindex e28e3fc97..38c3650b7 100644--- a/crates/recording/src/output_pipeline/core.rs+++ b/crates/recording/src/output_pipeline/core.rs@@ -1991,6 +1991,12 @@ pub struct SharedPauseState {}impl SharedPauseState {+ /// The flag that is set while the recording is paused.+ #[cfg(windows)]+ pub(crate) fn flag(&self) -> Arc<AtomicBool> {+ self.flag.clone()+ }+pub fn new(flag: Arc<AtomicBool>) -> Self {Self {flag,diff --git a/crates/recording/src/output_pipeline/frame_pacing.rs b/crates/recording/src/output_pipeline/frame_pacing.rsnew file mode 100644index 000000000..1d0de4906--- /dev/null+++ b/crates/recording/src/output_pipeline/frame_pacing.rs@@ -0,0 +1,191 @@+//! Repeats the last frame while a Windows video source is stalled, so the+//! encoder keeps a steady track, without taking the timestamp of a frame that+//! is only late.+//!+//! The Windows encoder loops used to repeat the last frame as soon as one+//! frame interval passed without a new one, stamped one interval after it:+//! the slot a slightly late real frame is stamped for. The real frame then+//! landed on the same timestamp and the muxer moved it by one tick, so a+//! camera with ordinary arrival jitter recorded duplicated frames followed by+//! frames shown for 11 µs (16 of 262 in real-capture run 39000141528).++use std::{+ sync::{+ atomic::{AtomicBool, Ordering},+ mpsc::{Receiver, RecvTimeoutError},+ },+ time::{Duration, Instant},+};++/// Frame intervals of silence before the last frame is repeated. A repeat is+/// stamped one interval after the previous frame and sent once a frame for+/// that slot would be two intervals late.+const STALL_INTERVALS: u32 = 3;++pub(crate) enum Paced<F> {+ /// A frame from the source and its timestamp.+ Frame(F, Duration),+ /// Encode the last frame again at this timestamp.+ Repeat(Duration),+ End,+}++pub(crate) struct FramePacer {+ interval: Duration,+ last: Option<LastFrame>,+}++struct LastFrame {+ timestamp: Duration,+ arrived: Instant,+ repeats: u32,+}++impl FramePacer {+ pub(crate) fn new(frame_rate: u32) -> Self {+ Self {+ interval: Duration::from_secs_f64(1.0 / f64::from(frame_rate.max(1))),+ last: None,+ }+ }++ /// Blocks until a frame arrives, the source has been silent long enough+ /// to repeat the last one, or the stream ends. While `paused`, nothing is+ /// repeated and the next frame starts afresh.+ pub(crate) fn next<F>(+ &mut self,+ rx: &Receiver<Option<(F, Duration)>>,+ paused: &AtomicBool,+ ) -> Paced<F> {+ loop {+ let received = match &self.last {+ None => rx.recv().map_err(|_| RecvTimeoutError::Disconnected),+ Some(last) => {+ let due = last.arrived + self.interval * (STALL_INTERVALS + last.repeats);+ rx.recv_timeout(due.saturating_duration_since(Instant::now()))+ }+ };+ match received {+ Ok(Some((frame, timestamp))) => {+ self.last = Some(LastFrame {+ timestamp,+ arrived: Instant::now(),+ repeats: 0,+ });+ return Paced::Frame(frame, timestamp);+ }+ Ok(None) | Err(RecvTimeoutError::Disconnected) => return Paced::End,+ Err(RecvTimeoutError::Timeout) => {+ if paused.load(Ordering::Acquire) {+ self.last = None;+ continue;+ }+ let Some(last) = self.last.as_mut() else {+ continue;+ };+ last.repeats += 1;+ return Paced::Repeat(last.timestamp + self.interval * last.repeats);+ }+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::sync::mpsc::sync_channel;++ const FPS: u32 = 50;+ const INTERVAL: Duration = Duration::from_millis(20);++ fn at(n: u32) -> Duration {+ INTERVAL * n+ }++ #[test]+ fn jittered_frames_pass_through_without_repeats() {+ let (tx, rx) = sync_channel(64);+ let sender = std::thread::spawn(move || {+ for n in 0..20u32 {+ // Frames arrive up to 15 ms (three quarters of an interval) late.+ std::thread::sleep(if n % 2 == 0 {+ INTERVAL + INTERVAL * 3 / 4+ } else {+ INTERVAL / 4+ });+ tx.send(Some((n, at(n)))).unwrap();+ }+ tx.send(None).unwrap();+ });+ let mut pacer = FramePacer::new(FPS);+ let paused = AtomicBool::new(false);+ let mut seen = Vec::new();+ loop {+ match pacer.next(&rx, &paused) {+ Paced::Frame(n, ts) => seen.push((n, ts)),+ Paced::Repeat(ts) => panic!("repeat at {ts:?} for a frame that was only late"),+ Paced::End => break,+ }+ }+ sender.join().unwrap();+ assert_eq!(seen, (0..20).map(|n| (n, at(n))).collect::<Vec<_>>());+ }++ #[test]+ fn a_stall_repeats_the_last_frame_on_its_grid_behind_real_time() {+ let (tx, rx) = sync_channel(4);+ let mut pacer = FramePacer::new(FPS);+ let paused = AtomicBool::new(false);+ tx.send(Some((7u32, at(10)))).unwrap();+ let start = Instant::now();+ assert!(matches!(pacer.next(&rx, &paused), Paced::Frame(7, ts) if ts == at(10)));+ let Paced::Repeat(first) = pacer.next(&rx, &paused) else {+ panic!("expected a repeat");+ };+ assert_eq!(first, at(11));+ assert!(start.elapsed() >= INTERVAL * STALL_INTERVALS);+ let Paced::Repeat(second) = pacer.next(&rx, &paused) else {+ panic!("expected a repeat");+ };+ assert_eq!(second, at(12));+ assert!(start.elapsed() >= INTERVAL * (STALL_INTERVALS + 1));+ // The source resumes with a frame captured after the stall.+ tx.send(Some((8, at(16)))).unwrap();+ assert!(matches!(pacer.next(&rx, &paused), Paced::Frame(8, ts) if ts == at(16)));+ }++ #[test]+ fn nothing_repeats_while_paused() {+ let (tx, rx) = sync_channel(4);+ let mut pacer = FramePacer::new(FPS);+ let paused = AtomicBool::new(false);+ tx.send(Some((1u32, at(0)))).unwrap();+ assert!(matches!(pacer.next(&rx, &paused), Paced::Frame(1, _)));+ paused.store(true, Ordering::Release);+ let sender = std::thread::spawn(move || {+ std::thread::sleep(INTERVAL * 8);+ tx.send(Some((2, at(40)))).unwrap();+ tx.send(None).unwrap();+ });+ assert!(matches!(pacer.next(&rx, &paused), Paced::Frame(2, ts) if ts == at(40)));+ assert!(matches!(pacer.next(&rx, &paused), Paced::End));+ sender.join().unwrap();+ }++ #[test]+ fn the_stream_ends_on_none_or_disconnect() {+ let paused = AtomicBool::new(false);+ let (tx, rx) = sync_channel::<Option<(u32, Duration)>>(1);+ tx.send(None).unwrap();+ assert!(matches!(+ FramePacer::new(FPS).next(&rx, &paused),+ Paced::End+ ));+ drop(tx);+ assert!(matches!(+ FramePacer::new(FPS).next(&rx, &paused),+ Paced::End+ ));+ }+}diff --git a/crates/recording/src/output_pipeline/mod.rs b/crates/recording/src/output_pipeline/mod.rsindex e8f069a70..d5223a339 100644--- a/crates/recording/src/output_pipeline/mod.rs+++ b/crates/recording/src/output_pipeline/mod.rs@@ -1,6 +1,8 @@mod async_camera;mod core;pub mod ffmpeg;+#[cfg(any(windows, test))]+mod frame_pacing;#[cfg(target_os = "macos")]mod macos_fragmented_m4s;#[cfg(target_os = "macos")]diff --git a/crates/recording/src/output_pipeline/oop_fragmented_m4s_win.rs b/crates/recording/src/output_pipeline/oop_fragmented_m4s_win.rsindex 4c67f24eb..d90b19cf7 100644--- a/crates/recording/src/output_pipeline/oop_fragmented_m4s_win.rs+++ b/crates/recording/src/output_pipeline/oop_fragmented_m4s_win.rs@@ -2,6 +2,7 @@ use super::core::{BlockingThreadFinish, DiskSpaceMonitor, HealthSender, PipelineHealthEvent, SharedHealthSender,resolve_oop_thread_finish, wait_for_blocking_thread_finish,};+use super::frame_pacing::{FramePacer, Paced};use super::oop_muxer::{MuxerSubprocessConfig, MuxerSubprocessError, RespawningMuxerSubprocess, VideoStreamInit,resolve_muxer_binary,@@ -258,6 +259,7 @@ impl WindowsOOPFragmentedM4SMuxer {let base_path = self.base_path.clone();let video_config = self.video_config;+ let pause_flag = self.pause.flag();let segment_duration = self.segment_duration;let preset = self.preset;let bpp = self.bpp;@@ -363,7 +365,7 @@ impl WindowsOOPFragmentedM4SMuxer {return Err(err);}- let frame_interval = Duration::from_secs_f64(1.0 / video_config.fps() as f64);+ let mut pacer = FramePacer::new(video_config.fps());let mut last_ffmpeg_frame: Option<ffmpeg::frame::Video> = None;let mut last_timestamp: Option<Duration> = None;let mut first_timestamp: Option<Duration> = None;@@ -460,27 +462,30 @@ impl WindowsOOPFragmentedM4SMuxer {let convert_start = std::time::Instant::now();- let (ffmpeg_frame, timestamp) = match video_rx.recv_timeout(frame_interval) {- Ok(Some((frame, ts))) => match frame.as_ffmpeg() {- Ok(f) => {- last_ffmpeg_frame = Some(reference_video_frame(&f));- last_timestamp = Some(ts);- (Some(f), ts)- }- Err(e) => {- warn!("Windows OOP failed to convert D3D11 frame: {e:?}");- match (&last_ffmpeg_frame, last_timestamp) {- (Some(f), Some(last_ts)) => {- let new_ts = last_ts.saturating_add(frame_interval);- last_timestamp = Some(new_ts);- duplicated_frames += 1;- (Some(reference_video_frame(f)), new_ts)- }- _ => (None, Duration::ZERO),+ let (ffmpeg_frame, timestamp) = match pacer.next(&video_rx, &pause_flag) {+ Paced::Frame(frame, ts) => {+ last_timestamp = Some(ts);+ match frame.as_ffmpeg() {+ Ok(f) => {+ last_ffmpeg_frame = Some(reference_video_frame(&f));+ (Some(f), ts)+ }+ Err(e) => {+ warn!("Windows OOP failed to convert D3D11 frame: {e:?}");+ duplicated_frames += 1;+ (last_ffmpeg_frame.as_ref().map(reference_video_frame), ts)}}- },- Ok(None) | Err(RecvTimeoutError::Disconnected) => {+ }+ Paced::Repeat(ts) => {+ let Some(f) = &last_ffmpeg_frame else {+ continue;+ };+ last_timestamp = Some(ts);+ duplicated_frames += 1;+ (Some(reference_video_frame(f)), ts)+ }+ Paced::End => {let drain_start = std::time::Instant::now();let drain_timeout = Duration::from_millis(500);loop {@@ -516,17 +521,6 @@ impl WindowsOOPFragmentedM4SMuxer {}break;}- Err(RecvTimeoutError::Timeout) => {- match (&last_ffmpeg_frame, last_timestamp) {- (Some(f), Some(last_ts)) => {- let new_ts = last_ts.saturating_add(frame_interval);- last_timestamp = Some(new_ts);- duplicated_frames += 1;- (Some(reference_video_frame(f)), new_ts)- }- _ => continue,- }- }};let convert_elapsed_ms = convert_start.elapsed().as_millis();@@ -543,15 +537,6 @@ impl WindowsOOPFragmentedM4SMuxer {}let Some(ffmpeg_frame) = ffmpeg_frame else {- match video_rx.recv() {- Ok(Some((frame, ts))) => {- if let Ok(f) = frame.as_ffmpeg() {- last_ffmpeg_frame = Some(f);- last_timestamp = Some(ts);- }- }- Ok(None) | Err(_) => break,- }continue;};diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rsindex f291851c3..7ecb8a0e4 100644--- a/crates/recording/src/output_pipeline/win.rs+++ b/crates/recording/src/output_pipeline/win.rs@@ -1,6 +1,9 @@use crate::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoFrame, VideoMuxer,- output_pipeline::{HealthSender, PipelineHealthEvent, emit_health},+ output_pipeline::{+ HealthSender, PipelineHealthEvent, emit_health,+ frame_pacing::{FramePacer, Paced},+ },screen_capture,};use anyhow::{Context, anyhow};@@ -337,70 +340,43 @@ impl Muxer for WindowsMuxer {match encoder {either::Left((mut encoder, mut muxer)) => {trace!("Running native encoder with frame pacing");- let frame_interval = Duration::from_secs_f64(1.0 / config.frame_rate as f64);+ let mut pacer = FramePacer::new(config.frame_rate);let mut last_texture: Option<windows::Win32::Graphics::Direct3D11::ID3D11Texture2D> = None;let mut first_timestamp: Option<Duration> = None;- let mut last_timestamp: Option<Duration> = None;let mut frame_count: u64 = 0;let mut frames_reused: u64 = 0;let result = encoder.run(Arc::new(AtomicBool::default()),- || {- loop {- match video_rx.recv_timeout(frame_interval) {- Ok(Some((frame, timestamp))) => {- last_texture = Some(frame.texture().clone());- last_timestamp = Some(timestamp);- }- Ok(None) => {- trace!("End of stream signal received");- return Ok(None);- }- Err(RecvTimeoutError::Timeout) => {- if pause_flag.load(Ordering::Acquire) {- last_timestamp = None;- continue;- } else if let Some(last_ts) = last_timestamp {- let new_ts = last_ts.saturating_add(frame_interval);- last_timestamp = Some(new_ts);- frames_reused += 1;- if frames_reused.is_multiple_of(30) {- debug!(- frames_reused = frames_reused,- frame_count = frame_count,- "Frame pacing: reusing frames due to slow capture"- );- }- }- }- Err(RecvTimeoutError::Disconnected) => {- trace!("Channel disconnected");- return Ok(None);- }+ || loop {+ let (texture, timestamp) = match pacer.next(&video_rx, &pause_flag) {+ Paced::Frame(frame, timestamp) => {+ let texture = frame.texture().clone();+ last_texture = Some(texture.clone());+ (texture, timestamp)}-- if let (Some(texture), Some(ts)) = (&last_texture, last_timestamp) {- let normalized_ts = normalize_timestamp(ts, &mut first_timestamp);- frame_count += 1;- let frame_time = duration_to_timespan(normalized_ts);- return Ok(Some((texture.clone(), frame_time)));- } else {- match video_rx.recv() {- Ok(Some((frame, timestamp))) => {- let texture = frame.texture().clone();- last_texture = Some(texture.clone());- last_timestamp = Some(timestamp);- let normalized_ts =- normalize_timestamp(timestamp, &mut first_timestamp);- frame_count = 1;- let frame_time = duration_to_timespan(normalized_ts);- return Ok(Some((texture, frame_time)));- }- Ok(None) | Err(_) => return Ok(None),+ Paced::Repeat(timestamp) => {+ let Some(texture) = &last_texture else {+ continue;+ };+ frames_reused += 1;+ if frames_reused.is_multiple_of(30) {+ debug!(+ frames_reused = frames_reused,+ frame_count = frame_count,+ "Frame pacing: reusing frames due to slow capture"+ );}+ (texture.clone(), timestamp)}- }+ Paced::End => {+ trace!("End of stream signal received");+ return Ok(None);+ }+ };+ let normalized_ts = normalize_timestamp(timestamp, &mut first_timestamp);+ frame_count += 1;+ return Ok(Some((texture, duration_to_timespan(normalized_ts))));},|output_sample| {let Ok(mut output) = output.lock() else {@@ -438,63 +414,33 @@ impl Muxer for WindowsMuxer {}either::Right(mut encoder) => {trace!("Running FFmpeg encoder with frame pacing");- let frame_interval = Duration::from_secs_f64(1.0 / config.frame_rate as f64);+ let mut pacer = FramePacer::new(config.frame_rate);let mut last_ffmpeg_frame: Option<ffmpeg::frame::Video> = None;let mut first_timestamp: Option<Duration> = None;- let mut last_timestamp: Option<Duration> = None;loop {- let (ffmpeg_frame, ts) = match video_rx.recv_timeout(frame_interval) {- Ok(Some((frame, timestamp))) => {- last_timestamp = Some(timestamp);- match frame.as_ffmpeg() {- Ok(f) => {- last_ffmpeg_frame = Some(reference_video_frame(&f));- (Some(f), timestamp)- }- Err(e) => {- warn!("Failed to convert frame: {e:?}");- (- last_ffmpeg_frame- .as_ref()- .map(reference_video_frame),- timestamp,- )- }- }- }- Ok(None) => break,- Err(RecvTimeoutError::Timeout) => {- if pause_flag.load(Ordering::Acquire) {- last_timestamp = None;- continue;+ let (ffmpeg_frame, ts) = match pacer.next(&video_rx, &pause_flag) {+ Paced::Frame(frame, timestamp) => match frame.as_ffmpeg() {+ Ok(f) => {+ last_ffmpeg_frame = Some(reference_video_frame(&f));+ (Some(f), timestamp)}- if let Some(last_ts) = last_timestamp {- let new_ts = last_ts.saturating_add(frame_interval);- last_timestamp = Some(new_ts);+ Err(e) => {+ warn!("Failed to convert frame: {e:?}");(- last_ffmpeg_frame- .as_ref()- .map(reference_video_frame),- new_ts,+ last_ffmpeg_frame.as_ref().map(reference_video_frame),+ timestamp,)- } else {- continue;}- }- Err(RecvTimeoutError::Disconnected) => break,+ },+ Paced::Repeat(timestamp) => (+ last_ffmpeg_frame.as_ref().map(reference_video_frame),+ timestamp,+ ),+ Paced::End => break,};let Some(ffmpeg_frame) = ffmpeg_frame else {- match video_rx.recv() {- Ok(Some((frame, timestamp))) => {- last_timestamp = Some(timestamp);- if let Ok(f) = frame.as_ffmpeg() {- last_ffmpeg_frame = Some(f);- }- }- Ok(None) | Err(_) => break,- }continue;};@@ -927,75 +873,38 @@ impl Muxer for WindowsCameraMuxer {frame_rate);- let frame_interval = Duration::from_secs_f64(1.0 / frame_rate as f64);+ let mut pacer = FramePacer::new(frame_rate);let mut last_frame: Option<NativeCameraFrame> = None;let mut first_timestamp: Option<Duration> = None;- let mut last_timestamp: Option<Duration> = None;let mut frame_count = 0u64;let mut camera_buffers = CameraBuffers::new();let result = encoder.run(Arc::new(AtomicBool::default()),- || {- loop {- match video_rx.recv_timeout(frame_interval) {- Ok(Some((frame, timestamp))) => {- last_frame = Some(frame);- last_timestamp = Some(timestamp);- }- Ok(None) => {- trace!("End of camera stream signal received");- return Ok(None);- }- Err(RecvTimeoutError::Timeout) => {- if pause_flag.load(Ordering::Acquire) {- last_timestamp = None;- continue;- } else if let Some(last_ts) = last_timestamp {- let new_ts = last_ts.saturating_add(frame_interval);- last_timestamp = Some(new_ts);- }- }- Err(RecvTimeoutError::Disconnected) => {- trace!("Camera channel disconnected");- return Ok(None);- }+ || loop {+ let timestamp = match pacer.next(&video_rx, &pause_flag) {+ Paced::Frame(frame, timestamp) => {+ last_frame = Some(frame);+ timestamp}-- if let (Some(frame), Some(ts)) = (&last_frame, last_timestamp) {- let normalized_ts =- normalize_camera_timestamp(ts, &mut first_timestamp);- frame_count += 1;- if frame_count.is_multiple_of(30) {- debug!(- "Windows camera encoder: processed {} frames",- frame_count- );- }- let texture =- upload_mf_buffer_to_texture(&d3d_device, frame, &mut camera_buffers)?;- return Ok(Some((texture, duration_to_timespan(normalized_ts))));- } else {- match video_rx.recv() {- Ok(Some((frame, timestamp))) => {- last_frame = Some(frame.clone());- last_timestamp = Some(timestamp);- let normalized_ts = normalize_camera_timestamp(- timestamp,- &mut first_timestamp,- );- frame_count = 1;- let texture = upload_mf_buffer_to_texture(- &d3d_device,- &frame,- &mut camera_buffers,- )?;- return Ok(Some((texture, duration_to_timespan(normalized_ts))));- }- Ok(None) | Err(_) => return Ok(None),- }+ Paced::Repeat(timestamp) => timestamp,+ Paced::End => {+ trace!("End of camera stream signal received");+ return Ok(None);}+ };+ let Some(frame) = &last_frame else {+ continue;+ };+ let normalized_ts =+ normalize_camera_timestamp(timestamp, &mut first_timestamp);+ frame_count += 1;+ if frame_count.is_multiple_of(30) {+ debug!("Windows camera encoder: processed {} frames", frame_count);}+ let texture =+ upload_mf_buffer_to_texture(&d3d_device, frame, &mut camera_buffers)?;+ return Ok(Some((texture, duration_to_timespan(normalized_ts))));},|output_sample| {let mut output = match output.lock() {@@ -1047,44 +956,21 @@ impl Muxer for WindowsCameraMuxer {frame_rate);- let frame_interval = Duration::from_secs_f64(1.0 / frame_rate as f64);+ let mut pacer = FramePacer::new(frame_rate);let mut last_frame: Option<NativeCameraFrame> = None;let mut first_timestamp: Option<Duration> = None;- let mut last_timestamp: Option<Duration> = None;let mut frame_count = 0u64;loop {- let (frame_to_encode, ts) = match video_rx.recv_timeout(frame_interval) {- Ok(Some((frame, timestamp))) => {- last_frame = Some(frame.clone());- last_timestamp = Some(timestamp);- (Some(frame), timestamp)+ let ts = match pacer.next(&video_rx, &pause_flag) {+ Paced::Frame(frame, timestamp) => {+ last_frame = Some(frame);+ timestamp}- Ok(None) => break,- Err(RecvTimeoutError::Timeout) => {- if pause_flag.load(Ordering::Acquire) {- last_timestamp = None;- continue;- }- if let Some(last_ts) = last_timestamp {- let new_ts = last_ts.saturating_add(frame_interval);- last_timestamp = Some(new_ts);- (last_frame.clone(), new_ts)- } else {- continue;- }- }- Err(RecvTimeoutError::Disconnected) => break,+ Paced::Repeat(timestamp) => timestamp,+ Paced::End => break,};-- let Some(frame) = frame_to_encode else {- match video_rx.recv() {- Ok(Some((frame, timestamp))) => {- last_frame = Some(frame.clone());- last_timestamp = Some(timestamp);- }- Ok(None) | Err(_) => break,- }+ let Some(frame) = last_frame.clone() else {continue;};diff --git a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs b/crates/recording/src/output_pipeline/win_fragmented_m4s.rsindex 303c82ec6..67278a913 100644--- a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs+++ b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs@@ -2,6 +2,7 @@ use super::core::{BlockingThreadFinish, DiskSpaceMonitor, HealthSender, PipelineHealthEvent, SharedHealthSender,combine_finish_errors, wait_for_blocking_thread_finish,};+use super::frame_pacing::{FramePacer, Paced};use super::win::reference_video_frame;use crate::{AudioFrame, AudioMuxer, Muxer, SharedPauseState, TaskPool, VideoMuxer,@@ -359,6 +360,7 @@ impl WindowsFragmentedM4SMuxer {let video_config = self.video_config;let health_tx = self.health_tx.clone();let base_path = self.base_path.clone();+ let pause_flag = self.pause.flag();let encoder_handle = std::thread::Builder::new().name("win-m4s-segment-encoder".to_string()).spawn(move || {@@ -368,9 +370,8 @@ impl WindowsFragmentedM4SMuxer {return Err(anyhow!("Failed to send ready signal - receiver dropped"));}- let frame_interval = Duration::from_secs_f64(1.0 / video_config.fps() as f64);+ let mut pacer = FramePacer::new(video_config.fps());let mut last_ffmpeg_frame: Option<ffmpeg::frame::Video> = None;- let mut last_timestamp: Option<Duration> = None;let mut first_timestamp: Option<Duration> = None;let mut slow_convert_count = 0u32;@@ -444,27 +445,26 @@ impl WindowsFragmentedM4SMuxer {let convert_start = std::time::Instant::now();- let (ffmpeg_frame, timestamp) = match video_rx.recv_timeout(frame_interval) {- Ok(Some((frame, ts))) => match frame.as_ffmpeg() {+ let (ffmpeg_frame, timestamp) = match pacer.next(&video_rx, &pause_flag) {+ Paced::Frame(frame, ts) => match frame.as_ffmpeg() {Ok(f) => {last_ffmpeg_frame = Some(reference_video_frame(&f));- last_timestamp = Some(ts);(Some(f), ts)}Err(e) => {warn!("Failed to convert D3D11 frame: {e:?}");- match (&last_ffmpeg_frame, last_timestamp) {- (Some(f), Some(last_ts)) => {- let new_ts = last_ts.saturating_add(frame_interval);- last_timestamp = Some(new_ts);- duplicated_frames += 1;- (Some(reference_video_frame(f)), new_ts)- }- _ => (None, Duration::ZERO),- }+ duplicated_frames += 1;+ (last_ffmpeg_frame.as_ref().map(reference_video_frame), ts)}},- Ok(None) | Err(RecvTimeoutError::Disconnected) => {+ Paced::Repeat(ts) => {+ let Some(f) = &last_ffmpeg_frame else {+ continue;+ };+ duplicated_frames += 1;+ (Some(reference_video_frame(f)), ts)+ }+ Paced::End => {let mut drained = 0u64;let drain_start = std::time::Instant::now();let drain_timeout = Duration::from_millis(500);@@ -503,31 +503,11 @@ impl WindowsFragmentedM4SMuxer {}break;}- Err(RecvTimeoutError::Timeout) => {- match (&last_ffmpeg_frame, last_timestamp) {- (Some(f), Some(last_ts)) => {- let new_ts = last_ts.saturating_add(frame_interval);- last_timestamp = Some(new_ts);- duplicated_frames += 1;- (Some(reference_video_frame(f)), new_ts)- }- _ => continue,- }- }};let convert_elapsed_ms = convert_start.elapsed().as_millis();let Some(ffmpeg_frame) = ffmpeg_frame else {- match video_rx.recv() {- Ok(Some((frame, ts))) => {- if let Ok(f) = frame.as_ffmpeg() {- last_ffmpeg_frame = Some(f);- last_timestamp = Some(ts);- }- }- Ok(None) | Err(_) => break,- }continue;};@@ -828,6 +808,7 @@ impl WindowsFragmentedM4SCameraMuxer {let video_config = self.video_config;let health_tx = self.health_tx.clone();let base_path = self.base_path.clone();+ let pause_flag = self.pause.flag();let encoder_handle = std::thread::Builder::new().name("win-m4s-camera-segment-encoder".to_string()).spawn(move || {@@ -839,9 +820,8 @@ impl WindowsFragmentedM4SCameraMuxer {));}- let frame_interval = Duration::from_secs_f64(1.0 / video_config.fps() as f64);+ let mut pacer = FramePacer::new(video_config.fps());let mut last_frame: Option<NativeCameraFrame> = None;- let mut last_timestamp: Option<Duration> = None;let mut first_timestamp: Option<Duration> = None;let mut slow_convert_count = 0u32;@@ -940,13 +920,19 @@ impl WindowsFragmentedM4SCameraMuxer {break;}- let (frame_to_encode, timestamp) = match video_rx.recv_timeout(frame_interval) {- Ok(Some((frame, ts))) => {+ let (camera_frame, timestamp) = match pacer.next(&video_rx, &pause_flag) {+ Paced::Frame(frame, ts) => {last_frame = Some(frame.clone());- last_timestamp = Some(ts);- (Some(frame), ts)+ (frame, ts)}- Ok(None) | Err(RecvTimeoutError::Disconnected) => {+ Paced::Repeat(ts) => {+ let Some(frame) = &last_frame else {+ continue;+ };+ duplicated_frames += 1;+ (frame.clone(), ts)+ }+ Paced::End => {let mut drained = 0u64;let drain_start = std::time::Instant::now();let drain_timeout = Duration::from_millis(500);@@ -987,28 +973,6 @@ impl WindowsFragmentedM4SCameraMuxer {}break;}- Err(RecvTimeoutError::Timeout) => {- match (&last_frame, last_timestamp) {- (Some(f), Some(last_ts)) => {- let new_ts = last_ts.saturating_add(frame_interval);- last_timestamp = Some(new_ts);- duplicated_frames += 1;- (Some(f.clone()), new_ts)- }- _ => continue,- }- }- };-- let Some(camera_frame) = frame_to_encode else {- match video_rx.recv() {- Ok(Some((frame, ts))) => {- last_frame = Some(frame);- last_timestamp = Some(ts);- }- Ok(None) | Err(_) => break,- }- continue;};let normalized_ts = normalize_timestamp(timestamp, &mut first_timestamp);