Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(export): keep offline export frames in order when decode or readback is slow

Richie McIlroy98d1b9e562b0
diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs
index 366a28826..1d93e0a16 100644
--- a/crates/rendering/src/decoder/mod.rs
+++ b/crates/rendering/src/decoder/mod.rs
@@ -731,6 +731,24 @@ pub(super) fn record_pts_hole(
}
}
+/// How long an offline export waits for one decoded frame or one GPU readback
+/// before it stops with an error. Deadlines shorter than this only log there:
+/// export output must not depend on how loaded the machine is.
+pub(crate) const EXPORT_STALL_LIMIT: Duration = Duration::from_secs(120);
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum FrameDeadline {
+ /// Playback: a missed deadline returns no frame so the caller can move on.
+ Drop,
+ /// Offline export: a missed deadline keeps waiting for the same reply.
+ Wait,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) struct DecodeStalled {
+ pub waited: Duration,
+}
+
#[derive(Clone)]
pub struct AsyncVideoDecoderHandle {
sender: mpsc::Sender<VideoDecoderMessage>,
@@ -767,8 +785,15 @@ impl AsyncVideoDecoderHandle {
}
pub async fn get_frame(&self, time: f32) -> Option<DecodedFrame> {
- self.get_frame_with_timeout(time, self.normal_timeout_ms(), self.max_fallback_distance)
- .await
+ self.get_frame_with_timeout(
+ time,
+ self.normal_timeout_ms(),
+ self.max_fallback_distance,
+ FrameDeadline::Drop,
+ )
+ .await
+ .ok()
+ .flatten()
}
pub async fn get_frame_initial(&self, time: f32) -> Option<DecodedFrame> {
@@ -777,8 +802,32 @@ impl AsyncVideoDecoderHandle {
Self::INITIAL_SEEK_TIMEOUT_MS,
self.max_fallback_distance
.min(Self::INITIAL_MAX_FALLBACK_DISTANCE),
+ FrameDeadline::Drop,
)
.await
+ .ok()
+ .flatten()
+ }
+
+ /// Offline export: the decode deadline only triggers a warning. The same
+ /// request keeps waiting, so a slow machine gets the same frame later
+ /// instead of none. `Err` only after `EXPORT_STALL_LIMIT`.
+ pub(crate) async fn get_frame_for_export(
+ &self,
+ time: f32,
+ initial: bool,
+ ) -> Result<Option<DecodedFrame>, DecodeStalled> {
+ let (timeout_ms, max_fallback_distance) = if initial {
+ (
+ Self::INITIAL_SEEK_TIMEOUT_MS,
+ self.max_fallback_distance
+ .min(Self::INITIAL_MAX_FALLBACK_DISTANCE),
+ )
+ } else {
+ (self.normal_timeout_ms(), self.max_fallback_distance)
+ };
+ self.get_frame_with_timeout(time, timeout_ms, max_fallback_distance, FrameDeadline::Wait)
+ .await
}
async fn get_frame_with_timeout(
@@ -786,24 +835,25 @@ impl AsyncVideoDecoderHandle {
time: f32,
timeout_ms: u64,
max_fallback_distance: u32,
- ) -> Option<DecodedFrame> {
+ deadline: FrameDeadline,
+ ) -> Result<Option<DecodedFrame>, DecodeStalled> {
#[cfg(target_os = "macos")]
if let Some(fallback) = &self.fallback {
if let Some(result) = fallback.decoder.get() {
return match result {
Ok(decoder) => {
decoder
- .request_frame(time, timeout_ms, max_fallback_distance)
+ .request_frame(time, timeout_ms, max_fallback_distance, deadline)
.await
}
- Err(_) => None,
+ Err(_) => Ok(None),
};
}
- if let Some(frame) = self
- .request_frame(time, timeout_ms, max_fallback_distance)
+ if let Ok(Some(frame)) = self
+ .request_frame(time, timeout_ms, max_fallback_distance, deadline)
.await
{
- return Some(frame);
+ return Ok(Some(frame));
}
let result = fallback
.decoder
@@ -829,16 +879,16 @@ impl AsyncVideoDecoderHandle {
return match result {
Ok(decoder) => {
decoder
- .request_frame(time, timeout_ms, max_fallback_distance)
+ .request_frame(time, timeout_ms, max_fallback_distance, deadline)
.await
}
Err(error) => {
tracing::warn!(%error, "Software video decoder fallback failed");
- None
+ Ok(None)
}
};
}
- self.request_frame(time, timeout_ms, max_fallback_distance)
+ self.request_frame(time, timeout_ms, max_fallback_distance, deadline)
.await
}
@@ -847,8 +897,9 @@ impl AsyncVideoDecoderHandle {
time: f32,
timeout_ms: u64,
max_fallback_distance: u32,
- ) -> Option<DecodedFrame> {
- let (tx, rx) = tokio::sync::oneshot::channel();
+ deadline: FrameDeadline,
+ ) -> Result<Option<DecodedFrame>, DecodeStalled> {
+ let (tx, mut rx) = tokio::sync::oneshot::channel();
let adjusted_time = self.get_time(time);
if self
@@ -864,18 +915,38 @@ impl AsyncVideoDecoderHandle {
time = adjusted_time,
"decoder thread is gone; frame request dropped"
);
- return None;
+ return Ok(None);
}
- match tokio::time::timeout(std::time::Duration::from_millis(timeout_ms), rx).await {
- Ok(result) => result.ok(),
- Err(_) => {
- tracing::warn!(
- time = adjusted_time,
- timeout_ms = timeout_ms,
- "Frame decode request timed out"
- );
- None
+ let started = tokio::time::Instant::now();
+ loop {
+ match tokio::time::timeout(Duration::from_millis(timeout_ms), &mut rx).await {
+ Ok(result) => return Ok(result.ok()),
+ Err(_) if deadline == FrameDeadline::Wait => {
+ let waited = started.elapsed();
+ if waited >= EXPORT_STALL_LIMIT {
+ tracing::error!(
+ time = adjusted_time,
+ waited_ms = waited.as_millis() as u64,
+ "Frame decode request stalled"
+ );
+ return Err(DecodeStalled { waited });
+ }
+ tracing::warn!(
+ time = adjusted_time,
+ timeout_ms = timeout_ms,
+ waited_ms = waited.as_millis() as u64,
+ "Frame decode is slow; still waiting for the same frame"
+ );
+ }
+ Err(_) => {
+ tracing::warn!(
+ time = adjusted_time,
+ timeout_ms = timeout_ms,
+ "Frame decode request timed out"
+ );
+ return Ok(None);
+ }
}
}
}
@@ -884,6 +955,30 @@ impl AsyncVideoDecoderHandle {
time + self.offset as f32
}
+ /// A handle served by a test thread instead of a real decoder.
+ #[cfg(test)]
+ pub(crate) fn for_test(
+ sender: mpsc::Sender<VideoDecoderMessage>,
+ video_width: u32,
+ video_height: u32,
+ ) -> Self {
+ Self {
+ sender,
+ offset: 0.0,
+ status: DecoderStatus {
+ decoder_type: DecoderType::FFmpegSoftware,
+ video_width,
+ video_height,
+ fallback_reason: None,
+ },
+ max_fallback_distance: DEFAULT_MAX_FALLBACK_DISTANCE,
+ #[cfg(target_os = "macos")]
+ fallback: None,
+ #[cfg(target_os = "macos")]
+ sequential_access: None,
+ }
+ }
+
pub fn decoder_status(&self) -> &DecoderStatus {
#[cfg(target_os = "macos")]
if let Some(Ok(decoder)) = self
@@ -2303,11 +2398,14 @@ mod managed_worker_tests {
};
let cloned = decoder.clone();
let (first, second) = tokio::join!(
- decoder.get_frame_with_timeout(0.0, 200, 2),
- cloned.get_frame_with_timeout(0.0, 200, 2)
+ decoder.get_frame_with_timeout(0.0, 200, 2, FrameDeadline::Drop),
+ cloned.get_frame_with_timeout(0.0, 200, 2, FrameDeadline::Drop)
+ );
+ let first = first.unwrap().unwrap();
+ assert_eq!(
+ frame_pixels(&first),
+ frame_pixels(&second.unwrap().unwrap())
);
- let first = first.unwrap();
- assert_eq!(frame_pixels(&first), frame_pixels(&second.unwrap()));
assert_eq!(decoder.decoder_type(), DecoderType::FFmpegSoftware);
assert_eq!(cloned.decoder_type(), DecoderType::FFmpegSoftware);
assert!(decoder.fallback_reason().is_some());
diff --git a/crates/rendering/src/frame_pipeline.rs b/crates/rendering/src/frame_pipeline.rs
index 938433383..3de1e865c 100644
--- a/crates/rendering/src/frame_pipeline.rs
+++ b/crates/rendering/src/frame_pipeline.rs
@@ -27,7 +27,22 @@ pub(crate) fn note_software_adapter_in_use() {
SOFTWARE_ADAPTER_IN_USE.store(true, Ordering::Release);
}
+tokio::task_local! {
+ static OFFLINE_EXPORT: ();
+}
+
+/// Runs an offline export render loop. Inside it a slow GPU readback is waited
+/// for up to the export stall limit instead of failing at the playback
+/// deadline: a failed readback costs the export a frame, and export output
+/// must not depend on how loaded the machine is.
+pub(crate) async fn offline_export<F: std::future::Future>(future: F) -> F::Output {
+ OFFLINE_EXPORT.scope((), future).await
+}
+
fn gpu_buffer_wait_timeout() -> std::time::Duration {
+ if OFFLINE_EXPORT.try_with(|_| ()).is_ok() {
+ return crate::decoder::EXPORT_STALL_LIMIT;
+ }
let secs = if SOFTWARE_ADAPTER_IN_USE.load(Ordering::Acquire) {
SOFTWARE_GPU_BUFFER_WAIT_TIMEOUT_SECS
} else {
@@ -36,6 +51,44 @@ fn gpu_buffer_wait_timeout() -> std::time::Duration {
std::time::Duration::from_secs(secs)
}
+/// How long the readback of `frame_number` has been waited for.
+fn readback_waited(started: Instant, frame_number: u32) -> std::time::Duration {
+ #[cfg(test)]
+ {
+ started.elapsed() + readback_delay::injected(frame_number)
+ }
+ #[cfg(not(test))]
+ {
+ let _ = frame_number;
+ started.elapsed()
+ }
+}
+
+/// Test seam: makes one frame's readback look older than it is, to exercise
+/// the readback deadlines without waiting for them.
+#[cfg(test)]
+pub(crate) mod readback_delay {
+ use std::{cell::RefCell, collections::HashMap, time::Duration};
+
+ thread_local! {
+ static DELAYS: RefCell<HashMap<u32, Duration>> = RefCell::new(HashMap::new());
+ }
+
+ pub(crate) fn inject(frame_number: u32, delay: Duration) {
+ DELAYS.with(|delays| delays.borrow_mut().insert(frame_number, delay));
+ }
+
+ pub(super) fn injected(frame_number: u32) -> Duration {
+ DELAYS.with(|delays| {
+ delays
+ .borrow()
+ .get(&frame_number)
+ .copied()
+ .unwrap_or_default()
+ })
+ }
+}
+
#[derive(Clone, Copy, Debug, Default)]
pub struct FinishEncoderTimings {
pub wait_previous_duration: std::time::Duration,
@@ -903,11 +956,15 @@ impl PendingNv12External {
return Err(RenderingError::BufferMapWaitingFailed);
};
let started = Instant::now();
+ let timeout_duration = gpu_buffer_wait_timeout();
let mut poll_count = 0u32;
- while !completed.load(Ordering::Acquire) {
- if started.elapsed() > gpu_buffer_wait_timeout() {
+ loop {
+ if readback_waited(started, self.frame_number) > timeout_duration {
return Err(RenderingError::BufferMapWaitingFailed);
}
+ if completed.load(Ordering::Acquire) {
+ break;
+ }
device.poll(wgpu::PollType::Poll)?;
poll_count += 1;
if poll_count < 10 {
@@ -979,11 +1036,15 @@ impl PendingNv12Surface {
return Err(RenderingError::BufferMapWaitingFailed);
};
let started = Instant::now();
+ let timeout_duration = gpu_buffer_wait_timeout();
let mut poll_count = 0u32;
- while !completed.load(Ordering::Acquire) {
- if started.elapsed() > gpu_buffer_wait_timeout() {
+ loop {
+ if readback_waited(started, self.frame_number) > timeout_duration {
return Err(RenderingError::BufferMapWaitingFailed);
}
+ if completed.load(Ordering::Acquire) {
+ break;
+ }
device.poll(wgpu::PollType::Poll)?;
poll_count += 1;
if poll_count < 10 {
@@ -1050,7 +1111,7 @@ impl PendingNv12Readback {
let timeout_duration = gpu_buffer_wait_timeout();
loop {
- if start_time.elapsed() > timeout_duration {
+ if readback_waited(start_time, self.frame_number) > timeout_duration {
return Err(self.cancel());
}
@@ -1597,7 +1658,7 @@ impl PendingReadback {
let timeout_duration = gpu_buffer_wait_timeout();
loop {
- if start_time.elapsed() > timeout_duration {
+ if readback_waited(start_time, self.frame_number) > timeout_duration {
tracing::error!(
frame_number = self.frame_number,
elapsed_secs = start_time.elapsed().as_secs(),
diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs
index 76b7f9490..036b1fac1 100644
--- a/crates/rendering/src/lib.rs
+++ b/crates/rendering/src/lib.rs
@@ -342,6 +342,27 @@ pub struct PreparedMask {
pub output_size: XY<u32>,
}
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum FrameFetch {
+ Playback,
+ PlaybackInitial,
+ Export { initial: bool },
+}
+
+impl FrameFetch {
+ async fn frame(
+ self,
+ decoder: &AsyncVideoDecoderHandle,
+ time: f32,
+ ) -> Result<Option<DecodedFrame>, decoder::DecodeStalled> {
+ match self {
+ Self::Playback => Ok(decoder.get_frame(time).await),
+ Self::PlaybackInitial => Ok(decoder.get_frame_initial(time).await),
+ Self::Export { initial } => decoder.get_frame_for_export(time, initial).await,
+ }
+ }
+}
+
#[derive(Clone)]
pub struct RecordingSegmentDecoders {
screen: AsyncVideoDecoderHandle,
@@ -427,65 +448,16 @@ impl RecordingSegmentDecoders {
needs_display: bool,
offsets: ClipOffsets,
) -> Option<DecodedSegmentFrames> {
- let (camera_request_time, recording_time) =
- segment_timing::segment_frame_times(segment_time, self.segment_offset, offsets);
- let (screen_width, screen_height) = self.screen_video_dimensions();
- let screen_size = XY::new(screen_width, screen_height);
-
- if needs_display {
- let (screen, camera) = tokio::join!(
- self.screen.get_frame(segment_time),
- OptionFuture::from(
- needs_camera
- .then(|| self
- .camera
- .as_ref()
- .map(|d| d.get_frame(camera_request_time)))
- .flatten()
- )
- );
-
- let camera_frame = camera.flatten();
-
- if screen.is_none() {
- tracing::warn!(segment_time, "screen decoder returned no frame");
- }
-
- Some(DecodedSegmentFrames {
- screen_size,
- screen_frame: Some(screen?),
- camera_frame,
- segment_time,
- recording_time,
- segment_has_camera: self.camera.is_some(),
- })
- } else {
- let camera_frame = OptionFuture::from(
- needs_camera
- .then(|| {
- self.camera
- .as_ref()
- .map(|d| d.get_frame(camera_request_time))
- })
- .flatten(),
- )
- .await
- .flatten();
-
- tracing::debug!(
- segment_time,
- "get_frames: skipping display decoding (needs_display=false)"
- );
-
- Some(DecodedSegmentFrames {
- screen_size,
- screen_frame: None,
- camera_frame,
- segment_time,
- recording_time,
- segment_has_camera: self.camera.is_some(),
- })
- }
+ self.fetch_frames(
+ segment_time,
+ needs_camera,
+ needs_display,
+ offsets,
+ FrameFetch::Playback,
+ )
+ .await
+ .ok()
+ .flatten()
}
pub async fn get_frames_initial(
@@ -495,60 +467,97 @@ impl RecordingSegmentDecoders {
needs_display: bool,
offsets: ClipOffsets,
) -> Option<DecodedSegmentFrames> {
+ self.fetch_frames(
+ segment_time,
+ needs_camera,
+ needs_display,
+ offsets,
+ FrameFetch::PlaybackInitial,
+ )
+ .await
+ .ok()
+ .flatten()
+ }
+
+ /// Offline export variant: screen and camera requests wait past their
+ /// playback deadlines for the same frame (see `EXPORT_STALL_LIMIT`).
+ async fn get_frames_for_export(
+ &self,
+ segment_time: f32,
+ needs_camera: bool,
+ needs_display: bool,
+ offsets: ClipOffsets,
+ initial: bool,
+ ) -> Result<Option<DecodedSegmentFrames>, decoder::DecodeStalled> {
+ self.fetch_frames(
+ segment_time,
+ needs_camera,
+ needs_display,
+ offsets,
+ FrameFetch::Export { initial },
+ )
+ .await
+ }
+
+ async fn fetch_frames(
+ &self,
+ segment_time: f32,
+ needs_camera: bool,
+ needs_display: bool,
+ offsets: ClipOffsets,
+ fetch: FrameFetch,
+ ) -> Result<Option<DecodedSegmentFrames>, decoder::DecodeStalled> {
let (camera_request_time, recording_time) =
segment_timing::segment_frame_times(segment_time, self.segment_offset, offsets);
let (screen_width, screen_height) = self.screen_video_dimensions();
let screen_size = XY::new(screen_width, screen_height);
+ let camera = needs_camera.then_some(self.camera.as_ref()).flatten();
if needs_display {
let (screen, camera) = tokio::join!(
- self.screen.get_frame_initial(segment_time),
- OptionFuture::from(
- needs_camera
- .then(|| self
- .camera
- .as_ref()
- .map(|d| d.get_frame_initial(camera_request_time)))
- .flatten()
- )
+ fetch.frame(&self.screen, segment_time),
+ OptionFuture::from(camera.map(|d| fetch.frame(d, camera_request_time)))
);
- let camera_frame = camera.flatten();
+ let camera_frame = camera.transpose()?.flatten();
+ let screen = screen?;
+
+ if screen.is_none() && fetch != FrameFetch::PlaybackInitial {
+ tracing::warn!(segment_time, "screen decoder returned no frame");
+ }
- Some(DecodedSegmentFrames {
+ let Some(screen) = screen else {
+ return Ok(None);
+ };
+
+ Ok(Some(DecodedSegmentFrames {
screen_size,
- screen_frame: Some(screen?),
+ screen_frame: Some(screen),
camera_frame,
segment_time,
recording_time,
segment_has_camera: self.camera.is_some(),
- })
+ }))
} else {
- let camera_frame = OptionFuture::from(
- needs_camera
- .then(|| {
- self.camera
- .as_ref()
- .map(|d| d.get_frame_initial(camera_request_time))
- })
- .flatten(),
- )
- .await
- .flatten();
+ let camera_frame =
+ OptionFuture::from(camera.map(|d| fetch.frame(d, camera_request_time)))
+ .await
+ .transpose()?
+ .flatten();
tracing::debug!(
segment_time,
- "get_frames_initial: skipping display decoding (needs_display=false)"
+ "get_frames: skipping display decoding (needs_display=false)"
);
- Some(DecodedSegmentFrames {
+ Ok(Some(DecodedSegmentFrames {
screen_size,
screen_frame: None,
camera_frame,
segment_time,
recording_time,
segment_has_camera: self.camera.is_some(),
- })
+ }))
}
}
@@ -618,6 +627,14 @@ pub enum RenderingError {
frame_number: u32,
consecutive_failures: u32,
},
+ #[error(
+ "Export stopped because frame {frame_number} took more than {waited_secs} seconds to decode. Close other apps that are using the computer heavily and try again."
+ )]
+ FrameDecodeStalled { frame_number: u32, waited_secs: u64 },
+ #[error(
+ "Export stopped because frame {frame_number} was lost while rendering. Try exporting again."
+ )]
+ ExportFrameLost { frame_number: u32 },
}
pub struct RenderSegment {
@@ -639,6 +656,34 @@ pub async fn render_video_to_channel(
resolution_base: XY<u32>,
recordings: &ProjectRecordingsMeta,
frame_windows: Option<FrameWindows>,
+) -> Result<(), RenderingError> {
+ frame_pipeline::offline_export(render_rgba_export(
+ constants,
+ project,
+ sender,
+ recording_meta,
+ meta,
+ render_segments,
+ fps,
+ resolution_base,
+ recordings,
+ frame_windows,
+ ))
+ .await
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn render_rgba_export(
+ constants: &RenderVideoConstants,
+ project: &ProjectConfiguration,
+ sender: mpsc::Sender<(RenderedFrame, u32)>,
+ recording_meta: &RecordingMeta,
+ meta: &StudioRecordingMeta,
+ render_segments: Vec<RenderSegment>,
+ fps: u32,
+ resolution_base: XY<u32>,
+ recordings: &ProjectRecordingsMeta,
+ frame_windows: Option<FrameWindows>,
) -> Result<(), RenderingError> {
ffmpeg::init().unwrap();
@@ -733,7 +778,8 @@ pub async fn render_video_to_channel(
let mut consecutive_failures = 0u32;
const MAX_CONSECUTIVE_FAILURES: u32 = 200;
- let mut prefetched_decode: Option<(u32, f64, usize, Option<DecodedSegmentFrames>)> = None;
+ let mut prefetched_decode: Option<PrefetchedDecode> = None;
+ let mut in_flight = InFlightFrame::default();
loop {
let Some(current_frame_number) = next_frame else {
@@ -811,7 +857,7 @@ pub async fn render_video_to_channel(
}
};
let (segment_frames, outgoing_frames) = if let Some((outgoing, _, _)) = transition_mapping {
- tokio::join!(
+ let (incoming, outgoing) = tokio::join!(
incoming_decode,
decode_timeline_source_frames(
project,
@@ -822,10 +868,12 @@ pub async fn render_video_to_channel(
is_initial_frame,
fps,
)
- )
+ );
+ (incoming, outgoing?)
} else {
(incoming_decode.await, None)
};
+ let segment_frames = segment_frames?;
if let Some(segment_frames) = segment_frames {
consecutive_failures = 0;
@@ -956,37 +1004,42 @@ pub async fn render_video_to_channel(
match render_result {
Ok(Some(frame)) if frame.width > 0 && frame.height > 0 => {
+ in_flight.rendered(current_frame_number, Some(frame.frame_number))?;
last_successful_frame = Some(frame.clone());
sender.send((frame, current_frame_number)).await?;
}
- Ok(Some(_)) => {
+ Ok(Some(frame)) => {
+ in_flight.rendered(current_frame_number, Some(frame.frame_number))?;
tracing::warn!(
- frame_number = current_frame_number,
+ frame_number = frame.frame_number,
"Rendered frame has zero dimensions"
);
if let Some(ref last_frame) = last_successful_frame {
- let mut fallback = last_frame.clone();
- fallback.frame_number = current_frame_number;
- fallback.target_time_ns =
- (current_frame_number as u64 * 1_000_000_000) / fps as u64;
+ let fallback = last_frame.held_as(frame.frame_number, fps);
sender.send((fallback, current_frame_number)).await?;
}
}
- Ok(None) => {}
+ Ok(None) => in_flight.rendered(current_frame_number, None)?,
Err(e) => {
tracing::error!(
frame_number = current_frame_number,
error = %e,
"Frame rendering failed"
);
+ if last_successful_frame.is_none() {
+ return Err(e);
+ }
+ // A substitute must not overtake the frame still in the GPU.
+ if let Some(frame) = in_flight.drain(frame_renderer.flush_pipeline().await)?
+ && frame.width > 0
+ && frame.height > 0
+ {
+ last_successful_frame = Some(frame.clone());
+ sender.send((frame, current_frame_number)).await?;
+ }
if let Some(ref last_frame) = last_successful_frame {
- let mut fallback = last_frame.clone();
- fallback.frame_number = current_frame_number;
- fallback.target_time_ns =
- (current_frame_number as u64 * 1_000_000_000) / fps as u64;
+ let fallback = last_frame.held_as(current_frame_number, fps);
sender.send((fallback, current_frame_number)).await?;
- } else {
- return Err(e);
}
}
}
@@ -1020,7 +1073,7 @@ pub async fn render_video_to_channel(
});
}
- if let Some(ref last_frame) = last_successful_frame {
+ if last_successful_frame.is_some() {
tracing::warn!(
frame_number = current_frame_number,
segment_time = segment_time,
@@ -1032,11 +1085,18 @@ pub async fn render_video_to_channel(
},
"Frame decode failed after retries - using previous frame"
);
- let mut fallback = last_frame.clone();
- fallback.frame_number = current_frame_number;
- fallback.target_time_ns =
- (current_frame_number as u64 * 1_000_000_000) / fps as u64;
- sender.send((fallback, current_frame_number)).await?;
+ // A substitute must not overtake the frame still in the GPU.
+ if let Some(frame) = in_flight.drain(frame_renderer.flush_pipeline().await)?
+ && frame.width > 0
+ && frame.height > 0
+ {
+ last_successful_frame = Some(frame.clone());
+ sender.send((frame, current_frame_number)).await?;
+ }
+ if let Some(ref last_frame) = last_successful_frame {
+ let fallback = last_frame.held_as(current_frame_number, fps);
+ sender.send((fallback, current_frame_number)).await?;
+ }
} else {
tracing::error!(
frame_number = current_frame_number,
@@ -1053,7 +1113,7 @@ pub async fn render_video_to_channel(
}
}
- if let Some(Ok(final_frame)) = frame_renderer.flush_pipeline().await
+ if let Some(final_frame) = in_flight.drain(frame_renderer.flush_pipeline().await)?
&& final_frame.width > 0
&& final_frame.height > 0
{
@@ -1200,6 +1260,38 @@ pub async fn render_video_to_channel_nv12(
frame_windows: Option<FrameWindows>,
stop_after_frames_sent: Option<u32>,
startup_breakdown_ms: Option<Arc<Mutex<Option<Nv12RenderStartupBreakdownMs>>>>,
+) -> Result<(), RenderingError> {
+ frame_pipeline::offline_export(render_nv12_export(
+ constants,
+ project,
+ sender,
+ recording_meta,
+ meta,
+ render_segments,
+ fps,
+ resolution_base,
+ recordings,
+ frame_windows,
+ stop_after_frames_sent,
+ startup_breakdown_ms,
+ ))
+ .await
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn render_nv12_export(
+ constants: &RenderVideoConstants,
+ project: &ProjectConfiguration,
+ sender: mpsc::Sender<(Nv12RenderedFrame, u32)>,
+ recording_meta: &RecordingMeta,
+ meta: &StudioRecordingMeta,
+ render_segments: Vec<RenderSegment>,
+ fps: u32,
+ resolution_base: XY<u32>,
+ recordings: &ProjectRecordingsMeta,
+ frame_windows: Option<FrameWindows>,
+ stop_after_frames_sent: Option<u32>,
+ startup_breakdown_ms: Option<Arc<Mutex<Option<Nv12RenderStartupBreakdownMs>>>>,
) -> Result<(), RenderingError> {
let ffmpeg_init_start = Instant::now();
ffmpeg::init().unwrap();
@@ -1315,7 +1407,8 @@ pub async fn render_video_to_channel_nv12(
let mut consecutive_failures = 0u32;
const MAX_CONSECUTIVE_FAILURES: u32 = 200;
- let mut prefetched_decode: Option<(u32, f64, usize, Option<DecodedSegmentFrames>)> = None;
+ let mut prefetched_decode: Option<PrefetchedDecode> = None;
+ let mut in_flight = InFlightFrame::default();
let mut channel_frames_sent = 0u32;
let mut stopped_after_frame_limit = false;
@@ -1401,7 +1494,7 @@ pub async fn render_video_to_channel_nv12(
}
};
let (segment_frames, outgoing_frames) = if let Some((outgoing, _, _)) = transition_mapping {
- tokio::join!(
+ let (incoming, outgoing) = tokio::join!(
incoming_decode,
decode_timeline_source_frames(
project,
@@ -1412,10 +1505,12 @@ pub async fn render_video_to_channel_nv12(
is_initial_frame,
fps,
)
- )
+ );
+ (incoming, outgoing?)
} else {
(incoming_decode.await, None)
};
+ let segment_frames = segment_frames?;
let this_decode_ms = decode_wall_start.elapsed().as_millis() as u64;
loop_stats::add(&loop_stats::DECODE, decode_wall_start.elapsed());
@@ -1663,6 +1758,7 @@ pub async fn render_video_to_channel_nv12(
match render_result {
Ok(Some(frame)) if frame.width > 0 && frame.height > 0 => {
+ in_flight.rendered(current_frame_number, Some(frame.frame_number))?;
if record_first_frame_nv12_phases {
if let Some(ref slot) = startup_breakdown_ms
&& let Ok(mut guard) = slot.lock()
@@ -1693,16 +1789,14 @@ pub async fn render_video_to_channel_nv12(
break;
}
}
- Ok(Some(_)) => {
+ Ok(Some(frame)) => {
+ in_flight.rendered(current_frame_number, Some(frame.frame_number))?;
tracing::warn!(
- frame_number = current_frame_number,
+ frame_number = frame.frame_number,
"Rendered NV12 frame has zero dimensions"
);
if let Some(ref last_frame) = last_successful_frame {
- let mut fallback = last_frame.clone_metadata_with_data();
- fallback.frame_number = current_frame_number;
- fallback.target_time_ns =
- (current_frame_number as u64 * 1_000_000_000) / fps as u64;
+ let fallback = last_frame.held_as(frame.frame_number, fps);
sender.send((fallback, current_frame_number)).await?;
channel_frames_sent += 1;
if stop_after_frames_sent.is_some_and(|m| channel_frames_sent >= m) {
@@ -1711,26 +1805,38 @@ pub async fn render_video_to_channel_nv12(
}
}
}
- Ok(None) => {}
+ Ok(None) => in_flight.rendered(current_frame_number, None)?,
Err(e) => {
tracing::error!(
frame_number = current_frame_number,
error = %e,
"NV12 frame rendering failed"
);
+ if last_successful_frame.is_none() {
+ return Err(e);
+ }
+ // A substitute must not overtake the frame still in the GPU.
+ if let Some(frame) =
+ in_flight.drain(frame_renderer.flush_pipeline_nv12().await)?
+ && frame.width > 0
+ && frame.height > 0
+ {
+ last_successful_frame = Some(frame.clone_metadata_with_data());
+ sender.send((frame, current_frame_number)).await?;
+ channel_frames_sent += 1;
+ if stop_after_frames_sent.is_some_and(|m| channel_frames_sent >= m) {
+ stopped_after_frame_limit = true;
+ break;
+ }
+ }
if let Some(ref last_frame) = last_successful_frame {
- let mut fallback = last_frame.clone_metadata_with_data();
- fallback.frame_number = current_frame_number;
- fallback.target_time_ns =
- (current_frame_number as u64 * 1_000_000_000) / fps as u64;
+ let fallback = last_frame.held_as(current_frame_number, fps);
sender.send((fallback, current_frame_number)).await?;
channel_frames_sent += 1;
if stop_after_frames_sent.is_some_and(|m| channel_frames_sent >= m) {
stopped_after_frame_limit = true;
break;
}
- } else {
- return Err(e);
}
}
}
@@ -1764,7 +1870,7 @@ pub async fn render_video_to_channel_nv12(
});
}
- if let Some(ref last_frame) = last_successful_frame {
+ if last_successful_frame.is_some() {
tracing::warn!(
frame_number = current_frame_number,
segment_time = segment_time,
@@ -1776,15 +1882,27 @@ pub async fn render_video_to_channel_nv12(
},
"Frame decode failed after retries - using previous NV12 frame"
);
- let mut fallback = last_frame.clone_metadata_with_data();
- fallback.frame_number = current_frame_number;
- fallback.target_time_ns =
- (current_frame_number as u64 * 1_000_000_000) / fps as u64;
- sender.send((fallback, current_frame_number)).await?;
- channel_frames_sent += 1;
- if stop_after_frames_sent.is_some_and(|m| channel_frames_sent >= m) {
- stopped_after_frame_limit = true;
- break;
+ // A substitute must not overtake the frame still in the GPU.
+ if let Some(frame) = in_flight.drain(frame_renderer.flush_pipeline_nv12().await)?
+ && frame.width > 0
+ && frame.height > 0
+ {
+ last_successful_frame = Some(frame.clone_metadata_with_data());
+ sender.send((frame, current_frame_number)).await?;
+ channel_frames_sent += 1;
+ if stop_after_frames_sent.is_some_and(|m| channel_frames_sent >= m) {
+ stopped_after_frame_limit = true;
+ break;
+ }
+ }
+ if let Some(ref last_frame) = last_successful_frame {
+ let fallback = last_frame.held_as(current_frame_number, fps);
+ sender.send((fallback, current_frame_number)).await?;
+ channel_frames_sent += 1;
+ if stop_after_frames_sent.is_some_and(|m| channel_frames_sent >= m) {
+ stopped_after_frame_limit = true;
+ break;
+ }
}
} else {
tracing::error!(
@@ -1803,7 +1921,7 @@ pub async fn render_video_to_channel_nv12(
}
if !stopped_after_frame_limit
- && let Some(final_frame) = frame_renderer.flush_pipeline_nv12().await.transpose()?
+ && let Some(final_frame) = in_flight.drain(frame_renderer.flush_pipeline_nv12().await)?
&& final_frame.width > 0
&& final_frame.height > 0
{
@@ -1825,6 +1943,101 @@ pub async fn render_video_to_channel_nv12(
Ok(())
}
+/// The next frame's decode, started while the current frame renders: frame
+/// number, segment time, clip index and the decode result.
+type PrefetchedDecode = (
+ u32,
+ f64,
+ usize,
+ Result<Option<DecodedSegmentFrames>, RenderingError>,
+);
+
+/// A frame the offline export loops send to the encoder.
+trait ExportFrame: Sized {
+ fn frame_number(&self) -> u32;
+ /// A copy of this picture shown at `frame_number`, for a frame that has
+ /// no picture of its own.
+ fn held_as(&self, frame_number: u32, fps: u32) -> Self;
+}
+
+impl ExportFrame for RenderedFrame {
+ fn frame_number(&self) -> u32 {
+ self.frame_number
+ }
+
+ fn held_as(&self, frame_number: u32, fps: u32) -> Self {
+ let mut held = self.clone();
+ held.frame_number = frame_number;
+ held.target_time_ns = (frame_number as u64 * 1_000_000_000) / fps as u64;
+ held
+ }
+}
+
+impl ExportFrame for Nv12RenderedFrame {
+ fn frame_number(&self) -> u32 {
+ self.frame_number
+ }
+
+ fn held_as(&self, frame_number: u32, fps: u32) -> Self {
+ let mut held = self.clone_metadata_with_data();
+ held.frame_number = frame_number;
+ held.target_time_ns = (frame_number as u64 * 1_000_000_000) / fps as u64;
+ held
+ }
+}
+
+/// The export renderer is pipelined one frame deep: rendering frame N submits
+/// it and returns frame N-1. This tracks the frame the GPU still holds, so a
+/// substitute frame is never sent ahead of it and a frame the renderer dropped
+/// (a session reset after a readback failure) stops the export instead of
+/// leaving a gap or shifting every later timestamp.
+#[derive(Debug, Default)]
+struct InFlightFrame(Option<u32>);
+
+impl InFlightFrame {
+ /// `submitted` was rendered and the renderer returned `returned`.
+ fn rendered(&mut self, submitted: u32, returned: Option<u32>) -> Result<(), RenderingError> {
+ let expected = self.0.replace(submitted);
+ Self::check(expected, returned)
+ }
+
+ /// Takes the in-flight frame out of the renderer (`flush_pipeline*`).
+ fn drain<F: ExportFrame>(
+ &mut self,
+ flushed: Option<Result<F, RenderingError>>,
+ ) -> Result<Option<F>, RenderingError> {
+ let expected = self.0.take();
+ let frame = match flushed.transpose() {
+ Ok(frame) => frame,
+ Err(error) => {
+ if let Some(frame_number) = expected {
+ tracing::error!(frame_number, %error, "In-flight export frame readback failed");
+ return Err(RenderingError::ExportFrameLost { frame_number });
+ }
+ return Err(error);
+ }
+ };
+ Self::check(expected, frame.as_ref().map(ExportFrame::frame_number))?;
+ Ok(frame)
+ }
+
+ fn check(expected: Option<u32>, returned: Option<u32>) -> Result<(), RenderingError> {
+ match (expected, returned) {
+ (Some(expected), Some(returned)) if expected == returned => Ok(()),
+ (None, None) => Ok(()),
+ (expected, returned) => {
+ let frame_number = expected.or(returned).unwrap_or_default();
+ tracing::error!(
+ ?expected,
+ ?returned,
+ "Export renderer lost a frame; stopping instead of writing it out of order"
+ );
+ Err(RenderingError::ExportFrameLost { frame_number })
+ }
+ }
+ }
+}
+
struct TransitionExportContext<'a> {
constants: &'a RenderVideoConstants,
project: &'a ProjectConfiguration,
@@ -1845,7 +2058,7 @@ async fn decode_timeline_source_frames(
current_frame_number: u32,
is_initial_frame: bool,
fps: u32,
-) -> Option<DecodedSegmentFrames> {
+) -> Result<Option<DecodedSegmentFrames>, RenderingError> {
let render_segment = &render_segments[source.segment.recording_clip as usize];
let clip_config = project
.clips
@@ -2009,11 +2222,16 @@ async fn recover_initial_frames_with_backtrack(
offsets: cap_project::ClipOffsets,
current_frame_number: u32,
fps: u32,
-) -> Option<DecodedSegmentFrames> {
+) -> Result<Option<DecodedSegmentFrames>, RenderingError> {
+ let stalled = |stall: decoder::DecodeStalled| RenderingError::FrameDecodeStalled {
+ frame_number: current_frame_number,
+ waited_secs: stall.waited.as_secs(),
+ };
for recovery_time in initial_decode_recovery_times(segment_time as f32, fps) {
let Some(_) = decoders
- .get_frames(recovery_time, needs_camera, needs_display, offsets)
+ .get_frames_for_export(recovery_time, needs_camera, needs_display, offsets, false)
.await
+ .map_err(stalled)?
else {
continue;
};
@@ -2027,16 +2245,26 @@ async fn recover_initial_frames_with_backtrack(
);
if let Some(recovered) = decoders
- .get_frames(segment_time as f32, needs_camera, needs_display, offsets)
+ .get_frames_for_export(
+ segment_time as f32,
+ needs_camera,
+ needs_display,
+ offsets,
+ false,
+ )
.await
+ .map_err(stalled)?
{
- return Some(recovered);
+ return Ok(Some(recovered));
}
}
- None
+ Ok(None)
}
+/// Offline export decode. A decode that is merely slow is waited for (see
+/// `EXPORT_STALL_LIMIT`); retries and `Ok(None)` are only for requests the
+/// decoder answered without a frame.
#[allow(clippy::too_many_arguments)]
async fn decode_segment_frames_with_retry(
decoders: &RecordingSegmentDecoders,
@@ -2047,7 +2275,7 @@ async fn decode_segment_frames_with_retry(
current_frame_number: u32,
is_initial_frame: bool,
fps: u32,
-) -> Option<DecodedSegmentFrames> {
+) -> Result<Option<DecodedSegmentFrames>, RenderingError> {
let mut result = None;
let mut retry_count = 0u32;
let max_retries = if is_initial_frame {
@@ -2066,15 +2294,19 @@ async fn decode_segment_frames_with_retry(
tokio::time::sleep(std::time::Duration::from_millis(delay)).await;
}
- result = if is_initial_frame {
- decoders
- .get_frames_initial(segment_time as f32, needs_camera, needs_display, offsets)
- .await
- } else {
- decoders
- .get_frames(segment_time as f32, needs_camera, needs_display, offsets)
- .await
- };
+ result = decoders
+ .get_frames_for_export(
+ segment_time as f32,
+ needs_camera,
+ needs_display,
+ offsets,
+ is_initial_frame,
+ )
+ .await
+ .map_err(|stall| RenderingError::FrameDecodeStalled {
+ frame_number: current_frame_number,
+ waited_secs: stall.waited.as_secs(),
+ })?;
if result.is_none() {
retry_count += 1;
@@ -2103,7 +2335,7 @@ async fn decode_segment_frames_with_retry(
.await;
}
- result
+ Ok(result)
}
pub fn get_duration(
@@ -5962,6 +6194,259 @@ mod style_image_tests {
}
}
+/// Offline export under load: a decode or GPU readback that misses its
+/// playback deadline must not change which frames are written or their order.
+#[cfg(test)]
+mod export_order_tests {
+ use super::*;
+ use std::time::Duration;
+
+ const FPS: u32 = 60;
+ const SIZE: u32 = 16;
+ const FRAMES: u32 = 45;
+
+ /// `software_path` renders through the software adapter path (RGBA
+ /// readback, CPU NV12) that Windows WARP and Linux lavapipe use.
+ async fn constants(software_path: bool) -> Option<RenderVideoConstants> {
+ let instance = create_wgpu_instance_sync();
+ let adapter = instance
+ .request_adapter(&wgpu::RequestAdapterOptions::default())
+ .await
+ .ok()?;
+ let is_software_adapter = adapter.get_info().device_type == wgpu::DeviceType::Cpu;
+ if is_software_adapter {
+ frame_pipeline::note_software_adapter_in_use();
+ }
+ let (device, queue) = adapter
+ .request_device(&wgpu::DeviceDescriptor::default())
+ .await
+ .expect("graphics device");
+ let recording_meta: RecordingMeta = serde_json::from_value(serde_json::json!({
+ "pretty_name": "export-order",
+ "display": { "path": "display.mp4", "fps": FPS },
+ "camera": null,
+ "audio": null,
+ "cursor": null
+ }))
+ .expect("recording metadata");
+ Some(RenderVideoConstants {
+ adapter_name: adapter.get_info().name,
+ _instance: instance,
+ _adapter: adapter,
+ device,
+ queue,
+ options: RenderOptions {
+ screen_size: XY::new(SIZE, SIZE),
+ camera_size: None,
+ preserve_screen_alpha: false,
+ },
+ meta: recording_meta
+ .studio_meta()
+ .expect("studio metadata")
+ .clone(),
+ recording_meta,
+ frozen_recorded_cursors: None,
+ background_textures: Arc::new(BackgroundTextureCache::default()),
+ is_software_adapter: is_software_adapter || software_path,
+ })
+ }
+
+ /// Answers every request with that source frame. Requests for
+ /// `slow_frame` are answered only after `delay`, like a decoder on a
+ /// starved machine.
+ fn scripted_decoder(slow_frame: Option<(u32, Duration)>) -> AsyncVideoDecoderHandle {
+ let (sender, receiver) = std::sync::mpsc::channel::<decoder::VideoDecoderMessage>();
+ std::thread::spawn(move || {
+ while let Ok(decoder::VideoDecoderMessage::GetFrame(time, _, reply)) = receiver.recv() {
+ let frame = (time * FPS as f32).round() as u32;
+ if let Some((slow, delay)) = slow_frame
+ && slow == frame
+ {
+ std::thread::sleep(delay);
+ }
+ let level = (frame * 5 % 256) as u8;
+ let _ = reply.send(DecodedFrame::new(
+ vec![level; (SIZE * SIZE * 4) as usize],
+ SIZE,
+ SIZE,
+ ));
+ }
+ });
+ AsyncVideoDecoderHandle::for_test(sender, SIZE, SIZE)
+ }
+
+ /// Exports `FRAMES` frames and returns the encoder-side frame order.
+ async fn export(
+ constants: &RenderVideoConstants,
+ screen: AsyncVideoDecoderHandle,
+ ) -> (Result<(), RenderingError>, Vec<u32>) {
+ let project = ProjectConfiguration {
+ timeline: Some(
+ serde_json::from_value(serde_json::json!({
+ "segments": [{
+ "recordingSegment": 0,
+ "timescale": 1.0,
+ "start": 0.0,
+ "end": FRAMES as f64 / FPS as f64
+ }],
+ "zoomSegments": []
+ }))
+ .expect("timeline"),
+ ),
+ ..Default::default()
+ };
+ let segments = vec![RenderSegment {
+ cursor: Arc::new(CursorEvents {
+ clicks: vec![],
+ moves: vec![],
+ }),
+ keyboard: Arc::new(Default::default()),
+ decoders: RecordingSegmentDecoders {
+ screen,
+ camera: None,
+ segment_offset: 0.0,
+ },
+ render_display: true,
+ }];
+ let recordings = ProjectRecordingsMeta { segments: vec![] };
+ let (sender, mut receiver) = mpsc::channel(8);
+ let render = render_video_to_channel_nv12(
+ constants,
+ &project,
+ sender,
+ &constants.recording_meta,
+ &constants.meta,
+ segments,
+ FPS,
+ XY::new(SIZE, SIZE),
+ &recordings,
+ None,
+ None,
+ None,
+ );
+ let collect = async {
+ let mut order = Vec::new();
+ while let Some((frame, _)) = receiver.recv().await {
+ order.push(frame.frame_number);
+ }
+ order
+ };
+ tokio::join!(render, collect)
+ }
+
+ /// Runs an export on its own thread: the unoptimized export future needs
+ /// more stack than a test thread has, and the readback delay seam is per
+ /// thread.
+ fn run_export(
+ software_path: bool,
+ slow_decode: Option<(u32, Duration)>,
+ slow_readback: Option<(u32, Duration)>,
+ ) -> Option<(Result<(), RenderingError>, Vec<u32>)> {
+ std::thread::Builder::new()
+ .stack_size(64 * 1024 * 1024)
+ .spawn(move || {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .expect("runtime")
+ .block_on(async move {
+ let Some(constants) = constants(software_path).await else {
+ eprintln!("No graphics adapter available for export order test");
+ return None;
+ };
+ if let Some((frame_number, delay)) = slow_readback {
+ frame_pipeline::readback_delay::inject(frame_number, delay);
+ }
+ Some(export(&constants, scripted_decoder(slow_decode)).await)
+ })
+ })
+ .expect("export thread")
+ .join()
+ .expect("export thread panicked")
+ }
+
+ #[test]
+ fn slow_decode_keeps_every_frame_in_order() {
+ let decode_deadline = Duration::from_secs(2);
+ for software_path in [false, true] {
+ let Some((result, order)) = run_export(
+ software_path,
+ Some((34, decode_deadline + Duration::from_millis(250))),
+ None,
+ ) else {
+ return;
+ };
+
+ result.expect("a slow decode is waited for, not substituted");
+ assert_eq!(
+ order,
+ (0..FRAMES).collect::<Vec<_>>(),
+ "software path: {software_path}"
+ );
+ }
+ }
+
+ #[test]
+ fn slow_readback_keeps_every_frame_in_order() {
+ for software_path in [false, true] {
+ let Some((result, order)) =
+ run_export(software_path, None, Some((20, Duration::from_secs(61))))
+ else {
+ return;
+ };
+
+ result.expect("a slow readback is waited for, not dropped");
+ assert_eq!(
+ order,
+ (0..FRAMES).collect::<Vec<_>>(),
+ "software path: {software_path}"
+ );
+ }
+ }
+
+ #[test]
+ fn stalled_readback_stops_the_export_instead_of_dropping_the_frame() {
+ for software_path in [false, true] {
+ let Some((result, order)) =
+ run_export(software_path, None, Some((20, Duration::from_secs(3600))))
+ else {
+ return;
+ };
+
+ assert!(
+ matches!(
+ result,
+ Err(RenderingError::ExportFrameLost { frame_number: 20 })
+ ),
+ "software path: {software_path}: {result:?}"
+ );
+ assert_eq!(
+ order,
+ (0..20).collect::<Vec<_>>(),
+ "software path: {software_path}"
+ );
+ }
+ }
+
+ #[test]
+ fn in_flight_frame_must_come_out_next() {
+ let mut in_flight = InFlightFrame::default();
+ in_flight.rendered(0, None).unwrap();
+ in_flight.rendered(1, Some(0)).unwrap();
+ assert!(matches!(
+ in_flight.rendered(2, None),
+ Err(RenderingError::ExportFrameLost { frame_number: 1 })
+ ));
+
+ let mut in_flight = InFlightFrame::default();
+ in_flight.rendered(5, None).unwrap();
+ assert!(matches!(
+ in_flight.rendered(6, Some(4)),
+ Err(RenderingError::ExportFrameLost { frame_number: 5 })
+ ));
+ }
+}
+
#[cfg(test)]
mod nv12_flush_tests {
use super::*;