Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(export): restart an export on libx264 when the hardware encoder fails mid-stream

Richie McIlroy8c1fca234959
diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs
index 0042b4610..6b2e6d41d 100644
--- a/crates/enc-ffmpeg/src/video/h264.rs
+++ b/crates/enc-ffmpeg/src/video/h264.rs
@@ -879,6 +879,14 @@ impl H264Encoder {
.map_err(|e| H264EncoderError::VideoToolboxHwInput(e.to_string()))
}
+ /// Whether the selected codec is a hardware encoder, which can fail at
+ /// runtime after opening (a busy or reset media engine).
+ pub fn is_hardware(&self) -> bool {
+ self.encoder
+ .codec()
+ .is_some_and(|codec| is_hardware_h264(codec.name()))
+ }
+
pub fn conversion_requirements(&self) -> ConversionRequirements {
let needs_conversion = self.input_format != self.output_format
|| self.input_width != self.output_width
diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml
index b1cae02e4..9a214783f 100644
--- a/crates/export/Cargo.toml
+++ b/crates/export/Cargo.toml
@@ -3,6 +3,10 @@ name = "cap-export"
version = "0.1.0"
edition = "2024"
+[features]
+# Encoder failure injection for the cap-verify suite. Never enabled by the app.
+test-hooks = []
+
[dependencies]
cap-utils = { path = "../utils" }
cap-project = { path = "../project" }
diff --git a/crates/export/src/estimates.rs b/crates/export/src/estimates.rs
index 99bd390dd..3d245e931 100644
--- a/crates/export/src/estimates.rs
+++ b/crates/export/src/estimates.rs
@@ -289,6 +289,8 @@ pub async fn estimate_export(
audio_cancellation: None,
sample_windows: Some(windows.clone()),
sample_timing: Some(timing.clone()),
+ #[cfg(feature = "test-hooks")]
+ video_encoder_failure_at: None,
};
let progress_editor = editor.clone();
let progress_cancel = cancel.clone();
diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs
index 86b5514ed..185d9d561 100644
--- a/crates/export/src/lib.rs
+++ b/crates/export/src/lib.rs
@@ -254,6 +254,8 @@ impl ExporterBuilder {
audio_cancellation,
sample_windows: None,
sample_timing: None,
+ #[cfg(feature = "test-hooks")]
+ video_encoder_failure_at: None,
})
}
}
@@ -440,6 +442,11 @@ impl Mp4ExporterBase {
pub fn uses_streaming_audio(&self) -> bool {
self.0.streaming_audio.is_some()
}
+
+ #[cfg(feature = "test-hooks")]
+ pub fn fail_video_encoder_at(&mut self, frames: u32) {
+ self.0.fail_video_encoder_at(frames);
+ }
}
pub struct ExporterBase {
@@ -456,9 +463,49 @@ pub struct ExporterBase {
audio_cancellation: Option<ExportAudioCancellation>,
sample_windows: Option<cap_rendering::FrameWindows>,
sample_timing: Option<Arc<estimates::SampleTiming>>,
+ #[cfg(feature = "test-hooks")]
+ video_encoder_failure_at: Option<u32>,
}
impl ExporterBase {
+ /// Makes the MP4 export's video encoder fail after `frames` frames the
+ /// way a hardware encoder does at runtime, once, so the software
+ /// encoder fallback runs on every machine.
+ #[cfg(feature = "test-hooks")]
+ pub fn fail_video_encoder_at(&mut self, frames: u32) {
+ self.video_encoder_failure_at = Some(frames);
+ }
+
+ /// Reopens what an MP4 export consumes (the staged output and the
+ /// streaming audio with its cancellation) so the export can start over.
+ async fn reopen_streaming(
+ &mut self,
+ user_cancellation: Option<Arc<AtomicBool>>,
+ ) -> Result<(), String> {
+ let Some(user) = user_cancellation else {
+ return Ok(());
+ };
+ let control = ExportAudioCancellation::new(user);
+ let recording = self.recording_meta.clone();
+ let studio = self.studio_meta.clone();
+ let cancellation = control.user.clone();
+ let abort = control.stop.clone();
+ let preparation = tokio::task::spawn_blocking(move || {
+ ExportAudioPreparation::open(&recording, &studio, cancellation, abort)
+ })
+ .await
+ .map_err(|error| error.to_string())?
+ .map_err(|error| error.to_string())?;
+ self.streaming_audio = Some(
+ preparation
+ .finish(&self.segments)
+ .map_err(|error| error.to_string())?,
+ );
+ self.streaming_output = prepare_streaming_output(&self.output_path, true);
+ self.audio_cancellation = Some(control);
+ Ok(())
+ }
+
pub fn total_frames(&self, fps: u32) -> u32 {
let duration = cap_rendering::get_duration(
&self.recordings,
diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs
index 8395ed740..cb893ab2b 100644
--- a/crates/export/src/mp4.rs
+++ b/crates/export/src/mp4.rs
@@ -16,7 +16,7 @@ use specta::Type;
use std::{
path::PathBuf,
sync::{
- Arc, Mutex,
+ Arc, Mutex, PoisonError,
atomic::{AtomicBool, AtomicU64, Ordering},
},
time::Duration,
@@ -169,6 +169,7 @@ impl ExportCompression {
#[derive(Clone, Default)]
struct ExportNv12Mode {
stop_after_frames_sent: Option<u32>,
+ software_encoder: bool,
record_first_queued_ms_since_pipeline: Option<Arc<AtomicU64>>,
nv12_render_startup_breakdown_ms:
Option<Arc<Mutex<Option<cap_rendering::Nv12RenderStartupBreakdownMs>>>>,
@@ -294,6 +295,7 @@ impl Mp4ExportSettings {
|_| true,
ExportNv12Mode {
stop_after_frames_sent: Some(1),
+ software_encoder: false,
record_first_queued_ms_since_pipeline: Some(first_ms_enc),
nv12_render_startup_breakdown_ms: Some(breakdown_enc),
},
@@ -326,7 +328,62 @@ impl Mp4ExportSettings {
output_size: (u32, u32),
fps: u32,
on_progress: impl FnMut(u32) -> bool + Send + 'static,
+ mut mode: ExportNv12Mode,
+ ) -> Result<PathBuf, String> {
+ let on_progress = Arc::new(Mutex::new(on_progress));
+ let user_cancellation = base
+ .audio_cancellation
+ .as_ref()
+ .map(|control| control.user.clone());
+ loop {
+ let progress = on_progress.clone();
+ let hardware_encoder_failed = Arc::new(AtomicBool::new(false));
+ let result = self
+ .export_nv12_attempt(
+ &mut base,
+ output_size,
+ fps,
+ move |frame| {
+ let mut on_progress =
+ progress.lock().unwrap_or_else(PoisonError::into_inner);
+ (*on_progress)(frame)
+ },
+ mode.clone(),
+ hardware_encoder_failed.clone(),
+ )
+ .await;
+ // A hardware encoder can fail mid-stream when the media engine is
+ // busy or resets. The MP4 cannot switch codecs partway, so the
+ // export starts over once with the software encoder.
+ match result {
+ Err(error)
+ if !mode.software_encoder
+ && base.sample_timing.is_none()
+ && hardware_encoder_failed.load(Ordering::Relaxed)
+ && !user_cancellation
+ .as_ref()
+ .is_some_and(|cancel| cancel.load(Ordering::Relaxed)) =>
+ {
+ warn!(
+ %error,
+ "Hardware video encoder failed during export, restarting with the software encoder"
+ );
+ base.reopen_streaming(user_cancellation.clone()).await?;
+ mode.software_encoder = true;
+ }
+ result => return result,
+ }
+ }
+ }
+
+ async fn export_nv12_attempt(
+ self,
+ base: &mut ExporterBase,
+ output_size: (u32, u32),
+ fps: u32,
+ on_progress: impl FnMut(u32) -> bool + Send + 'static,
mode: ExportNv12Mode,
+ hardware_encoder_failed: Arc<AtomicBool>,
) -> Result<PathBuf, String> {
let pipeline_start = std::time::Instant::now();
let sample_timing = base.sample_timing.clone();
@@ -379,6 +436,9 @@ impl Mp4ExportSettings {
let has_audio = has_recording_audio || !music.is_empty();
let record_first_queued_ms = mode.record_first_queued_ms_since_pipeline;
+ let software_encoder = mode.software_encoder;
+ #[cfg(feature = "test-hooks")]
+ let injected_failure_at = base.video_encoder_failure_at.filter(|_| !software_encoder);
let nv12_render_startup_breakdown_ms = mode.nv12_render_startup_breakdown_ms;
let project_for_audio = base.project_config.clone();
@@ -401,6 +461,7 @@ impl Mp4ExportSettings {
}
#[cfg(target_os = "macos")]
let want_hw_input = !self.optimize_filesize
+ && !software_encoder
&& matches!(
first_frame.as_ref().map(|frame| &frame.payload),
Some(ExportFramePayload::Surface(_))
@@ -442,6 +503,8 @@ impl Mp4ExportSettings {
.with_external_conversion();
let builder = if self.optimize_filesize {
builder.with_crf(self.compression.crf_value())
+ } else if software_encoder {
+ builder.with_encoder_priority_override(&["libx264"])
} else {
builder
};
@@ -465,8 +528,17 @@ impl Mp4ExportSettings {
encoder.video_mut().set_packet_stats(timing.packet_stats());
}
+ let hardware_video = encoder.video().is_hardware();
+ let video_failure = |error: String| {
+ if hardware_video {
+ hardware_encoder_failed.store(true, Ordering::Relaxed);
+ }
+ Mp4PipelineError::Failure(error)
+ };
+
info!(
zero_copy_input = encoder_is_hw,
+ hardware_video,
"Created MP4File encoder (NV12, export settings)"
);
@@ -535,7 +607,7 @@ impl Mp4ExportSettings {
&mut converted_frame,
Duration::MAX,
)
- .map_err(|err| err.to_string())?;
+ .map_err(|err| video_failure(err.to_string()))?;
}
#[cfg(target_os = "macos")]
ExportFramePayload::Surface(surface) => {
@@ -545,7 +617,7 @@ impl Mp4ExportSettings {
.video_encoder()
.wrap_videotoolbox_pixel_buffer(surface.as_pixel_buffer_ptr())
}
- .map_err(|err| err.to_string())?;
+ .map_err(|err| video_failure(err.to_string()))?;
frame.set_pts(Some(input.frame_number as i64));
encoder
.queue_video_frame_reusable(
@@ -553,7 +625,7 @@ impl Mp4ExportSettings {
&mut converted_frame,
Duration::MAX,
)
- .map_err(|err| err.to_string())?;
+ .map_err(|err| video_failure(err.to_string()))?;
} else {
fill_nv12_frame_from_surface(
&mut reusable_frame,
@@ -571,7 +643,7 @@ impl Mp4ExportSettings {
&mut converted_frame,
Duration::MAX,
)
- .map_err(|err| err.to_string())?;
+ .map_err(|err| video_failure(err.to_string()))?;
}
}
}
@@ -579,6 +651,13 @@ impl Mp4ExportSettings {
worker.write_ready(&mut encoder)?;
}
encoded_frames += 1;
+ #[cfg(feature = "test-hooks")]
+ if injected_failure_at == Some(encoded_frames) {
+ hardware_encoder_failed.store(true, Ordering::Relaxed);
+ return Err(Mp4PipelineError::Failure(
+ "Injected hardware encoder failure".to_string(),
+ ));
+ }
if let Some(timing) = &sample_timing {
timing.record_frame(input.timeline_frame);
}
@@ -617,7 +696,7 @@ impl Mp4ExportSettings {
.map_err(|e| format!("Failed to finish encoding: {e}"))?;
if let Err(e) = res.video_finish {
- return Err(Mp4PipelineError::Failure(format!("Video encoding failed: {e}")));
+ return Err(video_failure(format!("Video encoding failed: {e}")));
}
if let Err(e) = res.audio_finish {
return Err(Mp4PipelineError::Failure(format!("Audio encoding failed: {e}")));