Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
use crate::{ExporterBase, Mp4ExporterBase};
use cap_editor::{
AudioRenderer, ExportAudioError, ExportAudioRenderer, get_audio_segments,
load_music_tracks_uncached,
};
use cap_enc_ffmpeg::{AudioEncoder, PacketQueue, aac::AACEncoder, h264::H264Encoder, mp4::*};
use cap_media_info::{RawVideoFormat, VideoInfo};
use cap_project::XY;
use cap_rendering::{
GpuOutputFormat, Nv12RenderedFrame, ProjectUniforms, RenderSegment, SharedNv12Buffer,
};
use futures::FutureExt;
use image::ImageBuffer;
use serde::{Deserialize, Serialize};
use specta::Type;
use std::{
path::PathBuf,
sync::{
Arc, Mutex, PoisonError,
atomic::{AtomicBool, AtomicU64, Ordering},
},
time::Duration,
};
use tokio::sync::Notify;
use tracing::{info, trace, warn};
pub mod turbo;
enum Mp4PipelineError {
Interrupted,
Failure(String),
AudioSourceFailure {
source_index: usize,
message: String,
},
}
impl From<String> for Mp4PipelineError {
fn from(error: String) -> Self {
Self::Failure(error)
}
}
impl From<ExportAudioError> for Mp4PipelineError {
fn from(error: ExportAudioError) -> Self {
match (&error, error.source_index()) {
(ExportAudioError::Cancelled, _) => Self::Interrupted,
(ExportAudioError::Source { source, .. }, _) if source.is_cancelled() => {
Self::Interrupted
}
(_, Some(source_index)) => Self::AudioSourceFailure {
source_index,
message: error.to_string(),
},
_ => Self::Failure(error.to_string()),
}
}
}
impl Mp4PipelineError {
fn message(self) -> String {
match self {
Self::Interrupted => "Export cancelled".to_string(),
Self::Failure(error) | Self::AudioSourceFailure { message: error, .. } => error,
}
}
}
fn resolve_pipeline_results(
encoder: Result<PathBuf, Mp4PipelineError>,
renderer: Result<(), Mp4PipelineError>,
validation: Result<(), Mp4PipelineError>,
) -> Result<(), String> {
let mut first_source_failure: Option<(usize, String)> = None;
let mut first_other_failure = None;
let mut interrupted = false;
for error in [encoder.err(), renderer.err(), validation.err()]
.into_iter()
.flatten()
{
match error {
Mp4PipelineError::AudioSourceFailure {
source_index,
message,
} => {
if first_source_failure
.as_ref()
.is_none_or(|(previous, _)| source_index < *previous)
{
first_source_failure = Some((source_index, message));
}
}
Mp4PipelineError::Failure(error) => {
if first_other_failure.is_none() {
first_other_failure = Some(error);
}
}
Mp4PipelineError::Interrupted => interrupted = true,
}
}
if let Some((_, error)) = first_source_failure {
Err(error)
} else if let Some(error) = first_other_failure {
Err(error)
} else if interrupted {
Err("Export cancelled".to_string())
} else {
Ok(())
}
}
async fn complete_audio_validation(
task: tokio::task::JoinHandle<Result<(), Mp4PipelineError>>,
abort: Arc<AtomicBool>,
failure: Arc<Notify>,
) -> Result<(), Mp4PipelineError> {
match task.await {
Ok(result) => result,
Err(error) => {
abort.store(true, Ordering::Relaxed);
failure.notify_one();
Err(Mp4PipelineError::Failure(format!(
"Audio validation worker failed: {error}"
)))
}
}
}
async fn render_until_audio_failure(
render: impl std::future::Future<Output = Result<(), Mp4PipelineError>>,
failure: Option<Arc<Notify>>,
) -> Result<(), Mp4PipelineError> {
let Some(failure) = failure else {
return render.await;
};
tokio::select! {
biased;
_ = failure.notified() => Err(Mp4PipelineError::Interrupted),
result = render => result,
}
}
#[derive(Serialize, Deserialize, Type, Clone, Copy, Debug)]
pub enum ExportCompression {
Maximum,
Social,
Web,
Potato,
}
impl ExportCompression {
pub fn bits_per_pixel(&self) -> f32 {
match self {
Self::Maximum => 0.3,
Self::Social => 0.15,
Self::Web => 0.08,
Self::Potato => 0.04,
}
}
pub fn crf_value(&self) -> u8 {
match self {
Self::Maximum => 24,
Self::Social => 28,
Self::Web => 32,
Self::Potato => 36,
}
}
}
#[derive(Clone, Default)]
struct ExportNv12Mode {
stop_after_frames_sent: Option<u32>,
software_encoder: bool,
record_first_queued_ms_since_pipeline: Option<Arc<AtomicU64>>,
nv12_render_startup_breakdown_ms:
Option<Arc<Mutex<Option<cap_rendering::Nv12RenderStartupBreakdownMs>>>>,
}
#[derive(Debug, serde::Serialize)]
pub struct FirstFrameQueuedBenchmark {
pub ms_to_first_frame_queued_since_export_pipeline_start: u64,
pub nv12_render_startup_breakdown_ms: Option<cap_rendering::Nv12RenderStartupBreakdownMs>,
}
#[derive(Serialize, Deserialize, Type, Clone, Copy, Debug)]
pub struct Mp4ExportSettings {
pub fps: u32,
pub resolution_base: XY<u32>,
pub compression: ExportCompression,
pub custom_bpp: Option<f32>,
#[serde(default)]
pub force_ffmpeg_decoder: bool,
#[serde(default)]
pub optimize_filesize: bool,
}
impl Mp4ExportSettings {
pub fn effective_bpp(&self) -> f32 {
self.custom_bpp
.unwrap_or_else(|| self.compression.bits_per_pixel())
}
}
impl Mp4ExportSettings {
pub fn export_prepared(
self,
base: Mp4ExporterBase,
on_progress: impl FnMut(u32) -> bool + Send + 'static,
) -> impl std::future::Future<Output = Result<PathBuf, String>> {
self.export(base.0, on_progress)
}
/// Exports with every core: see [`turbo`]. Falls back to the standard
/// export when the turbo pipeline fails, unless it was cancelled or its
/// destination appeared while it ran.
pub async fn export_prepared_turbo(
self,
base: Mp4ExporterBase,
options: turbo::TurboOptions,
on_progress: impl FnMut(u32) -> bool + Send + 'static,
) -> Result<PathBuf, String> {
let mut base = base.0;
let on_progress = Arc::new(Mutex::new(on_progress));
let user_cancellation = base
.audio_cancellation
.as_ref()
.map(|control| control.user.clone());
let destination_exists = |path: &std::path::Path| {
std::fs::symlink_metadata(path).map_or_else(
|error| error.kind() != std::io::ErrorKind::NotFound,
|_| true,
)
};
let destination_was_free = !destination_exists(&base.output_path);
match turbo::export(self, &mut base, options, on_progress.clone()).await {
Ok(path) => Ok(path),
Err(turbo::TurboError::Cancelled) => Err("Export cancelled".to_string()),
Err(turbo::TurboError::Publish(error)) => Err(error),
Err(turbo::TurboError::Failed(error)) if options.strict => Err(error),
Err(turbo::TurboError::Failed(error)) => {
warn!(%error, "Turbo export failed, exporting with the standard pipeline");
base.reopen_streaming(user_cancellation).await?;
// Turbo staged its output, but a standard export that is not
// staged writes in place and would truncate a file created
// at the destination meanwhile.
if destination_was_free
&& base.streaming_output.is_none()
&& destination_exists(&base.output_path)
{
return Err(error);
}
let output_size = ProjectUniforms::get_output_size(
&base.render_constants.options,
&base.project_config,
self.resolution_base,
);
self.export_nv12(
base,
output_size,
self.fps,
move |frame| {
(on_progress.lock().unwrap_or_else(PoisonError::into_inner))(frame)
},
ExportNv12Mode::default(),
)
.await
}
}
}
pub async fn export(
self,
base: ExporterBase,
on_progress: impl FnMut(u32) -> bool + Send + 'static,
) -> Result<PathBuf, String> {
use cap_utils::operation_diagnostics::{Field, observe};
observe(
"export_mp4",
&[
Field::number("requested_fps", self.fps as u64),
Field::number("requested_width", self.resolution_base.x as u64),
Field::number("requested_height", self.resolution_base.y as u64),
Field::identifier(
"resource",
cap_utils::operation_diagnostics::resource_id(&base.project_path),
),
Field::number(
"source_width",
base.render_constants.options.screen_size.x as u64,
),
Field::number(
"source_height",
base.render_constants.options.screen_size.y as u64,
),
Field::number("source_segments", base.segments.len() as u64),
Field::number("clips", base.project_config.clips.len() as u64),
Field::flag("captions", base.project_config.captions.is_some()),
Field::flag("streaming_audio", base.streaming_audio.is_some()),
],
self.export_inner(base, on_progress),
)
.await
}
async fn export_inner(
self,
base: ExporterBase,
on_progress: impl FnMut(u32) -> bool + Send + 'static,
) -> Result<PathBuf, String> {
info!("Exporting mp4 with settings: {:?}", &self);
info!("Expected to render {} frames", base.total_frames(self.fps));
let fps = self.fps;
let output_size = ProjectUniforms::get_output_size(
&base.render_constants.options,
&base.project_config,
self.resolution_base,
);
info!(
width = output_size.0,
height = output_size.1,
"Exporting with NV12 pipeline (GPU when possible, CPU fallback otherwise)"
);
self.export_nv12(
base,
output_size,
fps,
on_progress,
ExportNv12Mode::default(),
)
.await
}
pub async fn benchmark_first_frame_with_breakdown(
self,
base: ExporterBase,
) -> Result<FirstFrameQueuedBenchmark, String> {
let fps = self.fps;
let output_size = ProjectUniforms::get_output_size(
&base.render_constants.options,
&base.project_config,
self.resolution_base,
);
let first_ms = Arc::new(AtomicU64::new(u64::MAX));
let first_ms_enc = Arc::clone(&first_ms);
let breakdown = Arc::new(Mutex::new(None));
let breakdown_enc = Arc::clone(&breakdown);
self.export_nv12(
base,
output_size,
fps,
|_| true,
ExportNv12Mode {
stop_after_frames_sent: Some(1),
software_encoder: false,
record_first_queued_ms_since_pipeline: Some(first_ms_enc),
nv12_render_startup_breakdown_ms: Some(breakdown_enc),
},
)
.await?;
let v = first_ms.load(Ordering::Relaxed);
if v == u64::MAX {
return Err("first frame was not queued to the encoder".to_string());
}
let nv12_render_startup_breakdown_ms = breakdown.lock().ok().and_then(|mut g| g.take());
Ok(FirstFrameQueuedBenchmark {
ms_to_first_frame_queued_since_export_pipeline_start: v,
nv12_render_startup_breakdown_ms,
})
}
pub async fn benchmark_ms_to_first_frame_queued(
self,
base: ExporterBase,
) -> Result<u64, String> {
Ok(self
.benchmark_first_frame_with_breakdown(base)
.await?
.ms_to_first_frame_queued_since_export_pipeline_start)
}
async fn export_nv12(
self,
mut base: ExporterBase,
output_size: (u32, u32),
fps: u32,
on_progress: impl FnMut(u32) -> bool + Send + 'static,
mut mode: ExportNv12Mode,
) -> Result<PathBuf, String> {
let on_progress = Arc::new(Mutex::new(on_progress));
let user_cancellation = base
.audio_cancellation
.as_ref()
.map(|control| control.user.clone());
loop {
let progress = on_progress.clone();
let hardware_encoder_failed = Arc::new(AtomicBool::new(false));
let result = self
.export_nv12_attempt(
&mut base,
output_size,
fps,
move |frame| {
let mut on_progress =
progress.lock().unwrap_or_else(PoisonError::into_inner);
(*on_progress)(frame)
},
mode.clone(),
hardware_encoder_failed.clone(),
)
.await;
// A hardware encoder can fail mid-stream when the media engine is
// busy or resets. The MP4 cannot switch codecs partway, so the
// export starts over once with the software encoder.
match result {
Err(error)
if restart_on_software_encoder(
mode.software_encoder,
base.sample_timing.is_some(),
hardware_encoder_failed.load(Ordering::Relaxed),
user_cancellation
.as_ref()
.is_some_and(|cancel| cancel.load(Ordering::Relaxed)),
) =>
{
warn!(
%error,
"Hardware video encoder failed during export, restarting with the software encoder"
);
base.reopen_streaming(user_cancellation.clone()).await?;
mode.software_encoder = true;
}
result => return result,
}
}
}
async fn export_nv12_attempt(
self,
base: &mut ExporterBase,
output_size: (u32, u32),
fps: u32,
on_progress: impl FnMut(u32) -> bool + Send + 'static,
mode: ExportNv12Mode,
hardware_encoder_failed: Arc<AtomicBool>,
) -> Result<PathBuf, String> {
let pipeline_start = std::time::Instant::now();
let sample_timing = base.sample_timing.clone();
let output_path = base.output_path.clone();
let mut streaming_audio = base.streaming_audio.take();
let audio_control = base.audio_cancellation.take();
let audio_cancellation = audio_control.as_ref().map(|control| control.stop.clone());
let user_cancellation = audio_control.as_ref().map(|control| control.user.clone());
let temporary_output = base.streaming_output.take().map(Arc::new);
let unused_audio = streaming_audio
.as_mut()
.and_then(|audio| audio.take_unused_sources(&base.project_config));
let validation_failure = unused_audio.as_ref().map(|_| Arc::new(Notify::new()));
let audio_validation = unused_audio.map(|validation| {
let abort = audio_cancellation.as_ref().unwrap().clone();
let failure = validation_failure.as_ref().unwrap().clone();
let worker_abort = abort.clone();
let worker_failure = failure.clone();
let temporary_output_guard = temporary_output.clone();
let task = tokio::task::spawn_blocking(move || {
let _temporary_output_guard = temporary_output_guard;
let result = validation.validate_to_end().map_err(Mp4PipelineError::from);
if result.is_err() {
worker_abort.store(true, Ordering::Relaxed);
worker_failure.notify_one();
}
result
});
(task, abort, failure)
});
let encoder_output_path = temporary_output
.as_ref()
.map(|path| path.to_path_buf())
.unwrap_or_else(|| output_path.clone());
let meta = &base.studio_meta;
let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::<ExportFrame>(4);
let mut video_info =
VideoInfo::from_raw(RawVideoFormat::Nv12, output_size.0, output_size.1, fps);
video_info.time_base = ffmpeg::Rational::new(1, fps as i32);
let audio_segments = get_audio_segments(&base.segments).await;
let music = load_music_tracks_uncached(&base.project_config, &base.project_path);
let has_recording_audio = audio_segments
.first()
.filter(|_| !base.project_config.audio.mute)
.is_some();
let has_audio = has_recording_audio || !music.is_empty();
let record_first_queued_ms = mode.record_first_queued_ms_since_pipeline;
let software_encoder = mode.software_encoder;
#[cfg(feature = "test-hooks")]
let injected_failure_at = base.video_encoder_failure_at.filter(|_| !software_encoder);
let nv12_render_startup_breakdown_ms = mode.nv12_render_startup_breakdown_ms;
let project_for_audio = base.project_config.clone();
let pipeline_start_for_encoder = pipeline_start;
let encoder_temporary_output = temporary_output.clone();
let encoder_cancellation = audio_cancellation.clone();
let encoder_user_cancellation = user_cancellation.clone();
let encoder_result_cancellation = audio_cancellation.clone();
let encoder_thread = tokio::task::spawn_blocking(move || {
let _temporary_output_guard = encoder_temporary_output;
trace!("Creating MP4File encoder (NV12 path)");
// The encoder's input mode has to match what the renderer actually
// produces, so the first frame decides: IOSurface frames open
// VideoToolbox for zero-copy hardware input (falling back to
// software frames if that fails), CPU frames keep today's path.
let first_frame = frame_rx.recv().ok();
if encoder_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) || encoder_user_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) {
return Err(Mp4PipelineError::Interrupted);
}
#[cfg(target_os = "macos")]
let want_hw_input = !self.optimize_filesize
&& !software_encoder
&& matches!(
first_frame.as_ref().map(|frame| &frame.payload),
Some(ExportFramePayload::Surface(_))
);
#[cfg(target_os = "macos")]
let mut encoder_is_hw = false;
#[cfg(not(target_os = "macos"))]
let encoder_is_hw = false;
let mut detached_audio = None;
let mut encoder = MP4File::init(
"output",
encoder_output_path.clone(),
self.optimize_filesize,
|o| {
#[cfg(target_os = "macos")]
if want_hw_input {
match H264Encoder::builder(video_info)
.with_bpp(self.effective_bpp())
.with_export_settings()
.build_videotoolbox_hw_input(o)
{
Ok(encoder) => {
encoder_is_hw = true;
return Ok(encoder);
}
Err(error) => info!(
%error,
"Zero-copy VideoToolbox export input unavailable, using software frames"
),
}
}
let builder = H264Encoder::builder(video_info)
.with_bpp(self.effective_bpp())
.with_export_priority()
.with_export_settings()
.with_external_conversion();
let builder = if self.optimize_filesize {
builder.with_crf(self.compression.crf_value())
} else if software_encoder {
builder.with_encoder_priority_override(&["libx264"])
} else {
builder
};
builder.build(o)
},
|o| {
if !has_audio {
return None;
}
match AACEncoder::init_for_export(AudioRenderer::info(), o) {
Ok(audio) => {
detached_audio = Some(audio);
None
}
Err(error) => Some(Err(error.into())),
}
},
)
.map_err(|v| v.to_string())?;
if let Some(timing) = &sample_timing {
encoder.video_mut().set_packet_stats(timing.packet_stats());
}
let hardware_video = encoder.video().is_hardware();
// The injected failure stands in for a hardware encoder's.
#[cfg(feature = "test-hooks")]
let hardware_video = hardware_video || injected_failure_at.is_some();
let video_failure = |error: String| {
video_encoder_failure(hardware_video, &hardware_encoder_failed, error)
};
info!(
zero_copy_input = encoder_is_hw,
hardware_video,
"Created MP4File encoder (NV12, export settings)"
);
let mut audio_worker = match detached_audio {
Some(audio_encoder) => {
let source = match streaming_audio.take() {
Some(audio) => ExportAudioSource::Streaming(audio),
None => ExportAudioSource::Renderer(AudioRenderer::new(audio_segments).with_music(music)),
};
let worker_cancellation = encoder_cancellation.clone();
let worker_user_cancellation = encoder_user_cancellation.clone();
Some(ExportAudioWorker::spawn(
encoder.stream_time_base(audio_encoder.stream_index()),
audio_encoder,
source,
ExportAudioTimeline {
project: project_for_audio,
fps,
sampled: sample_timing.is_some(),
},
move || {
worker_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed))
|| worker_user_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed))
},
)?)
}
None => None,
};
let mut reusable_frame = ffmpeg::frame::Video::new(
ffmpeg::format::Pixel::NV12,
output_size.0,
output_size.1,
);
let mut converted_frame: Option<ffmpeg::frame::Video> = None;
let mut encoded_frames = 0u32;
let encode_start = std::time::Instant::now();
let frames = first_frame
.into_iter()
.chain(std::iter::from_fn(|| frame_rx.recv().ok()));
for input in frames {
if sample_timing.as_ref().is_some_and(|timing| timing.is_cancelled()) {
return Err(Mp4PipelineError::Interrupted);
}
if encoder_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) || encoder_user_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) {
return Err(Mp4PipelineError::Interrupted);
}
if let Some(worker) = &mut audio_worker {
worker.request(input.frame_number, input.timeline_frame);
}
match &input.payload {
ExportFramePayload::Cpu(nv12_data) => {
fill_nv12_frame_direct(
&mut reusable_frame,
nv12_data,
input.width,
input.height,
input.y_stride,
input.frame_number as i64,
);
encoder
.queue_video_frame_reusable(
&mut reusable_frame,
&mut converted_frame,
Duration::MAX,
)
.map_err(|err| video_failure(err.to_string()))?;
}
#[cfg(target_os = "macos")]
ExportFramePayload::Surface(surface) => {
if encoder_is_hw {
let mut frame = unsafe {
encoder
.video_encoder()
.wrap_videotoolbox_pixel_buffer(surface.as_pixel_buffer_ptr())
}
.map_err(|err| video_failure(err.to_string()))?;
frame.set_pts(Some(input.frame_number as i64));
encoder
.queue_video_frame_reusable(
&mut frame,
&mut converted_frame,
Duration::MAX,
)
.map_err(|err| video_failure(err.to_string()))?;
} else {
fill_nv12_frame_from_surface(
&mut reusable_frame,
surface,
input.width,
input.height,
input.frame_number as i64,
)
.ok_or_else(|| {
"failed to lock NV12 surface for encoding".to_string()
})?;
encoder
.queue_video_frame_reusable(
&mut reusable_frame,
&mut converted_frame,
Duration::MAX,
)
.map_err(|err| video_failure(err.to_string()))?;
}
}
}
if let Some(worker) = &mut audio_worker {
worker.write_ready(&mut encoder)?;
}
encoded_frames += 1;
#[cfg(feature = "test-hooks")]
if injected_failure_at == Some(encoded_frames) {
return Err(video_failure(
cap_enc_ffmpeg::h264::QueueFrameError::Encode(ffmpeg::Error::External)
.to_string(),
));
}
if let Some(timing) = &sample_timing {
timing.record_frame(input.timeline_frame);
}
if encoded_frames == 1
&& let Some(atom) = record_first_queued_ms.as_ref()
{
let ms = pipeline_start_for_encoder.elapsed().as_millis() as u64;
let _ =
atom.compare_exchange(u64::MAX, ms, Ordering::Relaxed, Ordering::Relaxed);
}
}
let encode_elapsed = encode_start.elapsed();
if encoded_frames > 0 {
let encode_fps = encoded_frames as f64 / encode_elapsed.as_secs_f64().max(0.001);
info!(
encoded_frames = encoded_frames,
elapsed_secs = format!("{:.2}", encode_elapsed.as_secs_f64()),
encode_fps = format!("{:.1}", encode_fps),
"Encoder thread finished"
);
}
if let Some(worker) = audio_worker.take() {
worker.finish(&mut encoder)?;
}
if let Some(audio) = &mut streaming_audio {
audio.validate_to_end().map_err(Mp4PipelineError::from)?;
}
if encoder_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) || encoder_user_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) {
return Err(Mp4PipelineError::Interrupted);
}
let res = encoder
.finish()
.map_err(|e| format!("Failed to finish encoding: {e}"))?;
if let Err(e) = res.video_finish {
return Err(video_failure(format!("Video encoding failed: {e}")));
}
if let Err(e) = res.audio_finish {
return Err(Mp4PipelineError::Failure(format!("Audio encoding failed: {e}")));
}
Ok::<_, Mp4PipelineError>(encoder_output_path)
})
.then(move |r| async move {
let result = r.map_err(|e| Mp4PipelineError::Failure(e.to_string())).and_then(|v| v);
if result.is_err() && let Some(cancel) = &encoder_result_cancellation {
cancel.store(true, Ordering::Relaxed);
}
result
});
let stop_after_frames_sent = mode.stop_after_frames_sent;
let render_result_cancellation = audio_cancellation.clone();
let progress_cancellation = audio_cancellation.clone();
let progress_user_cancellation = user_cancellation;
let mut on_progress = on_progress;
let on_progress = move |frame| {
let keep_going = !progress_user_cancellation
.as_ref()
.is_some_and(|cancel| cancel.load(Ordering::Relaxed))
&& !progress_cancellation
.as_ref()
.is_some_and(|cancel| cancel.load(Ordering::Relaxed))
&& on_progress(frame);
if !keep_going && let Some(cancel) = &progress_cancellation {
cancel.store(true, Ordering::Relaxed);
}
keep_going
};
let render_video_task = export_render_to_channel(
&base.render_constants,
&base.project_config,
frame_tx,
&base.recording_meta,
meta,
base.segments
.iter()
.map(|s| RenderSegment {
cursor: s.cursor.clone(),
keyboard: s.keyboard.clone(),
decoders: s.decoders.clone(),
render_display: true,
})
.collect(),
fps,
self.resolution_base,
&base.recordings,
base.sample_windows.clone(),
stop_after_frames_sent,
nv12_render_startup_breakdown_ms,
audio_cancellation.is_some(),
on_progress,
base.sample_windows
.is_none()
.then(|| base.project_path.clone()),
)
.then(move |v| async move {
let result = v.map_err(|error| match &error {
cap_rendering::RenderingError::ImageLoadError(message)
if render_result_cancellation.is_some() && message == "Export cancelled" =>
{
Mp4PipelineError::Interrupted
}
_ => Mp4PipelineError::Failure(error.to_string()),
});
if result.is_err()
&& let Some(cancel) = &render_result_cancellation
{
cancel.store(true, Ordering::Relaxed);
}
result
});
if audio_cancellation.is_some() {
let validation = async move {
match audio_validation {
Some((task, abort, failure)) => {
complete_audio_validation(task, abort, failure).await
}
None => Ok(()),
}
};
let (encoder_result, render_result, validation_result) = tokio::join!(
encoder_thread,
render_until_audio_failure(render_video_task, validation_failure),
validation,
);
resolve_pipeline_results(encoder_result, render_result, validation_result)?;
if audio_cancellation
.as_ref()
.is_some_and(|cancel| cancel.load(Ordering::Relaxed))
{
return Err("Export cancelled".to_string());
}
if audio_control
.as_ref()
.is_some_and(|control| control.user.load(Ordering::Relaxed))
{
return Err("Export cancelled".to_string());
}
if let Some(temporary_output) = temporary_output {
Arc::try_unwrap(temporary_output)
.map_err(|_| "Export output is still in use".to_string())?
.persist_noclobber(&output_path)
.map_err(|error| error.to_string())?;
}
} else {
tokio::try_join!(encoder_thread, render_video_task)
.map_err(Mp4PipelineError::message)?;
}
Ok(output_path)
}
}
/// A video encoder error that fails this export attempt. A hardware
/// encoder's error also sets `restart`, so the export starts over on libx264.
fn video_encoder_failure(hardware: bool, restart: &AtomicBool, error: String) -> Mp4PipelineError {
if hardware {
restart.store(true, Ordering::Relaxed);
}
Mp4PipelineError::Failure(error)
}
/// Whether a failed attempt starts over on the software encoder: after a
/// hardware encoder error, once, and not for size-estimate samples or a
/// cancelled export.
fn restart_on_software_encoder(
attempt_was_software: bool,
sampled: bool,
hardware_encoder_failed: bool,
cancelled: bool,
) -> bool {
hardware_encoder_failed && !attempt_was_software && !sampled && !cancelled
}
enum ExportFramePayload {
Cpu(SharedNv12Buffer),
#[cfg(target_os = "macos")]
Surface(cap_rendering::Nv12Surface),
}
struct ExportFrame {
payload: ExportFramePayload,
width: u32,
height: u32,
y_stride: u32,
frame_number: u32,
timeline_frame: u32,
}
impl ExportFrame {
/// Contiguous NV12 bytes for CPU consumers (the first-frame screenshot);
/// surface frames are locked and copied once.
fn materialize_nv12(&self) -> Option<(Vec<u8>, u32)> {
match &self.payload {
ExportFramePayload::Cpu(data) => Some((data.as_ref().to_vec(), self.y_stride)),
#[cfg(target_os = "macos")]
ExportFramePayload::Surface(surface) => {
surface.with_locked_planes(|y_plane, y_stride, uv_plane, uv_stride| {
let width = self.width as usize;
let height = self.height as usize;
let mut data = Vec::with_capacity(width * height * 3 / 2);
for row in 0..height {
data.extend_from_slice(&y_plane[row * y_stride..row * y_stride + width]);
}
for row in 0..(height / 2) {
data.extend_from_slice(&uv_plane[row * uv_stride..row * uv_stride + width]);
}
(data, self.width)
})
}
}
}
}
struct FirstFrameNv12 {
data: Vec<u8>,
width: u32,
height: u32,
y_stride: u32,
}
fn nv12_from_rendered_frame(frame: Nv12RenderedFrame) -> ExportFrame {
#[cfg(target_os = "macos")]
if let Some(surface) = frame.surface {
return ExportFrame {
width: frame.width,
height: frame.height,
y_stride: frame.y_stride,
frame_number: frame.frame_number,
timeline_frame: frame.frame_number,
payload: ExportFramePayload::Surface(surface),
};
}
if frame.format != GpuOutputFormat::Rgba {
return ExportFrame {
width: frame.width,
height: frame.height,
y_stride: frame.y_stride,
frame_number: frame.frame_number,
timeline_frame: frame.frame_number,
payload: ExportFramePayload::Cpu(frame.data),
};
}
tracing::warn!(
frame_number = frame.frame_number,
"GPU NV12 converter returned RGBA - converting to NV12 on CPU"
);
let width = frame.width;
let height = frame.height;
let mut rgba_frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::RGBA, width, height);
let stride = rgba_frame.stride(0);
let src_stride = frame.y_stride as usize;
for row in 0..height as usize {
let src_start = row * src_stride;
let dst_start = row * stride;
let copy_width = (width as usize * 4).min(stride).min(src_stride);
if src_start + copy_width <= frame.data.len()
&& dst_start + copy_width <= rgba_frame.data_mut(0).len()
{
rgba_frame.data_mut(0)[dst_start..dst_start + copy_width]
.copy_from_slice(&frame.data[src_start..src_start + copy_width]);
}
}
if let Ok(mut converter) = ffmpeg::software::scaling::Context::get(
ffmpeg::format::Pixel::RGBA,
width,
height,
ffmpeg::format::Pixel::NV12,
width,
height,
ffmpeg::software::scaling::flag::Flags::FAST_BILINEAR,
) && cap_enc_ffmpeg::colour::use_bt709_for_rgb_input(
&mut converter,
ffmpeg::format::Pixel::RGBA,
ffmpeg::format::Pixel::NV12,
)
.is_ok()
{
let mut nv12_frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::NV12, width, height);
if converter.run(&rgba_frame, &mut nv12_frame).is_ok() {
let y_size = nv12_frame.stride(0) * height as usize;
let uv_size = nv12_frame.stride(1) * (height as usize / 2);
let y_data = &nv12_frame.data(0)[..y_size];
let uv_data = &nv12_frame.data(1)[..uv_size];
let mut result = Vec::with_capacity(width as usize * height as usize * 3 / 2);
if nv12_frame.stride(0) == width as usize {
result.extend_from_slice(y_data);
} else {
for row in 0..height as usize {
let start = row * nv12_frame.stride(0);
result.extend_from_slice(&y_data[start..start + width as usize]);
}
}
if nv12_frame.stride(1) == width as usize {
result.extend_from_slice(uv_data);
} else {
for row in 0..(height as usize / 2) {
let start = row * nv12_frame.stride(1);
result.extend_from_slice(&uv_data[start..start + width as usize]);
}
}
return ExportFrame {
payload: ExportFramePayload::Cpu(SharedNv12Buffer::from_vec(result)),
width,
height,
y_stride: width,
frame_number: frame.frame_number,
timeline_frame: frame.frame_number,
};
}
}
tracing::error!(
frame_number = frame.frame_number,
"swscale RGBA to NV12 conversion failed, using zeroed NV12"
);
ExportFrame {
payload: ExportFramePayload::Cpu(SharedNv12Buffer::from_vec(vec![
0u8;
width as usize
* height as usize
* 3
/ 2
])),
width,
height,
y_stride: width,
frame_number: frame.frame_number,
timeline_frame: frame.frame_number,
}
}
/// Per-output-frame audio budget for the encoder loop: the AAC frame PTS (the
/// running sample cursor) and the sample count to render for `frame_number`.
/// Returns `None` when the frame falls entirely within already-emitted audio, so
/// the stream stays gapless and strictly monotonic. The caller advances the
/// cursor to `pts + samples`. Shared with the tests so they exercise this exact
/// arithmetic rather than a re-implementation.
fn audio_frame_budget(
frame_number: u64,
sample_rate: u64,
fps: u64,
cursor: u64,
) -> Option<(i64, usize)> {
let end = ((frame_number + 1) * sample_rate) / fps;
if end <= cursor {
return None;
}
Some((cursor as i64, (end - cursor) as usize))
}
enum ExportAudioSource {
Renderer(AudioRenderer),
Streaming(ExportAudioRenderer),
}
struct ExportAudioTimeline {
project: ProjectConfiguration,
fps: u32,
sampled: bool,
}
struct ExportAudioJob {
frame_number: u32,
timeline_frame: u32,
}
/// How far audio may fall behind video before the encoder thread waits for
/// it. The muxer interleaves by timestamp only within its 10 s window, so
/// staying well inside it keeps the file identical to a single thread's.
const AUDIO_LAG_LIMIT_SECS: u32 = 2;
/// Renders and AAC-encodes each output frame's audio on its own thread, so
/// voice enhancement and the AAC encoder run alongside video encoding
/// instead of after it on the encoder thread. Packets come back in encoder
/// order, one batch per frame, and the muxer interleaves them by timestamp.
struct ExportAudioWorker {
jobs: std::sync::mpsc::Sender<ExportAudioJob>,
packets: std::sync::mpsc::Receiver<Vec<ffmpeg::Packet>>,
thread: Option<std::thread::JoinHandle<Result<AACEncoder, Mp4PipelineError>>>,
requested: u32,
completed: u32,
lag_limit: u32,
}
impl ExportAudioWorker {
fn spawn(
stream_time_base: ffmpeg::Rational,
mut encoder: AACEncoder,
mut source: ExportAudioSource,
timeline: ExportAudioTimeline,
is_cancelled: impl Fn() -> bool + Send + 'static,
) -> Result<Self, Mp4PipelineError> {
let (jobs, job_rx) = std::sync::mpsc::channel::<ExportAudioJob>();
let (packet_tx, packets) = std::sync::mpsc::channel();
let lag_limit = timeline.fps.max(1) * AUDIO_LAG_LIMIT_SECS;
let thread = std::thread::Builder::new()
.name("export-audio".to_string())
.spawn(move || {
let ExportAudioTimeline {
project,
fps,
sampled,
} = timeline;
let mut queue = PacketQueue::new(stream_time_base);
let sample_rate = u64::from(AudioRenderer::SAMPLE_RATE);
let fps_u64 = u64::from(fps);
let mut cursor = 0u64;
let mut last_timeline_frame: Option<u32> = None;
for (rendered_frames, job) in job_rx.into_iter().enumerate() {
if is_cancelled() {
return Err(Mp4PipelineError::Interrupted);
}
let frame_number = u64::from(job.frame_number);
match &mut source {
ExportAudioSource::Renderer(audio) => {
if rendered_frames == 0 {
audio.set_playhead(
if sampled {
job.timeline_frame as f64 / fps as f64
} else {
0.0
},
&project,
);
} else if sampled
&& last_timeline_frame
.is_some_and(|last| job.timeline_frame != last + 1)
{
audio
.set_playhead(job.timeline_frame as f64 / fps as f64, &project);
cursor = frame_number * sample_rate / fps_u64;
}
if let Some((pts, samples)) =
audio_frame_budget(frame_number, sample_rate, fps_u64, cursor)
{
cursor = pts as u64 + samples as u64;
let mut frame = audio
.render_frame(samples, &project)
.unwrap_or_else(|| silent_audio_frame(samples));
frame.set_pts(Some(pts));
let _ = encoder.send_frame_to(frame, Duration::MAX, &mut queue);
}
}
ExportAudioSource::Streaming(audio) => {
if let Some((pts, samples)) =
audio_frame_budget(frame_number, sample_rate, fps_u64, cursor)
{
cursor = pts as u64 + samples as u64;
let rendered = audio
.render_chunks(samples, &project, |offset, data| {
let mut frame = packed_audio_frame(data);
frame.set_pts(Some(pts + offset as i64));
encoder
.send_frame_to(frame, Duration::MAX, &mut queue)
.map_err(|error| {
ExportAudioError::Sink(error.to_string())
})
})
.map_err(Mp4PipelineError::from)?;
if rendered.is_none() {
let mut offset = 0;
while offset < samples {
if is_cancelled() {
return Err(Mp4PipelineError::Interrupted);
}
let count = (samples - offset).min(4_096);
let mut frame = silent_audio_frame(count);
frame.set_pts(Some(pts + offset as i64));
encoder
.send_frame_to(frame, Duration::MAX, &mut queue)
.map_err(|error| error.to_string())?;
offset += count;
}
}
}
}
}
last_timeline_frame = Some(job.timeline_frame);
if packet_tx.send(queue.take()).is_err() {
return Err(Mp4PipelineError::Interrupted);
}
}
if let ExportAudioSource::Streaming(audio) = &mut source {
audio.validate_to_end().map_err(Mp4PipelineError::from)?;
}
Ok(encoder)
})
.map_err(|error| {
Mp4PipelineError::Failure(format!(
"Could not start the export audio thread: {error}"
))
})?;
Ok(Self {
jobs,
packets,
thread: Some(thread),
requested: 0,
completed: 0,
lag_limit,
})
}
fn request(&mut self, frame_number: u32, timeline_frame: u32) {
self.requested += 1;
let _ = self.jobs.send(ExportAudioJob {
frame_number,
timeline_frame,
});
}
fn write_ready(&mut self, output: &mut MP4File) -> Result<(), Mp4PipelineError> {
loop {
let packets = if self.requested - self.completed > self.lag_limit {
self.packets.recv().ok()
} else {
match self.packets.try_recv() {
Ok(packets) => Some(packets),
Err(std::sync::mpsc::TryRecvError::Empty) => return Ok(()),
Err(std::sync::mpsc::TryRecvError::Disconnected) => None,
}
};
// The worker only hangs up before its jobs run out when it
// fails; surface that now rather than after the rest of the video.
let Some(packets) = packets else {
return Err(self
.thread
.take()
.and_then(|thread| join_audio_thread(thread).err())
.unwrap_or_else(|| {
Mp4PipelineError::Failure("Export audio thread stopped early".to_string())
}));
};
self.completed += 1;
output
.write_packets(packets)
.map_err(|error| Mp4PipelineError::Failure(error.to_string()))?;
}
}
fn finish(self, output: &mut MP4File) -> Result<(), Mp4PipelineError> {
let Self {
jobs,
packets,
thread,
..
} = self;
drop(jobs);
for packets in packets.iter() {
output
.write_packets(packets)
.map_err(|error| Mp4PipelineError::Failure(error.to_string()))?;
}
let thread = thread.ok_or_else(|| {
Mp4PipelineError::Failure("Export audio thread stopped early".to_string())
})?;
output.attach_audio(join_audio_thread(thread)?.boxed());
Ok(())
}
}
fn join_audio_thread(
thread: std::thread::JoinHandle<Result<AACEncoder, Mp4PipelineError>>,
) -> Result<AACEncoder, Mp4PipelineError> {
thread
.join()
.map_err(|_| Mp4PipelineError::Failure("Export audio thread panicked".to_string()))?
}
#[derive(Debug)]
pub(crate) struct TemporaryExportOutput {
path: tempfile::TempPath,
#[cfg(unix)]
file: std::fs::File,
#[cfg(unix)]
final_permissions: std::fs::Permissions,
}
impl AsRef<std::path::Path> for TemporaryExportOutput {
fn as_ref(&self) -> &std::path::Path {
self.path.as_ref()
}
}
impl TemporaryExportOutput {
pub(crate) fn to_path_buf(&self) -> PathBuf {
self.path.to_path_buf()
}
#[cfg(unix)]
fn restore_permissions(&self, output: Option<&std::path::Path>) -> std::io::Result<()> {
use std::os::unix::fs::MetadataExt;
let path_metadata = std::fs::symlink_metadata(&self.path)?;
let file_metadata = self.file.metadata()?;
if !path_metadata.is_file()
|| path_metadata.dev() != file_metadata.dev()
|| path_metadata.ino() != file_metadata.ino()
{
return Err(std::io::Error::other("Export temporary file was replaced"));
}
let existing_permissions = match output {
Some(path) => match std::fs::symlink_metadata(path) {
Ok(metadata) if metadata.is_file() => Some(metadata.permissions()),
Err(error) if error.kind() != std::io::ErrorKind::NotFound => return Err(error),
_ => None,
},
None => None,
};
self.file
.set_permissions(existing_permissions.unwrap_or_else(|| self.final_permissions.clone()))
}
pub(crate) fn persist_noclobber(
self,
output: &std::path::Path,
) -> Result<(), tempfile::PathPersistError> {
#[cfg(unix)]
if let Err(error) = self.restore_permissions(None) {
return Err(tempfile::PathPersistError {
error,
path: self.path,
});
}
self.path.persist_noclobber(output)
}
pub(crate) fn persist_replace(
self,
output: &std::path::Path,
) -> Result<(), tempfile::PathPersistError> {
#[cfg(unix)]
if let Err(error) = self.restore_permissions(Some(output)) {
return Err(tempfile::PathPersistError {
error,
path: self.path,
});
}
self.path.persist(output)
}
}
pub(crate) fn temporary_export_output(
output: &std::path::Path,
) -> Result<TemporaryExportOutput, String> {
let parent = output.parent().unwrap_or_else(|| std::path::Path::new("."));
let suffix = output
.extension()
.map(|extension| format!(".{}", extension.to_string_lossy()))
.unwrap_or_default();
let temporary = tempfile::Builder::new()
.prefix(".cap-export-")
.suffix(&suffix)
.tempfile_in(parent)
.map_err(|error| error.to_string())?;
#[cfg(unix)]
let final_permissions = {
use std::os::unix::fs::PermissionsExt;
// An empty sibling captures creation permissions without changing the
// process-wide umask or exposing incomplete media to other users.
let probe = tempfile::Builder::new()
.prefix(".cap-export-permissions-")
.permissions(std::fs::Permissions::from_mode(0o666))
.tempfile_in(parent)
.map_err(|error| error.to_string())?;
probe
.as_file()
.metadata()
.map_err(|error| error.to_string())?
.permissions()
};
let (file, path) = temporary.into_parts();
#[cfg(not(unix))]
drop(file);
Ok(TemporaryExportOutput {
path,
#[cfg(unix)]
file,
#[cfg(unix)]
final_permissions,
})
}
fn packed_audio_frame(data: &[f32]) -> ffmpeg::frame::Audio {
let mut frame = ffmpeg::frame::Audio::new(
AudioRenderer::SAMPLE_FORMAT,
data.len() / 2,
ffmpeg::ChannelLayout::STEREO,
);
frame.set_rate(AudioRenderer::SAMPLE_RATE);
for (sample, bytes) in data.iter().zip(
frame
.data_mut(0)
.chunks_exact_mut(std::mem::size_of::<f32>()),
) {
bytes.copy_from_slice(&sample.to_ne_bytes());
}
frame
}
fn silent_audio_frame(samples: usize) -> ffmpeg::frame::Audio {
let mut frame = ffmpeg::frame::Audio::new(
AudioRenderer::SAMPLE_FORMAT,
samples,
ffmpeg::ChannelLayout::STEREO,
);
frame.set_rate(AudioRenderer::SAMPLE_RATE);
for plane in 0..frame.planes() {
frame.data_mut(plane).fill(0);
}
frame
}
#[cfg(target_os = "macos")]
fn fill_nv12_frame_from_surface(
frame: &mut ffmpeg::frame::Video,
surface: &cap_rendering::Nv12Surface,
width: u32,
height: u32,
pts: i64,
) -> Option<()> {
surface.with_locked_planes(|y_plane, y_stride, uv_plane, uv_stride| {
frame.set_pts(Some(pts));
let width = width as usize;
let height = height as usize;
let dst_y_stride = frame.stride(0);
for row in 0..height {
let src_start = row * y_stride;
let dst_start = row * dst_y_stride;
frame.data_mut(0)[dst_start..dst_start + width]
.copy_from_slice(&y_plane[src_start..src_start + width]);
}
let dst_uv_stride = frame.stride(1);
for row in 0..(height / 2) {
let src_start = row * uv_stride;
let dst_start = row * dst_uv_stride;
frame.data_mut(1)[dst_start..dst_start + width]
.copy_from_slice(&uv_plane[src_start..src_start + width]);
}
})
}
fn fill_nv12_frame_direct(
frame: &mut ffmpeg::frame::Video,
nv12_data: &[u8],
width: u32,
height: u32,
y_stride: u32,
pts: i64,
) {
frame.set_pts(Some(pts));
let width = width as usize;
let height = height as usize;
let y_stride = y_stride as usize;
let y_plane_size = y_stride * height;
let y_src = &nv12_data[..y_plane_size.min(nv12_data.len())];
let uv_src = if y_plane_size < nv12_data.len() {
&nv12_data[y_plane_size..]
} else {
&[]
};
let dst_y_stride = frame.stride(0);
if dst_y_stride == y_stride {
let copy_len = y_src.len().min(frame.data_mut(0).len());
frame.data_mut(0)[..copy_len].copy_from_slice(&y_src[..copy_len]);
} else {
for row in 0..height {
let src_start = row * y_stride;
let dst_start = row * dst_y_stride;
let copy_width = width.min(y_stride).min(dst_y_stride);
if src_start + copy_width <= y_src.len()
&& dst_start + copy_width <= frame.data_mut(0).len()
{
frame.data_mut(0)[dst_start..dst_start + copy_width]
.copy_from_slice(&y_src[src_start..src_start + copy_width]);
}
}
}
let uv_height = height / 2;
let dst_uv_stride = frame.stride(1);
if dst_uv_stride == width {
let copy_len = uv_src.len().min(frame.data_mut(1).len());
frame.data_mut(1)[..copy_len].copy_from_slice(&uv_src[..copy_len]);
} else {
for row in 0..uv_height {
let src_start = row * width;
let dst_start = row * dst_uv_stride;
let copy_width = width.min(dst_uv_stride);
if src_start + copy_width <= uv_src.len()
&& dst_start + copy_width <= frame.data_mut(1).len()
{
frame.data_mut(1)[dst_start..dst_start + copy_width]
.copy_from_slice(&uv_src[src_start..src_start + copy_width]);
}
}
}
}
#[cfg(test)]
struct Nv12ExportFrame {
nv12_data: SharedNv12Buffer,
width: u32,
height: u32,
y_stride: u32,
pts: i64,
}
#[cfg(test)]
fn fill_nv12_frame(frame: &mut ffmpeg::frame::Video, input: &Nv12ExportFrame) {
fill_nv12_frame_direct(
frame,
&input.nv12_data,
input.width,
input.height,
input.y_stride,
input.pts,
);
}
fn rgba_into_rgb(mut pixels: Vec<u8>) -> Vec<u8> {
let pixel_count = pixels.len() / 4;
for pixel in 1..pixel_count {
let (src, dst) = (pixel * 4, pixel * 3);
pixels[dst] = pixels[src];
pixels[dst + 1] = pixels[src + 1];
pixels[dst + 2] = pixels[src + 2];
}
pixels.truncate(pixel_count * 3);
pixels
}
// Runs while the encoder is still draining its surface pool, so it converts in
// place instead of holding the NV12 copy, the RGBA frame and a growing RGB copy
// at once (measured at 4K: 48 MB of extra heap before, 27 MB after).
fn save_screenshot_from_nv12(
nv12_data: Vec<u8>,
width: u32,
height: u32,
y_stride: u32,
project_path: &std::path::Path,
) {
let y_plane_size = (y_stride as usize) * (height as usize);
if y_plane_size >= nv12_data.len() {
return;
}
let (y_data, uv_data) = nv12_data.split_at(y_plane_size);
let mut rgba = vec![0u8; (width * height * 4) as usize];
// The export's own NV12, so BT.709 like the stream it is encoded into.
cap_rendering::cpu_yuv::nv12_to_rgba_simd(
y_data,
uv_data,
width,
height,
y_stride,
width,
&mut rgba,
cap_rendering::cpu_yuv::YuvMatrix::Bt709,
);
drop(nv12_data);
let rgb_img =
ImageBuffer::<image::Rgb<u8>, Vec<u8>>::from_raw(width, height, rgba_into_rgb(rgba));
let Some(rgb_img) = rgb_img else {
return;
};
let screenshots_dir = project_path.join("screenshots");
if std::fs::create_dir_all(&screenshots_dir).is_err() {
return;
}
let screenshot_path = screenshots_dir.join("display.jpg");
let _ = rgb_img.save(&screenshot_path);
}
use cap_project::{ProjectConfiguration, RecordingMeta, StudioRecordingMeta};
use cap_rendering::{ProjectRecordingsMeta, RenderVideoConstants};
const FRAME_RECEIVE_INITIAL_TIMEOUT_SECS: u64 = 120;
const FRAME_RECEIVE_STEADY_TIMEOUT_SECS: u64 = 90;
const MAX_CONSECUTIVE_FRAME_TIMEOUTS: u32 = 3;
#[allow(clippy::too_many_arguments)]
async fn export_render_to_channel(
constants: &RenderVideoConstants,
project: &ProjectConfiguration,
sender: std::sync::mpsc::SyncSender<ExportFrame>,
recording_meta: &RecordingMeta,
meta: &StudioRecordingMeta,
render_segments: Vec<RenderSegment>,
fps: u32,
resolution_base: XY<u32>,
recordings: &ProjectRecordingsMeta,
frame_windows: Option<cap_rendering::FrameWindows>,
stop_after_frames_sent: Option<u32>,
startup_breakdown_ms: Option<Arc<Mutex<Option<cap_rendering::Nv12RenderStartupBreakdownMs>>>>,
stop_on_encoder_drop: bool,
mut on_progress: impl FnMut(u32) -> bool + Send + 'static,
screenshot_project_path: Option<PathBuf>,
) -> Result<(), cap_rendering::RenderingError> {
let (tx_image_data, mut video_rx) = tokio::sync::mpsc::channel::<(Nv12RenderedFrame, u32)>(8);
let sampling = frame_windows.is_some();
let render_result = {
let render_future = Box::pin(cap_rendering::render_video_to_channel_nv12(
constants,
project,
tx_image_data,
recording_meta,
meta,
render_segments,
fps,
resolution_base,
recordings,
frame_windows,
stop_after_frames_sent,
startup_breakdown_ms,
));
let forward_future = async {
let mut first_frame_data: Option<FirstFrameNv12> = None;
let mut frame_count = 0u32;
let mut consecutive_timeouts = 0u32;
let mut encoder_blocked = Duration::ZERO;
loop {
let timeout_secs = if frame_count == 0 {
FRAME_RECEIVE_INITIAL_TIMEOUT_SECS
} else {
FRAME_RECEIVE_STEADY_TIMEOUT_SECS
};
let Some((frame, _frame_number)) = (match tokio::time::timeout(
Duration::from_secs(timeout_secs),
video_rx.recv(),
)
.await
{
Ok(frame) => {
consecutive_timeouts = 0;
frame
}
Err(_) => {
consecutive_timeouts += 1;
if consecutive_timeouts >= MAX_CONSECUTIVE_FRAME_TIMEOUTS {
return Err(cap_rendering::RenderingError::ImageLoadError(format!(
"Export timed out {MAX_CONSECUTIVE_FRAME_TIMEOUTS} times consecutively after {timeout_secs}s each waiting for frame {frame_count}"
)));
}
warn!(
frame_count = frame_count,
timeout_secs = timeout_secs,
consecutive_timeouts = consecutive_timeouts,
"Timed out waiting for rendered frame"
);
continue;
}
}) else {
break;
};
if !(on_progress)(frame_count) {
return Err(cap_rendering::RenderingError::ImageLoadError(
"Export cancelled".to_string(),
));
}
let mut export_frame = nv12_from_rendered_frame(frame);
if sampling {
export_frame.frame_number = frame_count;
}
if screenshot_project_path.is_some()
&& first_frame_data.is_none()
&& let Some((data, y_stride)) = export_frame.materialize_nv12()
{
first_frame_data = Some(FirstFrameNv12 {
data,
width: export_frame.width,
height: export_frame.height,
y_stride,
});
}
let send_started = std::time::Instant::now();
let sent = sender.send(export_frame);
encoder_blocked += send_started.elapsed();
if sent.is_err() {
warn!("Encoder dropped, stopping render forwarding");
if stop_on_encoder_drop {
return Err(cap_rendering::RenderingError::ImageLoadError(
"Export cancelled".to_string(),
));
}
break;
}
frame_count += 1;
}
drop(sender);
if let Some(first) = first_frame_data
&& let Some(pp) = screenshot_project_path
{
let _screenshot_task = tokio::task::spawn_blocking(move || {
save_screenshot_from_nv12(
first.data,
first.width,
first.height,
first.y_stride,
&pp,
);
});
}
Ok::<_, cap_rendering::RenderingError>(encoder_blocked)
};
cap_rendering::loop_stats::take();
tokio::try_join!(render_future, forward_future)
};
let (_, encoder_blocked) = render_result?;
// The loop_stats counters are process-wide, so an export that overlaps another one reports
// a mix of both. That is fine for profiling one export at a time; per-export numbers need
// the counters passed through the render loop instead of kept in statics.
if cap_rendering::loop_stats::enabled() {
let [decode, render, prefetch, join, render_send, frames] =
cap_rendering::loop_stats::take();
info!(
frames,
decode_ms = decode / 1000,
render_ms = render / 1000,
prefetch_ms = prefetch / 1000,
join_ms = join / 1000,
render_send_ms = render_send / 1000,
encoder_send_ms = encoder_blocked.as_millis() as u64,
"NV12 export loop stats"
);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hardware_encoder_errors_restart_on_software() {
let queue = cap_enc_ffmpeg::h264::QueueFrameError::Encode(ffmpeg::Error::External);
let flush = format!("Video encoding failed: {}", ffmpeg::Error::External);
for error in [queue.to_string(), flush] {
let restart = AtomicBool::new(false);
let failure = video_encoder_failure(true, &restart, error.clone());
assert_eq!(failure.message(), error);
assert!(restart_on_software_encoder(
false,
false,
restart.load(Ordering::Relaxed),
false
));
}
}
#[test]
fn software_encoder_errors_do_not_restart() {
let restart = AtomicBool::new(false);
let error = cap_enc_ffmpeg::h264::QueueFrameError::Encode(ffmpeg::Error::External);
video_encoder_failure(false, &restart, error.to_string());
assert!(!restart_on_software_encoder(
false,
false,
restart.load(Ordering::Relaxed),
false
));
// Already on libx264, a size-estimate sample, or cancelled.
assert!(!restart_on_software_encoder(true, false, true, false));
assert!(!restart_on_software_encoder(false, true, true, false));
assert!(!restart_on_software_encoder(false, false, true, true));
}
fn sum_samples(sample_rate: u64, fps: u64, frames: u64) -> u64 {
(0..frames)
.map(|n| {
let start = (n * sample_rate) / fps;
let end = ((n + 1) * sample_rate) / fps;
end - start
})
.sum()
}
#[test]
fn audio_samples_match_duration_across_fps() {
let sample_rate = u64::from(AudioRenderer::SAMPLE_RATE);
for fps in [24u64, 30, 60, 90, 120, 144] {
let frames = fps * 10;
let expected = (frames * sample_rate) / fps;
assert_eq!(sum_samples(sample_rate, fps, frames), expected);
}
}
// Drives the SAME `audio_frame_budget` the encoder loop uses, so it guards the
// real arithmetic: each frame's PTS equals the previous frame's end (gapless,
// non-overlapping) and the cursor advances exactly to the duration budget — i.e.
// audio length stays anchored to the video frame grid with no drift. Read
// alignment of the source PCM to presentation time is covered by the
// export_audio_* tests in crates/editor/src/audio.rs.
#[test]
fn audio_frame_budget_is_monotonic_and_contiguous() {
let sample_rate = u64::from(AudioRenderer::SAMPLE_RATE);
for fps in [24u64, 30, 60] {
let frames = fps * 3;
let mut cursor = 0u64;
for n in 0..frames {
let (pts, samples) = audio_frame_budget(n, sample_rate, fps, cursor)
.expect("a normal-rate frame always has a budget");
assert_eq!(pts as u64, cursor, "fps {fps} frame {n}: pts == prior end");
assert!(samples > 0, "fps {fps} frame {n}: frame must carry samples");
cursor = pts as u64 + samples as u64;
}
assert_eq!(cursor, (frames * sample_rate) / fps);
}
}
#[test]
fn pipeline_interruption_preserves_the_peer_failure() {
assert_eq!(
resolve_pipeline_results(
Err(Mp4PipelineError::Interrupted),
Err(Mp4PipelineError::Failure("renderer failed".into())),
Ok(()),
),
Err("renderer failed".into())
);
assert_eq!(
resolve_pipeline_results(
Err(Mp4PipelineError::Failure("audio failed".into())),
Err(Mp4PipelineError::Interrupted),
Ok(()),
),
Err("audio failed".into())
);
assert_eq!(
resolve_pipeline_results(Err(Mp4PipelineError::Interrupted), Ok(()), Ok(())),
Err("Export cancelled".into())
);
}
#[test]
fn detached_and_active_source_errors_keep_original_order() {
for (active, detached) in [(1, 7), (7, 1)] {
let error = resolve_pipeline_results(
Err(Mp4PipelineError::AudioSourceFailure {
source_index: active,
message: format!("source {active}"),
}),
Err(Mp4PipelineError::Interrupted),
Err(Mp4PipelineError::AudioSourceFailure {
source_index: detached,
message: format!("source {detached}"),
}),
);
assert_eq!(error, Err("source 1".into()));
}
assert_eq!(
resolve_pipeline_results(
Err(Mp4PipelineError::Interrupted),
Err(Mp4PipelineError::Interrupted),
Err(Mp4PipelineError::from(ExportAudioError::Worker {
source_index: 3,
message: "decoder panicked".into(),
})),
),
Err("Audio source worker failed: decoder panicked".into()),
);
}
#[tokio::test]
async fn validation_failure_before_render_poll_skips_producer() {
let failure = Arc::new(Notify::new());
failure.notify_one();
let polled = AtomicBool::new(false);
let result = render_until_audio_failure(
async {
polled.store(true, Ordering::Relaxed);
Ok(())
},
Some(failure),
)
.await;
assert!(matches!(result, Err(Mp4PipelineError::Interrupted)));
assert!(!polled.load(Ordering::Relaxed));
}
#[tokio::test]
async fn validation_failure_releases_pending_producer_receiver_and_temp_owners() {
let directory = tempfile::tempdir().unwrap();
let output = directory.path().join("export.mp4");
let temporary = Arc::new(temporary_export_output(&output).unwrap());
let temporary_path = temporary.to_path_buf();
let abort = Arc::new(AtomicBool::new(false));
let user = AtomicBool::new(false);
let failure = Arc::new(Notify::new());
let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::<()>(1);
let (receiver_started_tx, receiver_started_rx) = tokio::sync::oneshot::channel();
let (producer_started_tx, producer_started_rx) = tokio::sync::oneshot::channel();
let receiver_guard = temporary.clone();
let receiver = tokio::task::spawn_blocking(move || {
let _guard = receiver_guard;
receiver_started_tx.send(()).unwrap();
assert!(frame_rx.recv().is_err());
Err::<PathBuf, _>(Mp4PipelineError::Interrupted)
});
receiver_started_rx.await.unwrap();
let producer_guard = temporary.clone();
let producer = async move {
let _sender = frame_tx;
let _guard = producer_guard;
producer_started_tx.send(()).unwrap();
std::future::pending::<Result<(), Mp4PipelineError>>().await
};
let validation_guard = temporary.clone();
let worker_abort = abort.clone();
let worker_failure = failure.clone();
let validator = tokio::spawn(async move {
let _guard = validation_guard;
producer_started_rx.await.unwrap();
worker_abort.store(true, Ordering::Relaxed);
worker_failure.notify_one();
Err(Mp4PipelineError::AudioSourceFailure {
source_index: 0,
message: "unused source failed".into(),
})
});
let result = tokio::time::timeout(Duration::from_secs(2), async {
let (encoder, render, validation) = tokio::join!(
async { receiver.await.unwrap() },
render_until_audio_failure(producer, Some(failure.clone())),
complete_audio_validation(validator, abort.clone(), failure),
);
resolve_pipeline_results(encoder, render, validation)
})
.await
.unwrap();
assert_eq!(result, Err("unused source failed".into()));
assert!(abort.load(Ordering::Relaxed));
assert!(!user.load(Ordering::Relaxed));
assert_eq!(Arc::strong_count(&temporary), 1);
drop(Arc::try_unwrap(temporary).unwrap());
assert!(!temporary_path.exists());
assert!(!output.exists());
}
#[tokio::test]
async fn validator_join_error_notifies_and_aborts() {
let abort = Arc::new(AtomicBool::new(false));
let failure = Arc::new(Notify::new());
let task = tokio::task::spawn_blocking(|| -> Result<(), Mp4PipelineError> {
panic!("validator wrapper panic");
});
let result = complete_audio_validation(task, abort.clone(), failure.clone()).await;
assert!(matches!(result, Err(Mp4PipelineError::Failure(_))));
assert!(abort.load(Ordering::Relaxed));
tokio::time::timeout(Duration::from_secs(2), failure.notified())
.await
.unwrap();
}
#[test]
fn temporary_output_is_removed_on_failure_and_never_overwrites() {
let directory = tempfile::tempdir().unwrap();
let output = directory.path().join("export.mp4");
let temporary = temporary_export_output(&output).unwrap();
let path = temporary.to_path_buf();
std::fs::write(&path, b"partial").unwrap();
drop(temporary);
assert!(!path.exists());
assert!(!output.exists());
let temporary = temporary_export_output(&output).unwrap();
std::fs::write(&temporary, b"completed").unwrap();
std::fs::write(&output, b"existing").unwrap();
drop(temporary.persist_noclobber(&output).unwrap_err());
assert_eq!(std::fs::read(&output).unwrap(), b"existing");
assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 1);
}
#[cfg(unix)]
#[test]
fn temporary_output_preserves_default_creation_permissions() {
use std::os::unix::fs::PermissionsExt;
let directory = tempfile::tempdir().unwrap();
let output = directory.path().join("export.mp4");
let reference = directory.path().join("reference.mp4");
std::fs::write(&reference, b"reference").unwrap();
let temporary = temporary_export_output(&output).unwrap();
assert_eq!(
std::fs::metadata(&temporary).unwrap().permissions().mode() & 0o077,
0
);
assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 2);
std::fs::write(&temporary, b"completed").unwrap();
temporary.persist_noclobber(&output).unwrap();
assert_eq!(
std::fs::metadata(output).unwrap().permissions().mode() & 0o777,
std::fs::metadata(reference).unwrap().permissions().mode() & 0o777
);
}
#[cfg(unix)]
#[test]
fn replaced_temporary_output_cannot_change_another_files_permissions() {
use std::os::unix::fs::{PermissionsExt, symlink};
let directory = tempfile::tempdir().unwrap();
let output = directory.path().join("export.mp4");
let private = directory.path().join("private.mp4");
std::fs::write(&private, b"private recording").unwrap();
std::fs::set_permissions(&private, std::fs::Permissions::from_mode(0o600)).unwrap();
let temporary = temporary_export_output(&output).unwrap();
let temporary_path = temporary.to_path_buf();
std::fs::remove_file(&temporary_path).unwrap();
symlink(&private, &temporary_path).unwrap();
drop(temporary.persist_noclobber(&output).unwrap_err());
assert!(!output.exists());
assert!(!temporary_path.exists());
assert_eq!(std::fs::read(&private).unwrap(), b"private recording");
assert_eq!(
std::fs::metadata(private).unwrap().permissions().mode() & 0o777,
0o600
);
}
#[test]
fn silent_audio_frame_matches_renderer_format() {
ffmpeg::init().unwrap();
let samples = 1600usize;
let frame = silent_audio_frame(samples);
assert_eq!(frame.samples(), samples);
assert_eq!(frame.rate(), AudioRenderer::SAMPLE_RATE);
assert_eq!(frame.format(), AudioRenderer::SAMPLE_FORMAT);
assert_eq!(frame.channel_layout(), ffmpeg::ChannelLayout::STEREO);
for plane in 0..frame.planes() {
assert!(frame.data(plane).iter().all(|byte| *byte == 0));
}
}
#[test]
fn rgba_into_rgb_matches_dropping_alpha() {
for pixel_count in [0usize, 1, 2, 3, 7, 64] {
let rgba = (0..pixel_count * 4)
.map(|value| (value * 37 % 251) as u8)
.collect::<Vec<_>>();
let expected = rgba
.chunks(4)
.flat_map(|chunk| [chunk[0], chunk[1], chunk[2]])
.collect::<Vec<_>>();
assert_eq!(rgba_into_rgb(rgba), expected);
}
}
#[test]
fn fill_nv12_frame_preserves_data_layout() {
ffmpeg::init().unwrap();
let width = 8u32;
let height = 4u32;
let y_size = (width * height) as usize;
let uv_size = (width * height / 2) as usize;
let mut nv12_data = vec![0u8; y_size + uv_size];
for (i, item) in nv12_data.iter_mut().take(y_size).enumerate() {
*item = (i % 256) as u8;
}
for (i, item) in nv12_data.iter_mut().skip(y_size).take(uv_size).enumerate() {
*item = (128 + i % 128) as u8;
}
let input = Nv12ExportFrame {
nv12_data: SharedNv12Buffer::from_vec(nv12_data.clone()),
width,
height,
y_stride: width,
pts: 42,
};
let mut frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::NV12, width, height);
fill_nv12_frame(&mut frame, &input);
assert_eq!(frame.pts(), Some(42));
for row in 0..height as usize {
for col in 0..width as usize {
let src_val = nv12_data[row * width as usize + col];
let dst_val = frame.data(0)[row * frame.stride(0) + col];
assert_eq!(src_val, dst_val, "Y mismatch at ({col}, {row})");
}
}
for row in 0..(height / 2) as usize {
for col in 0..width as usize {
let src_val = nv12_data[y_size + row * width as usize + col];
let dst_val = frame.data(1)[row * frame.stride(1) + col];
assert_eq!(src_val, dst_val, "UV mismatch at ({col}, {row})");
}
}
}
#[test]
fn nv12_from_rendered_frame_passthrough_for_nv12_format() {
use cap_rendering::{GpuOutputFormat, Nv12RenderedFrame};
let data = vec![1u8, 2, 3, 4, 5, 6];
let frame = Nv12RenderedFrame {
data: SharedNv12Buffer::from_vec(data.clone()),
width: 4,
height: 2,
y_stride: 4,
frame_number: 0,
target_time_ns: 0,
format: GpuOutputFormat::Nv12,
#[cfg(target_os = "macos")]
surface: None,
#[cfg(target_os = "linux")]
gpu: None,
};
let result = nv12_from_rendered_frame(frame);
let ExportFramePayload::Cpu(nv12_data) = &result.payload else {
panic!("CPU frame must stay a CPU payload");
};
assert_eq!(&**nv12_data, data.as_slice());
}
#[test]
fn nv12_export_frame_dimensions_match() {
let width = 1920u32;
let height = 1080u32;
assert!(
width.is_multiple_of(4),
"1920 should be NV12-compatible (divisible by 4)"
);
assert!(
height.is_multiple_of(2),
"1080 should be NV12-compatible (divisible by 2)"
);
let nv12_size = width as usize * height as usize * 3 / 2;
assert_eq!(nv12_size, 3_110_400);
let rgba_size = width as usize * height as usize * 4;
assert_eq!(rgba_size, 8_294_400);
let savings_pct = (1.0 - nv12_size as f64 / rgba_size as f64) * 100.0;
assert!(
savings_pct > 62.0 && savings_pct < 63.0,
"NV12 should save ~62.5% vs RGBA, got {savings_pct:.1}%"
);
}
}