Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
use std::sync::Arc;
use std::time::Instant;
use cap_project::{ClipTransitionType, CursorEvents, ProjectConfiguration};
#[cfg(target_os = "macos")]
use cap_rendering::SurfaceFrame;
use cap_rendering::{
DecodedSegmentFrames, FrameLayout, FrameRenderStageTimings, FrameRenderer, Nv12RenderedFrame,
ProjectUniforms, RenderVideoConstants, RenderedFrame, RendererLayers, TransitionRenderInput,
};
use tokio::sync::{mpsc, oneshot};
use crate::telemetry::{PlaybackRenderOutputFormat, PlaybackTelemetry, PlaybackTelemetryEvent};
#[allow(clippy::large_enum_variant)]
pub enum RendererMessage {
PrepareOutputSize {
width: u32,
height: u32,
},
RenderThumbnail {
input: RendererTransitionInput,
finished: oneshot::Sender<Result<RenderedFrame, String>>,
},
RenderFrame {
segment_frames: DecodedSegmentFrames,
uniforms: ProjectUniforms,
finished: oneshot::Sender<bool>,
cursor: Arc<CursorEvents>,
queued_at: Instant,
realtime: bool,
},
RenderTransition {
outgoing: RendererTransitionInput,
incoming: RendererTransitionInput,
kind: ClipTransitionType,
progress: f32,
finished: oneshot::Sender<bool>,
queued_at: Instant,
realtime: bool,
},
Stop {
finished: oneshot::Sender<()>,
},
}
pub struct RendererTransitionInput {
pub segment_frames: DecodedSegmentFrames,
pub uniforms: ProjectUniforms,
pub cursor: Arc<CursorEvents>,
}
pub enum EditorFrameOutput {
Rgba(RenderedFrame),
Nv12(Nv12RenderedFrame),
#[cfg(target_os = "macos")]
Surface(SurfaceFrame),
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum EditorFrameFormat {
#[default]
Rgba,
#[cfg(target_os = "macos")]
BgraSurface,
}
pub type RendererLayersReceiver = oneshot::Receiver<RendererLayers>;
pub type EditorFrameCallback = Box<dyn FnMut(EditorFrameOutput, FrameLayout) + Send>;
pub struct Renderer {
rx: mpsc::Receiver<RendererMessage>,
frame_cb: EditorFrameCallback,
render_constants: Arc<RenderVideoConstants>,
layers_rx: RendererLayersReceiver,
telemetry: Option<PlaybackTelemetry>,
output_format: EditorFrameFormat,
}
pub struct RendererHandle {
tx: mpsc::Sender<RendererMessage>,
telemetry: Option<PlaybackTelemetry>,
}
pub fn start_renderer_layers_creation(
render_constants: &Arc<RenderVideoConstants>,
project: &ProjectConfiguration,
) -> RendererLayersReceiver {
let (layers_tx, layers_rx) = oneshot::channel();
let constants = render_constants.clone();
let use_svg = project.cursor.use_svg;
let cursor_type = project.cursor.cursor_type().clone();
if let Err(error) = std::thread::Builder::new()
.name("renderer-layers-init".into())
.spawn(move || {
let mut layers = RendererLayers::new_with_options(
&constants.device,
&constants.queue,
constants.is_software_adapter,
);
layers.prepare_for_video_dimensions(
&constants.device,
constants.options.screen_size.x,
constants.options.screen_size.y,
constants.options.camera_size.map(|size| size.x),
constants.options.camera_size.map(|size| size.y),
);
layers.preload_cursor_assets(&constants, use_svg, &cursor_type);
let _ = layers_tx.send(layers);
})
{
tracing::warn!(%error, "renderer layer initialization thread unavailable; initializing inline");
}
layers_rx
}
pub async fn finish_renderer_layers_creation(
layers_rx: RendererLayersReceiver,
) -> RendererLayersReceiver {
let (layers_tx, ready_layers_rx) = oneshot::channel();
if let Ok(layers) = layers_rx.await {
let _ = layers_tx.send(layers);
}
ready_layers_rx
}
impl Renderer {
pub fn spawn(
render_constants: Arc<RenderVideoConstants>,
frame_cb: EditorFrameCallback,
layers_rx: RendererLayersReceiver,
) -> Result<RendererHandle, String> {
Self::spawn_with_format_and_telemetry(
render_constants,
frame_cb,
layers_rx,
EditorFrameFormat::Rgba,
None,
)
}
pub fn spawn_with_format(
render_constants: Arc<RenderVideoConstants>,
frame_cb: EditorFrameCallback,
layers_rx: RendererLayersReceiver,
output_format: EditorFrameFormat,
) -> Result<RendererHandle, String> {
Self::spawn_with_format_and_telemetry(
render_constants,
frame_cb,
layers_rx,
output_format,
None,
)
}
pub fn spawn_with_telemetry(
render_constants: Arc<RenderVideoConstants>,
frame_cb: EditorFrameCallback,
layers_rx: RendererLayersReceiver,
telemetry: Option<PlaybackTelemetry>,
) -> Result<RendererHandle, String> {
Self::spawn_with_format_and_telemetry(
render_constants,
frame_cb,
layers_rx,
EditorFrameFormat::Rgba,
telemetry,
)
}
pub fn spawn_with_format_and_telemetry(
render_constants: Arc<RenderVideoConstants>,
frame_cb: EditorFrameCallback,
layers_rx: RendererLayersReceiver,
output_format: EditorFrameFormat,
telemetry: Option<PlaybackTelemetry>,
) -> Result<RendererHandle, String> {
let (tx, rx) = mpsc::channel(64);
let this = Self {
rx,
frame_cb,
render_constants,
layers_rx,
telemetry: telemetry.clone(),
output_format,
};
tokio::spawn(this.run());
Ok(RendererHandle { tx, telemetry })
}
async fn run(self) {
let Renderer {
mut rx,
mut frame_cb,
render_constants,
layers_rx,
telemetry,
output_format,
} = self;
let mut frame_renderer = FrameRenderer::new(&render_constants);
let mut layers = match layers_rx.await {
Ok(layers) => layers,
Err(_) => {
tracing::warn!("Failed to receive pre-created renderer layers, creating inline");
let mut layers = RendererLayers::new_with_options(
&render_constants.device,
&render_constants.queue,
render_constants.is_software_adapter,
);
layers.prepare_for_video_dimensions(
&render_constants.device,
render_constants.options.screen_size.x,
render_constants.options.screen_size.y,
render_constants.options.camera_size.map(|size| size.x),
render_constants.options.camera_size.map(|size| size.y),
);
let project = render_constants.recording_meta.project_config();
layers.preload_cursor_assets(
&render_constants,
project.cursor.use_svg,
project.cursor.cursor_type(),
);
layers
}
};
struct PendingFrame {
input: PendingRenderInput,
finished: oneshot::Sender<bool>,
queued_at: Instant,
realtime: bool,
}
enum PendingRenderInput {
Single(RendererTransitionInput),
Transition {
outgoing: RendererTransitionInput,
incoming: Box<RendererTransitionInput>,
kind: ClipTransitionType,
progress: f32,
},
}
impl PendingRenderInput {
fn uniforms(&self) -> &ProjectUniforms {
match self {
Self::Single(input) => &input.uniforms,
Self::Transition { incoming, .. } => &incoming.uniforms,
}
}
}
let mut pending_frame: Option<PendingFrame> = None;
loop {
let frame_to_render = if let Some(pending) = pending_frame.take() {
Some(pending)
} else {
match rx.recv().await {
Some(RendererMessage::PrepareOutputSize { width, height }) => {
Self::prepare_output_size(&telemetry, &mut frame_renderer, width, height);
continue;
}
Some(RendererMessage::RenderThumbnail { input, finished }) => {
let result = frame_renderer
.render_immediate(
input.segment_frames,
input.uniforms,
&input.cursor,
true,
&mut layers,
)
.await
.map_err(|error| error.to_string());
let _ = finished.send(result);
continue;
}
Some(RendererMessage::RenderFrame {
segment_frames,
uniforms,
finished,
cursor,
queued_at,
realtime,
}) => Some(PendingFrame {
input: PendingRenderInput::Single(RendererTransitionInput {
segment_frames,
uniforms,
cursor,
}),
finished,
queued_at,
realtime,
}),
Some(RendererMessage::RenderTransition {
outgoing,
incoming,
kind,
progress,
finished,
queued_at,
realtime,
}) => Some(PendingFrame {
input: PendingRenderInput::Transition {
outgoing,
incoming: Box::new(incoming),
kind,
progress,
},
finished,
queued_at,
realtime,
}),
Some(RendererMessage::Stop { finished }) => {
let _ = finished.send(());
return;
}
None => return,
}
};
let Some(mut current) = frame_to_render else {
continue;
};
let mut drained_count = 0u32;
let queue_drain_start = Instant::now();
while let Ok(msg) = rx.try_recv() {
match msg {
RendererMessage::RenderThumbnail { input, finished } => {
let result = frame_renderer
.render_immediate(
input.segment_frames,
input.uniforms,
&input.cursor,
true,
&mut layers,
)
.await
.map_err(|error| error.to_string());
let _ = finished.send(result);
}
RendererMessage::PrepareOutputSize { width, height } => {
Self::prepare_output_size(&telemetry, &mut frame_renderer, width, height);
}
RendererMessage::RenderFrame {
segment_frames,
uniforms,
finished,
cursor,
queued_at,
realtime,
} => {
let dropped_frame_number = current.input.uniforms().frame_number;
let replacement_frame_number = uniforms.frame_number;
let _ = current.finished.send(false);
if let Some(telemetry) = &telemetry {
telemetry.emit(PlaybackTelemetryEvent::RendererDropped {
frame_number: dropped_frame_number,
replacement_frame_number,
});
}
current = PendingFrame {
input: PendingRenderInput::Single(RendererTransitionInput {
segment_frames,
uniforms,
cursor,
}),
finished,
queued_at,
realtime,
};
drained_count += 1;
}
RendererMessage::RenderTransition {
outgoing,
incoming,
kind,
progress,
finished,
queued_at,
realtime,
} => {
let dropped_frame_number = current.input.uniforms().frame_number;
let replacement_frame_number = incoming.uniforms.frame_number;
let _ = current.finished.send(false);
if let Some(telemetry) = &telemetry {
telemetry.emit(PlaybackTelemetryEvent::RendererDropped {
frame_number: dropped_frame_number,
replacement_frame_number,
});
}
current = PendingFrame {
input: PendingRenderInput::Transition {
outgoing,
incoming: Box::new(incoming),
kind,
progress,
},
finished,
queued_at,
realtime,
};
drained_count += 1;
}
RendererMessage::Stop { finished } => {
let _ = current.finished.send(false);
let _ = finished.send(());
return;
}
}
if queue_drain_start.elapsed().as_millis() > 5 {
break;
}
}
let queue_wait = current.queued_at.elapsed();
let drain_duration = queue_drain_start.elapsed();
let flush_start = Instant::now();
if drained_count > 0 {
let _ = frame_renderer.flush_pipeline().await;
}
let flush_duration = if drained_count > 0 {
flush_start.elapsed()
} else {
std::time::Duration::ZERO
};
// Playback frames composite with the newest finished camera mask;
// seeks and paused redraws wait for their own, since nothing
// redraws a still frame when a later mask lands.
layers.set_realtime_camera_effects(current.realtime);
let render_start = Instant::now();
let input_frame_number = current.input.uniforms().frame_number;
let frame_layout = current.input.uniforms().frame_layout();
let render_result = match (output_format, current.input) {
(EditorFrameFormat::Rgba, PendingRenderInput::Single(input)) => frame_renderer
.render_immediate_with_timings(
input.segment_frames,
input.uniforms,
&input.cursor,
true,
&mut layers,
)
.await
.map(|(frame, timings)| {
(
EditorFrameOutput::Rgba(frame),
PlaybackRenderOutputFormat::Rgba,
timings,
)
}),
(
EditorFrameFormat::Rgba,
PendingRenderInput::Transition {
outgoing,
incoming,
kind,
progress,
},
) => frame_renderer
.render_transition_immediate(
TransitionRenderInput {
segment_frames: outgoing.segment_frames,
uniforms: outgoing.uniforms,
cursor: &outgoing.cursor,
render_display: true,
},
TransitionRenderInput {
segment_frames: incoming.segment_frames,
uniforms: incoming.uniforms,
cursor: &incoming.cursor,
render_display: true,
},
kind,
progress,
&mut layers,
)
.await
.map(|frame| {
(
EditorFrameOutput::Rgba(frame),
PlaybackRenderOutputFormat::Rgba,
FrameRenderStageTimings::default(),
)
}),
#[cfg(target_os = "macos")]
(EditorFrameFormat::BgraSurface, PendingRenderInput::Single(input)) => {
frame_renderer
.render_immediate_bgra_surface(
input.segment_frames,
input.uniforms,
&input.cursor,
true,
&mut layers,
)
.await
.map(|frame| {
(
EditorFrameOutput::Surface(frame),
PlaybackRenderOutputFormat::Bgra,
FrameRenderStageTimings::default(),
)
})
}
#[cfg(target_os = "macos")]
(
EditorFrameFormat::BgraSurface,
PendingRenderInput::Transition {
outgoing,
incoming,
kind,
progress,
},
) => frame_renderer
.render_transition_bgra_surface(
TransitionRenderInput {
segment_frames: outgoing.segment_frames,
uniforms: outgoing.uniforms,
cursor: &outgoing.cursor,
render_display: true,
},
TransitionRenderInput {
segment_frames: incoming.segment_frames,
uniforms: incoming.uniforms,
cursor: &incoming.cursor,
render_display: true,
},
kind,
progress,
&mut layers,
)
.await
.map(|frame| {
(
EditorFrameOutput::Surface(frame),
PlaybackRenderOutputFormat::Bgra,
FrameRenderStageTimings::default(),
)
}),
};
match render_result {
Ok((frame, output_format, render_stage_timings)) => {
let render_duration = render_start.elapsed();
let frame_number = match &frame {
EditorFrameOutput::Rgba(frame) => frame.frame_number,
EditorFrameOutput::Nv12(frame) => frame.frame_number,
#[cfg(target_os = "macos")]
EditorFrameOutput::Surface(frame) => frame.frame_number,
};
let callback_start = Instant::now();
(frame_cb)(frame, frame_layout);
let callback_duration = callback_start.elapsed();
if let Some(telemetry) = &telemetry {
telemetry.emit(PlaybackTelemetryEvent::RendererFrame {
frame_number,
input_frame_number,
queue_wait,
drain_duration,
flush_duration,
render_duration,
render_stage_timings: Box::new(render_stage_timings),
callback_duration,
drained_count,
output_format,
});
}
let _ = current.finished.send(true);
}
Err(e) => {
tracing::error!(error = %e, "Failed to render frame in editor");
let _ = current.finished.send(false);
}
}
}
}
fn prepare_output_size(
telemetry: &Option<PlaybackTelemetry>,
frame_renderer: &mut FrameRenderer<'_>,
width: u32,
height: u32,
) {
let start = Instant::now();
frame_renderer.prepare_output_size(width, height);
if let Some(telemetry) = telemetry {
telemetry.emit(PlaybackTelemetryEvent::RendererPrepared {
output_width: width,
output_height: height,
duration: start.elapsed(),
});
}
}
}
impl RendererHandle {
pub(crate) async fn render_thumbnail(
&self,
input: RendererTransitionInput,
) -> Result<RenderedFrame, String> {
let (finished, result) = oneshot::channel();
self.tx
.send(RendererMessage::RenderThumbnail { input, finished })
.await
.map_err(|_| "Thumbnail renderer stopped".to_string())?;
result
.await
.map_err(|_| "Thumbnail render cancelled".to_string())?
}
pub fn prepare_output_size(&self, width: u32, height: u32) {
let _ = self
.tx
.try_send(RendererMessage::PrepareOutputSize { width, height });
}
pub fn render_frame(
&self,
segment_frames: DecodedSegmentFrames,
uniforms: ProjectUniforms,
cursor: Arc<CursorEvents>,
) {
let (finished_tx, _finished_rx) = oneshot::channel();
let frame_number = uniforms.frame_number;
if self
.tx
.try_send(RendererMessage::RenderFrame {
segment_frames,
uniforms,
finished: finished_tx,
cursor,
queued_at: Instant::now(),
realtime: true,
})
.is_err()
&& let Some(telemetry) = &self.telemetry
{
telemetry.emit(PlaybackTelemetryEvent::RendererSendFailed { frame_number });
}
}
pub fn render_transition_frame(
&self,
outgoing: RendererTransitionInput,
incoming: RendererTransitionInput,
kind: ClipTransitionType,
progress: f32,
) {
let (finished_tx, _finished_rx) = oneshot::channel();
let frame_number = incoming.uniforms.frame_number;
if self
.tx
.try_send(RendererMessage::RenderTransition {
outgoing,
incoming,
kind,
progress,
finished: finished_tx,
queued_at: Instant::now(),
realtime: true,
})
.is_err()
&& let Some(telemetry) = &self.telemetry
{
telemetry.emit(PlaybackTelemetryEvent::RendererSendFailed { frame_number });
}
}
pub fn render_frame_blocking(
&self,
segment_frames: DecodedSegmentFrames,
uniforms: ProjectUniforms,
cursor: Arc<CursorEvents>,
) {
let (finished_tx, _finished_rx) = oneshot::channel();
let frame_number = uniforms.frame_number;
let msg = RendererMessage::RenderFrame {
segment_frames,
uniforms,
finished: finished_tx,
cursor,
queued_at: Instant::now(),
realtime: false,
};
if self.tx.blocking_send(msg).is_err()
&& let Some(telemetry) = &self.telemetry
{
telemetry.emit(PlaybackTelemetryEvent::RendererSendFailed { frame_number });
}
}
pub async fn render_frame_confirmed(
&self,
segment_frames: DecodedSegmentFrames,
uniforms: ProjectUniforms,
cursor: Arc<CursorEvents>,
) -> bool {
let (finished_tx, finished_rx) = oneshot::channel();
let frame_number = uniforms.frame_number;
let msg = RendererMessage::RenderFrame {
segment_frames,
uniforms,
finished: finished_tx,
cursor,
queued_at: Instant::now(),
realtime: false,
};
if self.tx.send(msg).await.is_err() {
if let Some(telemetry) = &self.telemetry {
telemetry.emit(PlaybackTelemetryEvent::RendererSendFailed { frame_number });
}
return false;
}
finished_rx.await.unwrap_or(false)
}
pub async fn render_transition_frame_confirmed(
&self,
outgoing: RendererTransitionInput,
incoming: RendererTransitionInput,
kind: ClipTransitionType,
progress: f32,
) -> bool {
let (finished_tx, finished_rx) = oneshot::channel();
let frame_number = incoming.uniforms.frame_number;
let message = RendererMessage::RenderTransition {
outgoing,
incoming,
kind,
progress,
finished: finished_tx,
queued_at: Instant::now(),
realtime: false,
};
if self.tx.send(message).await.is_err() {
if let Some(telemetry) = &self.telemetry {
telemetry.emit(PlaybackTelemetryEvent::RendererSendFailed { frame_number });
}
return false;
}
finished_rx.await.unwrap_or(false)
}
pub fn render_frame_wait(
&self,
segment_frames: DecodedSegmentFrames,
uniforms: ProjectUniforms,
cursor: Arc<CursorEvents>,
) -> bool {
let (finished_tx, finished_rx) = oneshot::channel();
let frame_number = uniforms.frame_number;
let msg = RendererMessage::RenderFrame {
segment_frames,
uniforms,
finished: finished_tx,
cursor,
queued_at: Instant::now(),
realtime: false,
};
if self.tx.blocking_send(msg).is_err() {
if let Some(telemetry) = &self.telemetry {
telemetry.emit(PlaybackTelemetryEvent::RendererSendFailed { frame_number });
}
return false;
}
finished_rx.blocking_recv().unwrap_or(false)
}
pub fn render_transition_frame_wait(
&self,
outgoing: RendererTransitionInput,
incoming: RendererTransitionInput,
kind: ClipTransitionType,
progress: f32,
) -> bool {
let (finished_tx, finished_rx) = oneshot::channel();
let frame_number = incoming.uniforms.frame_number;
let message = RendererMessage::RenderTransition {
outgoing,
incoming,
kind,
progress,
finished: finished_tx,
queued_at: Instant::now(),
realtime: false,
};
if self.tx.blocking_send(message).is_err() {
if let Some(telemetry) = &self.telemetry {
telemetry.emit(PlaybackTelemetryEvent::RendererSendFailed { frame_number });
}
return false;
}
finished_rx.blocking_recv().unwrap_or(false)
}
pub async fn stop(&self) {
let (tx, rx) = oneshot::channel();
if self
.tx
.send(RendererMessage::Stop { finished: tx })
.await
.is_err()
{
tracing::debug!("Renderer stop message skipped because renderer task already stopped");
}
let _ = rx.await;
}
}