Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

23.3 KB
Raw Blame
use std::{path::PathBuf, sync::Arc};

use base64::{Engine, engine::general_purpose::STANDARD};
use cap_editor::EditorInstance;
use cap_project::{CursorEvents, ProjectConfiguration, RecordingMeta, TimelineFrameMapping, XY};
use cap_rendering::{
    FrameRenderer, PrecomputedCursorTimeline, ProjectUniforms, RecordingSegmentDecoders,
    RenderVideoConstants, RenderedFrame, RendererLayers, TransitionRenderInput,
    ZoomTransformTimeline,
};
use image::{
    Rgba,
    codecs::jpeg::JpegEncoder,
    flat::{FlatSamples, SampleLayout},
};
use serde::{Deserialize, Serialize};

use crate::{
    ExportError, ExporterBase, ExporterBuilder, make_cursor_only_project,
    synthesize_default_timeline,
};

#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct ExportPreviewSettings {
    pub fps: u32,
    pub resolution_base: XY<u32>,
    pub compression_bpp: f32,
    #[serde(default)]
    pub cursor_only: bool,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct ExportPreviewResult {
    pub jpeg_base64: String,
    pub estimated_size_mb: f64,
    pub actual_width: u32,
    pub actual_height: u32,
    pub frame_render_time_ms: f64,
    pub total_frames: u32,
}

pub async fn render_preview(
    project_path: PathBuf,
    frame_time: f64,
    settings: ExportPreviewSettings,
    force_ffmpeg_decoder: bool,
) -> Result<ExportPreviewResult, ExportError> {
    let mut exporter_builder =
        ExporterBase::builder(project_path.clone()).with_force_ffmpeg_decoder(force_ffmpeg_decoder);

    if settings.cursor_only {
        let meta = RecordingMeta::load_for_project(&project_path)
            .map_err(|e| ExportError::Other(format!("Failed to load recording meta: {e}")))?;
        exporter_builder =
            exporter_builder.with_config(make_cursor_only_project(meta.project_config()));
    }

    let exporter_base = exporter_builder
        .build()
        .await
        .map_err(|e| ExportError::Other(format!("Exporter build error: {e}")))?;

    render_preview_with_base(exporter_base, frame_time, settings).await
}

fn preview_builder_with_config(
    project_path: PathBuf,
    project_config: ProjectConfiguration,
    settings: ExportPreviewSettings,
    force_ffmpeg_decoder: bool,
) -> ExporterBuilder {
    let project_config = if settings.cursor_only {
        make_cursor_only_project(project_config)
    } else {
        project_config
    };
    ExporterBase::builder(project_path)
        .with_config(project_config)
        .with_force_ffmpeg_decoder(force_ffmpeg_decoder)
}

pub async fn render_preview_with_config(
    project_path: PathBuf,
    project_config: ProjectConfiguration,
    frame_time: f64,
    settings: ExportPreviewSettings,
    force_ffmpeg_decoder: bool,
) -> Result<ExportPreviewResult, ExportError> {
    let exporter_base =
        preview_builder_with_config(project_path, project_config, settings, force_ffmpeg_decoder)
            .build()
            .await
            .map_err(|error| ExportError::Other(format!("Exporter build error: {error}")))?;
    render_preview_with_base(exporter_base, frame_time, settings).await
}

/// Everything a preview frame needs, borrowed from whichever owner already
/// holds the decoders and GPU pipelines.
struct PreviewSource<'a> {
    project_config: &'a ProjectConfiguration,
    render_constants: &'a RenderVideoConstants,
    segments: Vec<PreviewSegment<'a>>,
    total_duration: f64,
    layers_cache: Option<&'a std::sync::Mutex<Option<RendererLayers>>>,
}

struct PreviewSegment<'a> {
    decoders: &'a RecordingSegmentDecoders,
    cursor: &'a CursorEvents,
}

async fn render_preview_with_base(
    exporter_base: ExporterBase,
    frame_time: f64,
    settings: ExportPreviewSettings,
) -> Result<ExportPreviewResult, ExportError> {
    let total_duration = cap_rendering::get_duration(
        &exporter_base.recordings,
        &exporter_base.recording_meta,
        &exporter_base.studio_meta,
        &exporter_base.project_config,
    );
    let source = PreviewSource {
        project_config: &exporter_base.project_config,
        render_constants: &exporter_base.render_constants,
        segments: exporter_base
            .segments
            .iter()
            .map(|segment| PreviewSegment {
                decoders: &segment.decoders,
                cursor: &segment.cursor,
            })
            .collect(),
        total_duration,
        layers_cache: None,
    };
    render_preview_frame(source, frame_time, settings).await
}

/// Render the preview through an open editor's decoders and GPU pipelines
/// instead of building a fresh exporter (a new wgpu device, every shader, and
/// a cold decoder per recording segment) for each frame. This is what makes the
/// export page's preview respond in one frame's worth of work rather than
/// seconds, and it is the same path the Tauri editor's fast preview takes.
pub async fn render_preview_with_editor(
    editor: &EditorInstance,
    project_config: ProjectConfiguration,
    frame_time: f64,
    settings: ExportPreviewSettings,
) -> Result<ExportPreviewResult, ExportError> {
    let recording_meta = editor.meta();
    let studio_meta = recording_meta
        .studio_meta()
        .ok_or_else(|| ExportError::Other("Cannot preview non-studio recordings".to_string()))?;
    let mut project_config = project_config;
    if settings.cursor_only {
        project_config = make_cursor_only_project(project_config);
    }
    synthesize_default_timeline(&mut project_config, &editor.recordings);
    cap_project::synchronize_legacy_keyboard(recording_meta, &mut project_config);
    cap_project::synchronize_captions(
        &mut project_config,
        &editor
            .recordings
            .segments
            .iter()
            .map(|segment| segment.display.duration)
            .collect::<Vec<_>>(),
    );
    let total_duration = cap_rendering::get_duration(
        &editor.recordings,
        recording_meta,
        studio_meta,
        &project_config,
    );

    let _pause_prefetch = EditorPreviewGuard::new(editor);
    let source = PreviewSource {
        project_config: &project_config,
        render_constants: &editor.render_constants,
        segments: editor
            .segment_medias
            .iter()
            .map(|segment| PreviewSegment {
                decoders: &segment.decoders,
                cursor: &segment.cursor,
            })
            .collect(),
        total_duration,
        layers_cache: Some(&editor.preview_layers),
    };
    render_preview_frame(source, frame_time, settings).await
}

/// Holds the editor's `export_preview_active` flag so its idle prefetch does
/// not compete with the preview for the decoders.
struct EditorPreviewGuard<'a>(&'a EditorInstance);

impl<'a> EditorPreviewGuard<'a> {
    fn new(editor: &'a EditorInstance) -> Self {
        editor
            .export_preview_active
            .store(true, std::sync::atomic::Ordering::Release);
        Self(editor)
    }
}

impl Drop for EditorPreviewGuard<'_> {
    fn drop(&mut self) {
        self.0
            .export_preview_active
            .store(false, std::sync::atomic::Ordering::Release);
    }
}

async fn render_preview_frame(
    source: PreviewSource<'_>,
    frame_time: f64,
    settings: ExportPreviewSettings,
) -> Result<ExportPreviewResult, ExportError> {
    let PreviewSource {
        project_config,
        render_constants,
        segments,
        total_duration,
        layers_cache,
    } = source;
    let project_config = &Arc::new(project_config.clone());
    let transition_mapping = project_config.timeline.as_ref().and_then(|timeline| {
        if timeline.transitions.is_empty() {
            return None;
        }
        match timeline.get_frame_mapping(frame_time) {
            Some(TimelineFrameMapping::Transition {
                outgoing,
                kind,
                progress,
                ..
            }) => Some((outgoing, kind, progress)),
            _ => None,
        }
    });
    let Some((segment_time, segment)) = project_config.get_segment_time(frame_time) else {
        return Err(ExportError::Other(
            "Frame time is outside video duration".to_string(),
        ));
    };

    let segment_media = segments
        .get(segment.recording_clip as usize)
        .ok_or_else(|| ExportError::Other("Recording clip is unavailable".to_string()))?;
    let clip_config = project_config
        .clips
        .iter()
        .find(|v| v.index == segment.recording_clip);

    let render_start = std::time::Instant::now();

    let segment_frames = segment_media
        .decoders
        .get_frames(
            segment_time as f32,
            project_config.requires_camera() && !settings.cursor_only,
            !settings.cursor_only,
            clip_config.map(|v| v.offsets).unwrap_or_default(),
        )
        .await
        .ok_or_else(|| ExportError::Other("Failed to decode frame".to_string()))?;

    let frame_number = (frame_time * settings.fps as f64).floor() as u32;

    let mut zoom_timeline = ZoomTransformTimeline::from_project_for_clip(
        project_config,
        segment_media.cursor,
        total_duration,
        render_constants.options.screen_size,
        segment.recording_clip,
    );
    zoom_timeline.ensure_precomputed_until((frame_number as f32 + 1.0) / settings.fps as f32);

    let cursor_timeline =
        PrecomputedCursorTimeline::from_project(segment_media.cursor, project_config);
    let uniforms = ProjectUniforms::new_with_precomputed_cursor(
        render_constants,
        project_config,
        frame_number,
        settings.fps,
        settings.resolution_base,
        segment_media.cursor,
        &segment_frames,
        total_duration,
        &zoom_timeline,
        &cursor_timeline,
    );

    let mut frame_renderer = FrameRenderer::new(render_constants);
    let mut layers = layers_cache
        .and_then(|cache| {
            let mut guard = cache.lock().ok()?;
            guard.take()
        })
        .map(|mut cached| {
            cached.reset_frame_state();
            cached
        })
        .unwrap_or_else(|| {
            RendererLayers::new_with_options(
                &render_constants.device,
                &render_constants.queue,
                render_constants.is_software_adapter,
            )
        });

    let frame = if let Some((outgoing, kind, progress)) = transition_mapping {
        let outgoing_media = segments
            .get(outgoing.segment.recording_clip as usize)
            .ok_or_else(|| {
                ExportError::Other("Outgoing recording clip is unavailable".to_string())
            })?;
        let outgoing_offsets = project_config
            .clips
            .iter()
            .find(|clip| clip.index == outgoing.segment.recording_clip)
            .map(|clip| clip.offsets)
            .unwrap_or_default();
        let outgoing_decoders = if outgoing.segment.recording_clip == segment.recording_clip {
            outgoing_media.decoders.transition_side().await
        } else {
            outgoing_media.decoders
        };
        let outgoing_frames = outgoing_decoders
            .get_frames(
                outgoing.source_time as f32,
                project_config.requires_camera() && !settings.cursor_only,
                !settings.cursor_only,
                outgoing_offsets,
            )
            .await
            .ok_or_else(|| ExportError::Other("Failed to decode outgoing frame".to_string()))?;
        let mut outgoing_zoom = ZoomTransformTimeline::from_project_for_outgoing_clip(
            project_config,
            outgoing_media.cursor,
            total_duration,
            render_constants.options.screen_size,
            outgoing.segment.recording_clip,
        );
        outgoing_zoom.ensure_precomputed_until((frame_number as f32 + 1.0) / settings.fps as f32);
        let outgoing_cursor_timeline = if outgoing.segment.recording_clip == segment.recording_clip
        {
            None
        } else {
            Some(PrecomputedCursorTimeline::from_project(
                outgoing_media.cursor,
                project_config,
            ))
        };
        let outgoing_uniforms = ProjectUniforms::new_with_precomputed_cursor(
            render_constants,
            project_config,
            frame_number,
            settings.fps,
            settings.resolution_base,
            outgoing_media.cursor,
            &outgoing_frames,
            total_duration,
            &outgoing_zoom,
            outgoing_cursor_timeline
                .as_ref()
                .unwrap_or(&cursor_timeline),
        );

        frame_renderer
            .render_transition_immediate(
                TransitionRenderInput {
                    segment_frames: outgoing_frames,
                    uniforms: outgoing_uniforms,
                    cursor: outgoing_media.cursor,
                    render_display: !settings.cursor_only,
                },
                TransitionRenderInput {
                    segment_frames,
                    uniforms,
                    cursor: segment_media.cursor,
                    render_display: !settings.cursor_only,
                },
                kind,
                progress as f32,
                &mut layers,
            )
            .await?
    } else {
        frame_renderer
            .render_immediate(
                segment_frames,
                uniforms,
                segment_media.cursor,
                !settings.cursor_only,
                &mut layers,
            )
            .await?
    };

    let frame_render_time_ms = render_start.elapsed().as_secs_f64() * 1000.0;
    let width = frame.width;
    let height = frame.height;

    let jpeg_buffer = encode_preview_jpeg(&frame, bpp_to_jpeg_quality(settings.compression_bpp))?;
    drop(frame);

    let duration_seconds = total_duration;
    let fps_f64 = settings.fps as f64;
    let total_frames = (duration_seconds * fps_f64).ceil() as u32;
    let total_pixels = (settings.resolution_base.x * settings.resolution_base.y) as f64;
    let estimated_size_mb = if settings.cursor_only {
        let total_frames_f64 = (duration_seconds * fps_f64).ceil();
        estimate_cursor_only_size_mb(total_pixels, total_frames_f64)
    } else {
        let effective_fps = ((fps_f64 - 30.0).max(0.0) * 0.6) + fps_f64.min(30.0);
        let video_bitrate = total_pixels * settings.compression_bpp as f64 * effective_fps;
        let audio_bitrate = 192_000.0;
        let total_bitrate = video_bitrate + audio_bitrate;
        let encoder_efficiency = 0.5;
        (total_bitrate * encoder_efficiency * duration_seconds) / (8.0 * 1024.0 * 1024.0)
    };

    if let Some(cache) = layers_cache
        && let Ok(mut guard) = cache.lock()
    {
        *guard = Some(layers);
    }

    Ok(ExportPreviewResult {
        jpeg_base64: STANDARD.encode(&jpeg_buffer),
        estimated_size_mb,
        actual_width: width,
        actual_height: height,
        frame_render_time_ms,
        total_frames,
    })
}

fn estimate_cursor_only_size_mb(total_pixels: f64, total_frames: f64) -> f64 {
    let bytes_per_frame = total_pixels * 0.4;
    (bytes_per_frame * total_frames) / (1024.0 * 1024.0)
}

fn bpp_to_jpeg_quality(bpp: f32) -> u8 {
    ((bpp - 0.04) / (0.3 - 0.04) * (95.0 - 40.0) + 40.0).clamp(40.0, 95.0) as u8
}

fn encode_preview_jpeg(frame: &RenderedFrame, quality: u8) -> Result<Vec<u8>, ExportError> {
    let width = usize::try_from(frame.width)
        .map_err(|_| ExportError::Other("Preview frame width is too large".to_string()))?;
    let height = usize::try_from(frame.height)
        .map_err(|_| ExportError::Other("Preview frame height is too large".to_string()))?;
    if width == 0 || height == 0 {
        return Err(ExportError::Other(
            "Preview frame dimensions must be non-zero".to_string(),
        ));
    }

    let row_bytes = width
        .checked_mul(4)
        .ok_or_else(|| ExportError::Other("Preview frame row is too large".to_string()))?;
    let height_stride = usize::try_from(frame.padded_bytes_per_row)
        .map_err(|_| ExportError::Other("Preview frame row stride is too large".to_string()))?;
    if height_stride < row_bytes {
        return Err(ExportError::Other("Preview frame rows overlap".to_string()));
    }
    let required_len = height_stride
        .checked_mul(height - 1)
        .and_then(|offset| offset.checked_add(row_bytes))
        .ok_or_else(|| ExportError::Other("Preview frame buffer is too large".to_string()))?;
    if frame.data.len() < required_len {
        return Err(ExportError::Other(
            "Preview frame buffer is shorter than its layout".to_string(),
        ));
    }

    let samples = FlatSamples {
        samples: frame.data.as_ref().as_slice(),
        layout: SampleLayout {
            channels: 4,
            channel_stride: 1,
            width: frame.width,
            width_stride: 4,
            height: frame.height,
            height_stride,
        },
        color_hint: None,
    };
    let view = samples
        .as_view::<Rgba<u8>>()
        .map_err(|e| ExportError::Other(format!("Invalid preview frame layout: {e}")))?;

    let mut jpeg_buffer = Vec::new();
    let mut encoder = JpegEncoder::new_with_quality(&mut jpeg_buffer, quality);
    encoder
        .encode_image(&view)
        .map_err(|e| ExportError::Other(format!("Failed to encode JPEG: {e}")))?;
    Ok(jpeg_buffer)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn frame(width: u32, height: u32, stride: usize, mut state: u64) -> RenderedFrame {
        let mut data = vec![0xa5; stride * height as usize];
        for y in 0..height as usize {
            for x in 0..width as usize {
                state = state
                    .wrapping_mul(6364136223846793005)
                    .wrapping_add(1442695040888963407);
                let offset = y * stride + x * 4;
                data[offset] = (state >> 56) as u8;
                data[offset + 1] = (state >> 48) as u8;
                data[offset + 2] = (state >> 40) as u8;
                data[offset + 3] = (state >> 32) as u8;
            }
        }
        RenderedFrame {
            data: Arc::new(data),
            width,
            height,
            padded_bytes_per_row: stride as u32,
            frame_number: 0,
            target_time_ns: 0,
        }
    }

    fn legacy_jpeg(frame: &RenderedFrame, quality: u8) -> Vec<u8> {
        let rgb_data: Vec<u8> = frame
            .data
            .chunks(frame.padded_bytes_per_row as usize)
            .flat_map(|row| {
                row[..(frame.width * 4) as usize]
                    .chunks(4)
                    .flat_map(|chunk| [chunk[0], chunk[1], chunk[2]])
            })
            .collect();
        let mut jpeg_buffer = Vec::new();
        let mut encoder = JpegEncoder::new_with_quality(&mut jpeg_buffer, quality);
        encoder
            .encode(
                &rgb_data,
                frame.width,
                frame.height,
                image::ExtendedColorType::Rgb8,
            )
            .unwrap();
        jpeg_buffer
    }

    #[test]
    fn flat_samples_jpeg_matches_legacy_bytes() {
        for (width, height, stride) in [(1, 1, 4), (13, 9, 64)] {
            let frame = frame(width, height, stride, 0x1234_5678_9abc_def0);
            for quality in [40, 70, 95] {
                assert_eq!(
                    encode_preview_jpeg(&frame, quality).unwrap(),
                    legacy_jpeg(&frame, quality),
                    "{width}x{height} stride={stride} quality={quality}"
                );
            }
        }

        let padded = frame(13, 9, 64, 0x1234_5678_9abc_def0);
        let required_len = 64 * (9 - 1) + 13 * 4;
        let mut without_trailing_padding = padded.clone();
        without_trailing_padding.data = Arc::new(padded.data[..required_len].to_vec());
        for quality in [40, 70, 95] {
            assert_eq!(
                encode_preview_jpeg(&without_trailing_padding, quality).unwrap(),
                legacy_jpeg(&without_trailing_padding, quality),
                "13x9 stride=64 without trailing padding quality={quality}"
            );
        }
    }

    #[test]
    fn rejects_overlapping_rows_and_short_buffers() {
        let mut overlapping = frame(13, 9, 64, 1);
        overlapping.padded_bytes_per_row = 51;
        assert!(encode_preview_jpeg(&overlapping, 70).is_err());

        let valid = frame(13, 9, 64, 2);
        let mut short = valid.clone();
        let required_len = 64 * (9 - 1) + 13 * 4;
        short.data = Arc::new(vec![0; required_len - 1]);
        assert!(encode_preview_jpeg(&short, 70).is_err());

        let mut empty = valid;
        empty.width = 0;
        assert!(encode_preview_jpeg(&empty, 70).is_err());
    }
}

#[cfg(test)]
mod subtitle_preview_tests {
    use super::*;

    #[test]
    fn immediate_export_preview_keeps_enabled_captions_regardless_of_legacy_export_toggle() {
        let temp = tempfile::tempdir().unwrap();
        let path = temp.path().join("project-config.json");
        for export in [false, true] {
            let current = ProjectConfiguration {
                captions: Some(cap_project::CaptionsData {
                    settings: cap_project::CaptionSettings {
                        enabled: true,
                        export_with_subtitles: export,
                        ..Default::default()
                    },
                    ..Default::default()
                }),
                ..Default::default()
            };
            let original = serde_json::to_value(&current).unwrap();
            let mut stale = current.clone();
            stale
                .captions
                .as_mut()
                .unwrap()
                .settings
                .export_with_subtitles = !export;
            let stale_bytes = serde_json::to_vec(&stale).unwrap();
            std::fs::write(&path, &stale_bytes).unwrap();
            let mut builder = preview_builder_with_config(
                temp.path().to_path_buf(),
                current.clone(),
                ExportPreviewSettings {
                    fps: 30,
                    resolution_base: XY::new(1280, 720),
                    compression_bpp: 0.15,
                    cursor_only: false,
                },
                true,
            );
            let preview = builder.load_project_config().unwrap();
            assert!(preview.captions.as_ref().unwrap().settings.enabled);
            assert_eq!(
                preview.captions.unwrap().settings.export_with_subtitles,
                export
            );
            assert_eq!(std::fs::read(&path).unwrap(), stale_bytes);
            assert_eq!(serde_json::to_value(current).unwrap(), original);
            assert!(builder.force_ffmpeg_decoder);
        }
    }

    #[test]
    fn current_config_export_preview_preserves_cursor_only_override() {
        let current = ProjectConfiguration {
            captions: Some(cap_project::CaptionsData {
                settings: cap_project::CaptionSettings {
                    enabled: true,
                    export_with_subtitles: true,
                    ..Default::default()
                },
                ..Default::default()
            }),
            ..Default::default()
        };
        let mut builder = preview_builder_with_config(
            PathBuf::from("unused-current-config-project"),
            current,
            ExportPreviewSettings {
                fps: 30,
                resolution_base: XY::new(1280, 720),
                compression_bpp: 0.15,
                cursor_only: true,
            },
            false,
        );
        assert!(builder.load_project_config().unwrap().captions.is_none());
    }
}