Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(rendering): fit imported videos to the output aspect ratio
Aditya Garud19de9e59167a
diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rsindex 8e95cd3b6..59c791271 100644--- a/crates/rendering/src/lib.rs+++ b/crates/rendering/src/lib.rs@@ -3545,7 +3545,10 @@ impl ProjectUniforms {project: &ProjectConfiguration,resolution_base: XY<u32>,) -> Coord<FrameSpace> {- Coord::new(Self::display_layout(options, project, resolution_base).content_offset)+ Coord::new(+ Self::display_layout(options, project, resolution_base, options.screen_size)+ .content_offset,+ )}/// Placement of the display card in output-frame pixels.@@ -3561,10 +3564,23 @@ impl ProjectUniforms {options: &RenderOptions,project: &ProjectConfiguration,resolution_base: XY<u32>,+ source_size: XY<u32>,) -> DisplayLayout {- let base = Self::display_base_offset(options, project, resolution_base).coord;+ let aspect = Self::display_source_aspect(options, project, source_size);+ let base = Self::display_base_offset(options, project, resolution_base, aspect).coord;let output_size = Self::get_output_size(options, project, resolution_base);- Self::display_layout_in_box(options, project, output_size, base)+ Self::display_layout_in_box(project, output_size, base, aspect)+ }++ fn display_source_aspect(+ options: &RenderOptions,+ project: &ProjectConfiguration,+ source_size: XY<u32>,+ ) -> f64 {+ let crop_size = Self::get_crop(options, project).size.map(f64::from);+ let source_scale = source_size.map(f64::from) / options.screen_size.map(f64::from);+ let crop_size = crop_size * source_scale;+ (crop_size.x / crop_size.y.max(f64::EPSILON)).max(f64::EPSILON)}fn styled_display_layout(@@ -3572,22 +3588,25 @@ impl ProjectUniforms {project: &ProjectConfiguration,output_size: (u32, u32),resolution_base: XY<u32>,+ source_size: XY<u32>,) -> DisplayLayout {let canvas = XY::new(output_size.0 as f64, output_size.1 as f64);let styled_output = Self::get_output_size(options, project, resolution_base);let styled_canvas = XY::new(styled_output.0 as f64, styled_output.1 as f64);- let styled_offset = Self::display_base_offset(options, project, resolution_base).coord;+ let aspect = Self::display_source_aspect(options, project, source_size);+ let styled_offset =+ Self::display_base_offset(options, project, resolution_base, aspect).coord;let content = (styled_canvas - styled_offset) - styled_offset;let scale = (canvas.x / styled_canvas.x).min(canvas.y / styled_canvas.y);let base = (canvas - content * scale) / 2.0;- Self::display_layout_in_box(options, project, output_size, base)+ Self::display_layout_in_box(project, output_size, base, aspect)}fn display_layout_in_box(- options: &RenderOptions,project: &ProjectConfiguration,output_size: (u32, u32),base: XY<f64>,+ aspect: f64,) -> DisplayLayout {let output_size = XY::new(output_size.0 as f64, output_size.1 as f64);// Same op order as the legacy display_size (end - offset - offset) so@@ -3600,8 +3619,6 @@ impl ProjectUniforms {(base, box_size, base, box_size)} else {let insets = frame_chrome::chrome_insets(style);- let crop = Self::get_crop(options, project);- let aspect = (crop.size.x as f64 / f64::from(crop.size.y.max(1))).max(f64::EPSILON);// Insets are fractions of the content height; solve for the// largest content that keeps the outer card inside the box.let outer_w_per_h = aspect + insets.left + insets.right;@@ -3643,6 +3660,7 @@ impl ProjectUniforms {options: &RenderOptions,project: &ProjectConfiguration,resolution_base: XY<u32>,+ cropped_aspect: f64,) -> Coord<FrameSpace> {let output_size = Self::get_output_size(options, project, resolution_base);let output_size = XY::new(output_size.0 as f64, output_size.1 as f64);@@ -3673,8 +3691,6 @@ impl ProjectUniforms {let cropped_size = crop_end.coord - crop_start.coord;- let cropped_aspect = cropped_size.x / cropped_size.y;-let padding = {let padding_factor = project.background.padding / 100.0 * SCREEN_MAX_PADDING;let crop_basis = f64::max(cropped_size.x, cropped_size.y);@@ -3719,7 +3735,10 @@ impl ProjectUniforms {project: &ProjectConfiguration,resolution_base: XY<u32>,) -> Coord<FrameSpace> {- Coord::new(Self::display_layout(options, project, resolution_base).content_size)+ Coord::new(+ Self::display_layout(options, project, resolution_base, options.screen_size)+ .content_size,+ )}fn display_bounds(@@ -4036,6 +4055,12 @@ impl ProjectUniforms {) -> Self {let options = &constants.options;let output_size = Self::get_output_size(options, project, resolution_base);+ let source_size = segment_frames+ .screen_frame+ .as_ref()+ .map_or(segment_frames.screen_size, |frame| {+ XY::new(frame.width(), frame.height())+ });let fps_f32 = fps.max(1) as f32;let timeline_time = f64::from(frame_number) / f64::from(fps.max(1));let frame_time = timeline_time as f32;@@ -4049,9 +4074,9 @@ impl ProjectUniforms {let has_layout_override =base_crop.size != styled_crop.size || base_padding != project.background.padding;let display_layout = if has_layout_override {- Self::styled_display_layout(options, project, output_size, resolution_base)+ Self::styled_display_layout(options, project, output_size, resolution_base, source_size)} else {- Self::display_layout(options, project, resolution_base)+ Self::display_layout(options, project, resolution_base, source_size)};let prev_frame_time = if frame_number == 0 {0.0@@ -4711,12 +4736,6 @@ impl ProjectUniforms {)};- let source_size = segment_frames- .screen_frame- .as_ref()- .map_or(segment_frames.screen_size, |frame| {- XY::new(frame.width(), frame.height())- });let display = display.for_source_frame(source_size);let display_outer_bounds = if frame_chrome.is_none() {display.target_bounds@@ -5397,9 +5416,19 @@ mod tests {};project.background.padding = 25.0;let output = ProjectUniforms::get_output_size(&options, &project, resolution);- let base = ProjectUniforms::display_layout(&options, &project, resolution);- let copied =- ProjectUniforms::styled_display_layout(&options, &project, output, resolution);+ let base = ProjectUniforms::display_layout(+ &options,+ &project,+ resolution,+ options.screen_size,+ );+ let copied = ProjectUniforms::styled_display_layout(+ &options,+ &project,+ output,+ resolution,+ options.screen_size,+ );for (before, after) in [(base.content_size.x, copied.content_size.x),(base.content_size.y, copied.content_size.y),@@ -5411,8 +5440,13 @@ mod tests {);}project.background.padding += 0.01;- let resized =- ProjectUniforms::styled_display_layout(&options, &project, output, resolution);+ let resized = ProjectUniforms::styled_display_layout(+ &options,+ &project,+ output,+ resolution,+ options.screen_size,+ );assert!((copied.content_size.x - resized.content_size.x).abs() < 4.0);assert!((copied.content_size.y - resized.content_size.y).abs() < 4.0);}@@ -5480,7 +5514,12 @@ mod tests {let mut project = ProjectConfiguration::default();project.background.padding = 25.0;- let layout = ProjectUniforms::display_layout(&options, &project, XY::new(2688, 1512));+ let layout = ProjectUniforms::display_layout(+ &options,+ &project,+ XY::new(2688, 1512),+ options.screen_size,+ );assert_eq!(layout.outer_offset, layout.content_offset);assert_eq!(layout.outer_size, layout.content_size);@@ -5498,7 +5537,12 @@ mod tests {let resolution_base = XY::new(2112, 1188);let (out_w, out_h) = ProjectUniforms::get_output_size(&options, &project, resolution_base);- let layout = ProjectUniforms::display_layout(&options, &project, resolution_base);+ let layout = ProjectUniforms::display_layout(+ &options,+ &project,+ resolution_base,+ options.screen_size,+ );let insets = frame_chrome::chrome_insets(FrameStyle::Browser);// The video keeps the recording's aspect ratio.@@ -5537,9 +5581,19 @@ mod tests {});let resolution_base = XY::new(2304, 1296);- let centered = ProjectUniforms::display_layout(&options, &project, resolution_base);+ let centered = ProjectUniforms::display_layout(+ &options,+ &project,+ resolution_base,+ options.screen_size,+ );project.background.display_position = Some(XY::new(0.25, 0.75));- let moved = ProjectUniforms::display_layout(&options, &project, resolution_base);+ let moved = ProjectUniforms::display_layout(+ &options,+ &project,+ resolution_base,+ options.screen_size,+ );let outer_delta = moved.outer_offset - centered.outer_offset;let content_delta = moved.content_offset - centered.content_offset;@@ -8841,7 +8895,7 @@ mod notch_bounds_tests {NOTCH,&options(),project,- ProjectUniforms::display_layout(&options(), project, SCREEN),+ ProjectUniforms::display_layout(&options(), project, SCREEN, SCREEN),zoom,)}diff --git a/crates/rendering/tests/display_layout.rs b/crates/rendering/tests/display_layout.rsnew file mode 100644index 000000000..37176d5b5--- /dev/null+++ b/crates/rendering/tests/display_layout.rs@@ -0,0 +1,164 @@+use cap_project::{+ AspectRatio, BackgroundSource, Crop, CursorEvents, FrameConfiguration, FrameStyle,+ ProjectConfiguration, RecordingMeta, XY,+};+use cap_rendering::{+ DecodedFrame, DecodedSegmentFrames, FrameRenderer, ProjectUniforms, RenderOptions,+ RenderVideoConstants, RendererLayers, ZoomTransformTimeline,+};++#[test]+fn portrait_vertical_without_padding_fills_the_output() {+ let options = RenderOptions {+ screen_size: XY::new(1080, 1920),+ camera_size: None,+ preserve_screen_alpha: false,+ };+ let project = ProjectConfiguration {+ aspect_ratio: Some(AspectRatio::Vertical),+ ..ProjectConfiguration::default()+ };+ let resolution = XY::new(1080, 1920);++ assert_eq!(+ ProjectUniforms::get_output_size(&options, &project, resolution),+ (1080, 1920)+ );+ assert_eq!(+ ProjectUniforms::display_offset(&options, &project, resolution).coord,+ XY::new(0.0, 0.0)+ );+ assert_eq!(+ ProjectUniforms::display_size(&options, &project, resolution).coord,+ XY::new(1080.0, 1920.0)+ );+}++#[tokio::test]+async fn imported_portrait_fills_a_vertical_output() {+ let options = RenderOptions {+ screen_size: XY::new(1920, 1080),+ camera_size: None,+ preserve_screen_alpha: false,+ };+ let recording_meta: RecordingMeta = serde_json::from_value(serde_json::json!({+ "pretty_name": "portrait-import",+ "display": { "path": "display.mp4", "fps": 30 },+ "camera": null,+ "audio": null,+ "cursor": null+ }))+ .unwrap();+ let meta = recording_meta.studio_meta().unwrap().clone();+ let mut constants = RenderVideoConstants::new_with_options(options, recording_meta, meta)+ .await+ .unwrap();+ let mut project = ProjectConfiguration {+ aspect_ratio: Some(AspectRatio::Vertical),+ ..ProjectConfiguration::default()+ };+ project.background.shadow = 0.0;+ project.background.source = BackgroundSource::Color {+ value: [0, 0, 0],+ alpha: 255,+ };+ project.cursor.hide = true;+ let cursor = CursorEvents::default();+ let source_size = XY::new(1080, 1920);+ let frames = DecodedSegmentFrames {+ screen_size: options.screen_size,+ screen_frame: Some(DecodedFrame::new(+ [255, 0, 0, 255].repeat((source_size.x * source_size.y) as usize),+ source_size.x,+ source_size.y,+ )),+ camera_frame: None,+ segment_time: 0.0,+ recording_time: 0.0,+ segment_has_camera: false,+ };+ for decoded in [true, false] {+ let mut frames = frames.clone();+ if !decoded {+ frames.screen_frame = None;+ frames.screen_size = source_size;+ }+ let uniforms = uniforms_for(&constants, &project, &frames, source_size);+ assert_eq!(uniforms.frame_layout().output_size, [1080, 1920]);+ assert_bounds(uniforms.frame_layout().display, [0.0, 0.0, 1080.0, 1920.0]);+ }++ constants.options.screen_size = source_size;+ project.aspect_ratio = Some(AspectRatio::Wide);+ let mut landscape = frames.clone();+ landscape.screen_frame = Some(DecodedFrame::new(Vec::new(), 1920, 1080));+ let imported = uniforms_for(&constants, &project, &landscape, options.screen_size);+ assert_eq!(imported.frame_layout().output_size, [1920, 1080]);+ assert_bounds(imported.frame_layout().display, [0.0, 0.0, 1920.0, 1080.0]);+ constants.options = options;+ project.aspect_ratio = Some(AspectRatio::Vertical);++ for style in [FrameStyle::Browser, FrameStyle::MacOS] {+ project.background.frame = Some(FrameConfiguration {+ style,+ ..Default::default()+ });+ let imported = uniforms_for(&constants, &project, &frames, source_size);+ constants.options.screen_size = source_size;+ let native = uniforms_for(&constants, &project, &frames, source_size);+ constants.options = options;+ assert_bounds(+ imported.frame_layout().display,+ native.frame_layout().display,+ );+ }++ project.background.frame = None;+ project.background.crop = Some(Crop {+ position: XY::new(480, 0),+ size: XY::new(960, 1080),+ });+ let cropped = uniforms_for(&constants, &project, &frames, source_size);+ assert_bounds(cropped.frame_layout().display, [270.0, 0.0, 810.0, 1920.0]);++ project.background.crop = None;+ let uniforms = uniforms_for(&constants, &project, &frames, XY::new(180, 320));+ let mut renderer = FrameRenderer::new(&constants);+ let mut layers = RendererLayers::new(&constants.device, &constants.queue);+ let rendered = renderer+ .render_immediate(frames, uniforms, &cursor, true, &mut layers)+ .await+ .unwrap();+ assert_eq!((rendered.width, rendered.height), (180, 320));+ for (x, y) in [(90, 5), (5, 160), (174, 160), (90, 314)] {+ let offset = y * rendered.padded_bytes_per_row as usize + x * 4;+ let pixel = &rendered.data[offset..offset + 4];+ assert!(+ pixel[0] > 240 && pixel[1] < 10 && pixel[2] < 10,+ "{x},{y}: {pixel:?}"+ );+ }+}++fn assert_bounds(actual: [f32; 4], expected: [f32; 4]) {+ for (actual_value, expected_value) in actual.iter().zip(expected) {+ assert!(+ (actual_value - expected_value).abs() < 2.0,+ "display {actual:?}, expected {expected:?}",+ );+ }+}++fn uniforms_for(+ constants: &RenderVideoConstants,+ project: &ProjectConfiguration,+ frames: &DecodedSegmentFrames,+ resolution: XY<u32>,+) -> ProjectUniforms {+ let cursor = CursorEvents::default();+ let zoom =+ ZoomTransformTimeline::from_project(project, &cursor, 1.0, constants.options.screen_size);+ ProjectUniforms::new(+ constants, project, 0, 30, resolution, &cursor, frames, 1.0, &zoom,+ )+}