Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(rendering): fit imported videos to the output aspect ratio

Aditya Garud19de9e59167a
diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs
index 8e95cd3b6..59c791271 100644
--- a/crates/rendering/src/lib.rs
+++ b/crates/rendering/src/lib.rs
@@ -3545,7 +3545,10 @@ impl ProjectUniforms {
project: &ProjectConfiguration,
resolution_base: XY<u32>,
) -> Coord<FrameSpace> {
- Coord::new(Self::display_layout(options, project, resolution_base).content_offset)
+ Coord::new(
+ Self::display_layout(options, project, resolution_base, options.screen_size)
+ .content_offset,
+ )
}
/// Placement of the display card in output-frame pixels.
@@ -3561,10 +3564,23 @@ impl ProjectUniforms {
options: &RenderOptions,
project: &ProjectConfiguration,
resolution_base: XY<u32>,
+ source_size: XY<u32>,
) -> DisplayLayout {
- let base = Self::display_base_offset(options, project, resolution_base).coord;
+ let aspect = Self::display_source_aspect(options, project, source_size);
+ let base = Self::display_base_offset(options, project, resolution_base, aspect).coord;
let output_size = Self::get_output_size(options, project, resolution_base);
- Self::display_layout_in_box(options, project, output_size, base)
+ Self::display_layout_in_box(project, output_size, base, aspect)
+ }
+
+ fn display_source_aspect(
+ options: &RenderOptions,
+ project: &ProjectConfiguration,
+ source_size: XY<u32>,
+ ) -> f64 {
+ let crop_size = Self::get_crop(options, project).size.map(f64::from);
+ let source_scale = source_size.map(f64::from) / options.screen_size.map(f64::from);
+ let crop_size = crop_size * source_scale;
+ (crop_size.x / crop_size.y.max(f64::EPSILON)).max(f64::EPSILON)
}
fn styled_display_layout(
@@ -3572,22 +3588,25 @@ impl ProjectUniforms {
project: &ProjectConfiguration,
output_size: (u32, u32),
resolution_base: XY<u32>,
+ source_size: XY<u32>,
) -> DisplayLayout {
let canvas = XY::new(output_size.0 as f64, output_size.1 as f64);
let styled_output = Self::get_output_size(options, project, resolution_base);
let styled_canvas = XY::new(styled_output.0 as f64, styled_output.1 as f64);
- let styled_offset = Self::display_base_offset(options, project, resolution_base).coord;
+ let aspect = Self::display_source_aspect(options, project, source_size);
+ let styled_offset =
+ Self::display_base_offset(options, project, resolution_base, aspect).coord;
let content = (styled_canvas - styled_offset) - styled_offset;
let scale = (canvas.x / styled_canvas.x).min(canvas.y / styled_canvas.y);
let base = (canvas - content * scale) / 2.0;
- Self::display_layout_in_box(options, project, output_size, base)
+ Self::display_layout_in_box(project, output_size, base, aspect)
}
fn display_layout_in_box(
- options: &RenderOptions,
project: &ProjectConfiguration,
output_size: (u32, u32),
base: XY<f64>,
+ aspect: f64,
) -> DisplayLayout {
let output_size = XY::new(output_size.0 as f64, output_size.1 as f64);
// Same op order as the legacy display_size (end - offset - offset) so
@@ -3600,8 +3619,6 @@ impl ProjectUniforms {
(base, box_size, base, box_size)
} else {
let insets = frame_chrome::chrome_insets(style);
- let crop = Self::get_crop(options, project);
- let aspect = (crop.size.x as f64 / f64::from(crop.size.y.max(1))).max(f64::EPSILON);
// Insets are fractions of the content height; solve for the
// largest content that keeps the outer card inside the box.
let outer_w_per_h = aspect + insets.left + insets.right;
@@ -3643,6 +3660,7 @@ impl ProjectUniforms {
options: &RenderOptions,
project: &ProjectConfiguration,
resolution_base: XY<u32>,
+ cropped_aspect: f64,
) -> Coord<FrameSpace> {
let output_size = Self::get_output_size(options, project, resolution_base);
let output_size = XY::new(output_size.0 as f64, output_size.1 as f64);
@@ -3673,8 +3691,6 @@ impl ProjectUniforms {
let cropped_size = crop_end.coord - crop_start.coord;
- let cropped_aspect = cropped_size.x / cropped_size.y;
-
let padding = {
let padding_factor = project.background.padding / 100.0 * SCREEN_MAX_PADDING;
let crop_basis = f64::max(cropped_size.x, cropped_size.y);
@@ -3719,7 +3735,10 @@ impl ProjectUniforms {
project: &ProjectConfiguration,
resolution_base: XY<u32>,
) -> Coord<FrameSpace> {
- Coord::new(Self::display_layout(options, project, resolution_base).content_size)
+ Coord::new(
+ Self::display_layout(options, project, resolution_base, options.screen_size)
+ .content_size,
+ )
}
fn display_bounds(
@@ -4036,6 +4055,12 @@ impl ProjectUniforms {
) -> Self {
let options = &constants.options;
let output_size = Self::get_output_size(options, project, resolution_base);
+ let source_size = segment_frames
+ .screen_frame
+ .as_ref()
+ .map_or(segment_frames.screen_size, |frame| {
+ XY::new(frame.width(), frame.height())
+ });
let fps_f32 = fps.max(1) as f32;
let timeline_time = f64::from(frame_number) / f64::from(fps.max(1));
let frame_time = timeline_time as f32;
@@ -4049,9 +4074,9 @@ impl ProjectUniforms {
let has_layout_override =
base_crop.size != styled_crop.size || base_padding != project.background.padding;
let display_layout = if has_layout_override {
- Self::styled_display_layout(options, project, output_size, resolution_base)
+ Self::styled_display_layout(options, project, output_size, resolution_base, source_size)
} else {
- Self::display_layout(options, project, resolution_base)
+ Self::display_layout(options, project, resolution_base, source_size)
};
let prev_frame_time = if frame_number == 0 {
0.0
@@ -4711,12 +4736,6 @@ impl ProjectUniforms {
)
};
- let source_size = segment_frames
- .screen_frame
- .as_ref()
- .map_or(segment_frames.screen_size, |frame| {
- XY::new(frame.width(), frame.height())
- });
let display = display.for_source_frame(source_size);
let display_outer_bounds = if frame_chrome.is_none() {
display.target_bounds
@@ -5397,9 +5416,19 @@ mod tests {
};
project.background.padding = 25.0;
let output = ProjectUniforms::get_output_size(&options, &project, resolution);
- let base = ProjectUniforms::display_layout(&options, &project, resolution);
- let copied =
- ProjectUniforms::styled_display_layout(&options, &project, output, resolution);
+ let base = ProjectUniforms::display_layout(
+ &options,
+ &project,
+ resolution,
+ options.screen_size,
+ );
+ let copied = ProjectUniforms::styled_display_layout(
+ &options,
+ &project,
+ output,
+ resolution,
+ options.screen_size,
+ );
for (before, after) in [
(base.content_size.x, copied.content_size.x),
(base.content_size.y, copied.content_size.y),
@@ -5411,8 +5440,13 @@ mod tests {
);
}
project.background.padding += 0.01;
- let resized =
- ProjectUniforms::styled_display_layout(&options, &project, output, resolution);
+ let resized = ProjectUniforms::styled_display_layout(
+ &options,
+ &project,
+ output,
+ resolution,
+ options.screen_size,
+ );
assert!((copied.content_size.x - resized.content_size.x).abs() < 4.0);
assert!((copied.content_size.y - resized.content_size.y).abs() < 4.0);
}
@@ -5480,7 +5514,12 @@ mod tests {
let mut project = ProjectConfiguration::default();
project.background.padding = 25.0;
- let layout = ProjectUniforms::display_layout(&options, &project, XY::new(2688, 1512));
+ let layout = ProjectUniforms::display_layout(
+ &options,
+ &project,
+ XY::new(2688, 1512),
+ options.screen_size,
+ );
assert_eq!(layout.outer_offset, layout.content_offset);
assert_eq!(layout.outer_size, layout.content_size);
@@ -5498,7 +5537,12 @@ mod tests {
let resolution_base = XY::new(2112, 1188);
let (out_w, out_h) = ProjectUniforms::get_output_size(&options, &project, resolution_base);
- let layout = ProjectUniforms::display_layout(&options, &project, resolution_base);
+ let layout = ProjectUniforms::display_layout(
+ &options,
+ &project,
+ resolution_base,
+ options.screen_size,
+ );
let insets = frame_chrome::chrome_insets(FrameStyle::Browser);
// The video keeps the recording's aspect ratio.
@@ -5537,9 +5581,19 @@ mod tests {
});
let resolution_base = XY::new(2304, 1296);
- let centered = ProjectUniforms::display_layout(&options, &project, resolution_base);
+ let centered = ProjectUniforms::display_layout(
+ &options,
+ &project,
+ resolution_base,
+ options.screen_size,
+ );
project.background.display_position = Some(XY::new(0.25, 0.75));
- let moved = ProjectUniforms::display_layout(&options, &project, resolution_base);
+ let moved = ProjectUniforms::display_layout(
+ &options,
+ &project,
+ resolution_base,
+ options.screen_size,
+ );
let outer_delta = moved.outer_offset - centered.outer_offset;
let content_delta = moved.content_offset - centered.content_offset;
@@ -8841,7 +8895,7 @@ mod notch_bounds_tests {
NOTCH,
&options(),
project,
- ProjectUniforms::display_layout(&options(), project, SCREEN),
+ ProjectUniforms::display_layout(&options(), project, SCREEN, SCREEN),
zoom,
)
}
diff --git a/crates/rendering/tests/display_layout.rs b/crates/rendering/tests/display_layout.rs
new file mode 100644
index 000000000..37176d5b5
--- /dev/null
+++ b/crates/rendering/tests/display_layout.rs
@@ -0,0 +1,164 @@
+use cap_project::{
+ AspectRatio, BackgroundSource, Crop, CursorEvents, FrameConfiguration, FrameStyle,
+ ProjectConfiguration, RecordingMeta, XY,
+};
+use cap_rendering::{
+ DecodedFrame, DecodedSegmentFrames, FrameRenderer, ProjectUniforms, RenderOptions,
+ RenderVideoConstants, RendererLayers, ZoomTransformTimeline,
+};
+
+#[test]
+fn portrait_vertical_without_padding_fills_the_output() {
+ let options = RenderOptions {
+ screen_size: XY::new(1080, 1920),
+ camera_size: None,
+ preserve_screen_alpha: false,
+ };
+ let project = ProjectConfiguration {
+ aspect_ratio: Some(AspectRatio::Vertical),
+ ..ProjectConfiguration::default()
+ };
+ let resolution = XY::new(1080, 1920);
+
+ assert_eq!(
+ ProjectUniforms::get_output_size(&options, &project, resolution),
+ (1080, 1920)
+ );
+ assert_eq!(
+ ProjectUniforms::display_offset(&options, &project, resolution).coord,
+ XY::new(0.0, 0.0)
+ );
+ assert_eq!(
+ ProjectUniforms::display_size(&options, &project, resolution).coord,
+ XY::new(1080.0, 1920.0)
+ );
+}
+
+#[tokio::test]
+async fn imported_portrait_fills_a_vertical_output() {
+ let options = RenderOptions {
+ screen_size: XY::new(1920, 1080),
+ camera_size: None,
+ preserve_screen_alpha: false,
+ };
+ let recording_meta: RecordingMeta = serde_json::from_value(serde_json::json!({
+ "pretty_name": "portrait-import",
+ "display": { "path": "display.mp4", "fps": 30 },
+ "camera": null,
+ "audio": null,
+ "cursor": null
+ }))
+ .unwrap();
+ let meta = recording_meta.studio_meta().unwrap().clone();
+ let mut constants = RenderVideoConstants::new_with_options(options, recording_meta, meta)
+ .await
+ .unwrap();
+ let mut project = ProjectConfiguration {
+ aspect_ratio: Some(AspectRatio::Vertical),
+ ..ProjectConfiguration::default()
+ };
+ project.background.shadow = 0.0;
+ project.background.source = BackgroundSource::Color {
+ value: [0, 0, 0],
+ alpha: 255,
+ };
+ project.cursor.hide = true;
+ let cursor = CursorEvents::default();
+ let source_size = XY::new(1080, 1920);
+ let frames = DecodedSegmentFrames {
+ screen_size: options.screen_size,
+ screen_frame: Some(DecodedFrame::new(
+ [255, 0, 0, 255].repeat((source_size.x * source_size.y) as usize),
+ source_size.x,
+ source_size.y,
+ )),
+ camera_frame: None,
+ segment_time: 0.0,
+ recording_time: 0.0,
+ segment_has_camera: false,
+ };
+ for decoded in [true, false] {
+ let mut frames = frames.clone();
+ if !decoded {
+ frames.screen_frame = None;
+ frames.screen_size = source_size;
+ }
+ let uniforms = uniforms_for(&constants, &project, &frames, source_size);
+ assert_eq!(uniforms.frame_layout().output_size, [1080, 1920]);
+ assert_bounds(uniforms.frame_layout().display, [0.0, 0.0, 1080.0, 1920.0]);
+ }
+
+ constants.options.screen_size = source_size;
+ project.aspect_ratio = Some(AspectRatio::Wide);
+ let mut landscape = frames.clone();
+ landscape.screen_frame = Some(DecodedFrame::new(Vec::new(), 1920, 1080));
+ let imported = uniforms_for(&constants, &project, &landscape, options.screen_size);
+ assert_eq!(imported.frame_layout().output_size, [1920, 1080]);
+ assert_bounds(imported.frame_layout().display, [0.0, 0.0, 1920.0, 1080.0]);
+ constants.options = options;
+ project.aspect_ratio = Some(AspectRatio::Vertical);
+
+ for style in [FrameStyle::Browser, FrameStyle::MacOS] {
+ project.background.frame = Some(FrameConfiguration {
+ style,
+ ..Default::default()
+ });
+ let imported = uniforms_for(&constants, &project, &frames, source_size);
+ constants.options.screen_size = source_size;
+ let native = uniforms_for(&constants, &project, &frames, source_size);
+ constants.options = options;
+ assert_bounds(
+ imported.frame_layout().display,
+ native.frame_layout().display,
+ );
+ }
+
+ project.background.frame = None;
+ project.background.crop = Some(Crop {
+ position: XY::new(480, 0),
+ size: XY::new(960, 1080),
+ });
+ let cropped = uniforms_for(&constants, &project, &frames, source_size);
+ assert_bounds(cropped.frame_layout().display, [270.0, 0.0, 810.0, 1920.0]);
+
+ project.background.crop = None;
+ let uniforms = uniforms_for(&constants, &project, &frames, XY::new(180, 320));
+ let mut renderer = FrameRenderer::new(&constants);
+ let mut layers = RendererLayers::new(&constants.device, &constants.queue);
+ let rendered = renderer
+ .render_immediate(frames, uniforms, &cursor, true, &mut layers)
+ .await
+ .unwrap();
+ assert_eq!((rendered.width, rendered.height), (180, 320));
+ for (x, y) in [(90, 5), (5, 160), (174, 160), (90, 314)] {
+ let offset = y * rendered.padded_bytes_per_row as usize + x * 4;
+ let pixel = &rendered.data[offset..offset + 4];
+ assert!(
+ pixel[0] > 240 && pixel[1] < 10 && pixel[2] < 10,
+ "{x},{y}: {pixel:?}"
+ );
+ }
+}
+
+fn assert_bounds(actual: [f32; 4], expected: [f32; 4]) {
+ for (actual_value, expected_value) in actual.iter().zip(expected) {
+ assert!(
+ (actual_value - expected_value).abs() < 2.0,
+ "display {actual:?}, expected {expected:?}",
+ );
+ }
+}
+
+fn uniforms_for(
+ constants: &RenderVideoConstants,
+ project: &ProjectConfiguration,
+ frames: &DecodedSegmentFrames,
+ resolution: XY<u32>,
+) -> ProjectUniforms {
+ let cursor = CursorEvents::default();
+ let zoom =
+ ZoomTransformTimeline::from_project(project, &cursor, 1.0, constants.options.screen_size);
+ ProjectUniforms::new(
+ constants, project, 0, 30, resolution, &cursor, frames, 1.0, &zoom,
+ )
+}