Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(render): a background behind the main picture, and Fit and Fill (#232, #255)

Jareer03dac1f98de5
diff --git a/src/crates/concat-export/src/flatten.rs b/src/crates/concat-export/src/flatten.rs
index 3c2416a0..0157f8c2 100644
--- a/src/crates/concat-export/src/flatten.rs
+++ b/src/crates/concat-export/src/flatten.rs
@@ -15,10 +15,12 @@
use std::path::Path;
-use concat_project::model::{ClipKind as ModelClipKind, KeyProperty, Project, Timeline};
+use concat_project::model::{
+ Background, ClipKind as ModelClipKind, KeyProperty, MediaKind, Project, Timeline,
+};
use crate::chains::audio_filter_chain;
-use crate::{ClipKind, ExportClip, ExportKey, TransitionSpec};
+use crate::{ClipKind, ExportBackdrop, ExportClip, ExportKey, TransitionSpec};
/// Flattens one timeline of `project` for the exporter - the active one
/// when `timeline_id` is `None`. Text clips are excluded (they rasterise
@@ -39,6 +41,8 @@ pub fn flatten_timeline_in(
let Some(timeline) = pick_timeline(project, timeline_id) else {
return Vec::new();
};
+ // Found once: asked of every clip, it is a walk of every clip.
+ let lane = timeline.background_lane();
timeline
.clips
@@ -133,6 +137,9 @@ pub fn flatten_timeline_in(
media_height: media.height,
media_duration: media.duration,
has_audio: Some(media.has_audio),
+ backdrop: timeline
+ .background_on(clip, lane)
+ .and_then(|background| backdrop_of(project, background)),
cutout: clip.cutout.clone(),
mask_dir: match (&clip.cutout, project_dir) {
(Some(cutout), Some(dir)) => {
@@ -148,6 +155,26 @@ pub fn flatten_timeline_in(
.collect()
}
+/// A background as it is drawn: a blur and a colour as they are, a still
+/// found in the bin. None for black, and for a still that is no longer in
+/// the bin or is not a still - a background that cannot be drawn is no
+/// background, never a failed frame.
+fn backdrop_of(project: &Project, background: &Background) -> Option<ExportBackdrop> {
+ match background {
+ Background::None => None,
+ Background::Blur { amount } => Some(ExportBackdrop::Blur { amount: *amount }),
+ Background::Color { .. } => background.rgb().map(|rgb| ExportBackdrop::Colour { rgb }),
+ Background::Image { media_id } => project
+ .media_by_id(media_id)
+ .filter(|item| item.kind == MediaKind::Image)
+ .map(|item| ExportBackdrop::Image {
+ path: item.path.clone(),
+ width: item.width,
+ height: item.height,
+ }),
+ }
+}
+
/// The timeline to flatten: the named one, or the document's active one,
/// falling back to the first - the same preference the UI's tab strip has.
fn pick_timeline<'a>(project: &'a Project, timeline_id: Option<&str>) -> Option<&'a Timeline> {
diff --git a/src/crates/concat-export/src/lib.rs b/src/crates/concat-export/src/lib.rs
index 1e92e548..cf6d6964 100644
--- a/src/crates/concat-export/src/lib.rs
+++ b/src/crates/concat-export/src/lib.rs
@@ -226,6 +226,53 @@ pub struct ExportClip {
/// it back through `reveal_order()`.
#[serde(skip)]
pub reveal_map: Option<Arc<RevealMap>>,
+ /// What fills the frame behind the picture where it does not reach, as
+ /// the flattener resolved it - the clip's own background or its
+ /// timeline's, for a picture on the lowest picture lane; absent for
+ /// black and for every other clip. See `Timeline::background_of`.
+ #[serde(default)]
+ pub backdrop: Option<ExportBackdrop>,
+}
+
+/// A clip's background, resolved for drawing: the document's
+/// `Background` with its colour read and its still found in the bin.
+#[derive(Deserialize, Clone, PartialEq, Debug)]
+#[serde(
+ tag = "kind",
+ rename_all = "camelCase",
+ rename_all_fields = "camelCase"
+)]
+pub enum ExportBackdrop {
+ /// The clip's own picture, enlarged to cover the frame and blurred.
+ Blur {
+ /// How soft, `0..=1`: a Gaussian of a twelfth of the frame's
+ /// shorter side at one.
+ amount: f64,
+ },
+ /// One opaque colour.
+ Colour {
+ /// Its channels.
+ rgb: [u8; 3],
+ },
+ /// A still, enlarged to cover the frame.
+ Image {
+ /// The file.
+ path: String,
+ /// Its pixel width, where the bin knows it: what it is decoded at.
+ width: Option<u32>,
+ /// Its pixel height.
+ height: Option<u32>,
+ },
+}
+
+impl ExportBackdrop {
+ /// The blur's spread, in pixels of a `width` by `height` frame, at
+ /// `amount`: a twelfth of the shorter side at the most, so the same
+ /// amount is the same softness at every size the frame is drawn at -
+ /// the monitor's quarter and the export's whole alike.
+ pub fn sigma(amount: f64, width: u32, height: u32) -> f32 {
+ (amount.clamp(0.0, 1.0) * f64::from(width.min(height)) / 12.0) as f32
+ }
}
impl ExportClip {
@@ -277,6 +324,7 @@ impl ExportClip {
mask_dir: String::new(),
highlighted: false,
reveal_map: None,
+ backdrop: None,
}
}
}
@@ -1308,6 +1356,89 @@ fn shapes_at(
}
}
+/// `layers` with every picture's background beneath it, in a `width` by
+/// `height` frame: the one place the export and the monitor both put
+/// backgrounds in, so the two cannot draw them two ways. A picture that
+/// already hides the whole frame gets none - `opaque` says whose pixels
+/// have no holes to see one through - and a still that cannot be read is
+/// no background rather than a failed frame.
+fn with_backdrops(
+ layers: Vec<PlannedLayer>,
+ backdrops: &HashMap<ClipId, ExportBackdrop>,
+ opaque: impl Fn(ClipId) -> bool,
+ pool: &concat_media::ReaderPool,
+ (width, height): (u32, u32),
+) -> Vec<PlannedLayer> {
+ if backdrops.is_empty() {
+ return layers;
+ }
+ let mut out = Vec::with_capacity(layers.len() + 1);
+ for layer in layers {
+ let drawn = backdrops
+ .get(&layer.clip)
+ .filter(|_| !layer.covers(width, height, opaque(layer.clip)))
+ .and_then(|backdrop| backdrop_of(backdrop, pool, (width, height)))
+ .and_then(|backdrop| layer.backdrop(&backdrop, width, height, blur_pass));
+ out.extend(drawn);
+ out.push(layer);
+ }
+ out
+}
+
+/// A clip's background as the frame plan draws it, at a `width` by
+/// `height` frame: the blur's spread in that frame's pixels, a still read
+/// through the pool at a size that covers it.
+fn backdrop_of(
+ backdrop: &ExportBackdrop,
+ pool: &concat_media::ReaderPool,
+ (width, height): (u32, u32),
+) -> Option<concat_render::Backdrop> {
+ match backdrop {
+ ExportBackdrop::Blur { amount } => Some(concat_render::Backdrop::Blur {
+ sigma: ExportBackdrop::sigma(*amount, width, height),
+ }),
+ ExportBackdrop::Colour { rgb } => Some(concat_render::Backdrop::Colour(*rgb)),
+ ExportBackdrop::Image {
+ path,
+ width: still_width,
+ height: still_height,
+ } => {
+ // Read at the size that covers the frame, its own shape kept;
+ // the frame's when the bin never learnt the still's.
+ let size = match (still_width, still_height) {
+ (Some(w), Some(h)) if *w > 0 && *h > 0 => {
+ let cover =
+ (f64::from(width) / f64::from(*w)).max(f64::from(height) / f64::from(*h));
+ (
+ ((f64::from(*w) * cover).round() as u32).max(2),
+ ((f64::from(*h) * cover).round() as u32).max(2),
+ )
+ }
+ _ => (width, height),
+ };
+ let request = concat_media::FrameRequest::new(path, Rational::ZERO, size.0, size.1)
+ .as_still(true)
+ .at_any_size(true);
+ match pool.frame(&request) {
+ Ok(still) => Some(concat_render::Backdrop::Image(still)),
+ Err(error) => {
+ log::debug!("background image {path}: {error}");
+ None
+ }
+ }
+ }
+ }
+}
+
+/// The Gaussian blur's pass at `radius` pixels: the package every blurred
+/// background is drawn with, so it blurs exactly as the effect does.
+fn blur_pass(radius: f32) -> Option<ShaderPass> {
+ Catalogue::builtin().get("concat.gaussian-blur")?.pass(
+ &BTreeMap::from([("radius".to_owned(), f64::from(radius))]),
+ None,
+ )
+}
+
/// A stack already drawn, as a layer over the whole frame on the lowest
/// track: texel for pixel, since it is the frame's own size.
fn ground_layer(ground: Frame) -> PlannedLayer {
@@ -1472,6 +1603,7 @@ fn render_picture(
reveal_maps,
cutouts,
highlight: _,
+ backdrops,
} = build_timeline(request, rate, visible, transitions);
let options = EncodeOptions {
@@ -1640,7 +1772,13 @@ fn render_picture(
time,
width: request.width,
height: request.height,
- layers,
+ layers: with_backdrops(
+ layers,
+ &backdrops,
+ |clip| !stills.contains(&clip) && !cutouts.contains_key(&clip),
+ &sought,
+ (request.width, request.height),
+ ),
treatments: Vec::new(),
output: output_of(request.color_space),
};
@@ -2077,6 +2215,7 @@ pub fn preview_sources_of(
reveal_maps,
cutouts,
highlight,
+ backdrops,
} = &plan.built;
let highlight = *highlight;
let time = quantise(seconds, rate);
@@ -2138,7 +2277,13 @@ pub fn preview_sources_of(
time,
width: plan.width,
height: plan.height,
- layers,
+ layers: with_backdrops(
+ layers,
+ backdrops,
+ |clip| !stills.contains(&clip) && !cutouts.contains_key(&clip),
+ pool,
+ (plan.width, plan.height),
+ ),
treatments: Vec::new(),
output: plan.output,
};
diff --git a/src/crates/concat-export/src/resolve.rs b/src/crates/concat-export/src/resolve.rs
index ac8f92ed..31013621 100644
--- a/src/crates/concat-export/src/resolve.rs
+++ b/src/crates/concat-export/src/resolve.rs
@@ -115,6 +115,9 @@ pub(crate) struct BuiltTimeline {
pub(crate) transitions: Vec<TransitionSpan>,
/// The clips whose background a mask takes away.
pub(crate) cutouts: HashMap<ClipId, CutoutJob>,
+ /// What fills the frame behind each picture that has a background; see
+ /// [`crate::with_backdrops`].
+ pub(crate) backdrops: HashMap<ClipId, crate::ExportBackdrop>,
}
/// A packaged transition over a cut. The incoming clip has been overlapped
@@ -212,6 +215,7 @@ pub(crate) fn build_timeline(
let mut chains: HashMap<ClipId, Vec<AppliedFilter>> = HashMap::new();
let mut reveal_maps: HashMap<ClipId, Arc<RevealMap>> = HashMap::new();
let mut cutouts: HashMap<ClipId, CutoutJob> = HashMap::new();
+ let mut backdrops: HashMap<ClipId, crate::ExportBackdrop> = HashMap::new();
let mut highlight: Option<ClipId> = None;
let lanes = visible.iter().map(|clip| clip.track).max().unwrap_or(0) + 1;
@@ -303,6 +307,9 @@ pub(crate) fn build_timeline(
if clip.highlighted {
highlight = Some(id);
}
+ if let Some(backdrop) = &clip.backdrop {
+ backdrops.insert(id, backdrop.clone());
+ }
}
}
@@ -320,6 +327,7 @@ pub(crate) fn build_timeline(
reveal_maps,
cutouts,
highlight,
+ backdrops,
}
}
diff --git a/src/crates/concat-host/tests/export.rs b/src/crates/concat-host/tests/export.rs
index e99c8bc7..9b78fb12 100644
--- a/src/crates/concat-host/tests/export.rs
+++ b/src/crates/concat-host/tests/export.rs
@@ -860,6 +860,145 @@ fn crop_and_flips_reach_the_picture() {
);
}
+/// A background fills the bars of a wide picture in a tall frame - a
+/// colour, or the picture itself enlarged and blurred - and the monitor
+/// shows what the export writes: the paused monitor's frame, drawn from the
+/// same flattened clips, agrees with the file at every point checked. A
+/// fill then takes the bars away altogether.
+#[test]
+fn a_background_fills_the_bars_in_the_export_and_the_monitor() {
+ use concat_project::model::{Background, Framing};
+ let scratch = Scratch::new("background");
+ let path = scratch.path().join("quadrants.mp4");
+ quadrants(&path);
+ let [red, green, blue, yellow] = QUADRANTS;
+ let black = [0, 0, 0];
+ let (width, height) = (90, 160);
+
+ let mut studio = Studio::new(scratch.path(), "Background", video(width, height, 30, 1));
+ let media = studio.import(&path);
+ let clip = studio
+ .apply(Command::AddClipAtFirstFree {
+ media_id: media,
+ start: 0.0,
+ })
+ .expect("the clip has an id");
+ let timeline_id = studio.project().active_timeline_id.clone();
+ // The picture is 90x51 across the middle of the frame: its quadrants
+ // either side of (0.5, 0.5), the bars above 0.34 and below 0.66.
+ let picture = [
+ ((0.25, 0.42), red),
+ ((0.75, 0.42), green),
+ ((0.25, 0.58), blue),
+ ((0.75, 0.58), yellow),
+ ];
+ let monitor = |studio: &Studio| {
+ let request = concat_export::PreviewFrameRequest {
+ time: 1.0,
+ width,
+ height,
+ rate_num: 30,
+ rate_den: 1,
+ clips: studio.session.flattened_clips(),
+ color_space: Default::default(),
+ };
+ let pixels =
+ concat_export::preview_frame(&concat_media::ReaderPool::with_defaults(), &request)
+ .expect("the monitor draws the frame");
+ Frame::from_rgba(width, height, pixels).expect("a whole frame")
+ };
+ let agree = |label: &str, exported: &Exported, shown: &Frame| {
+ for y in [0.1, 0.25, 0.42, 0.58, 0.75, 0.9] {
+ for x in [0.1, 0.25, 0.4, 0.6, 0.75, 0.9] {
+ let file = exported.colour_at_point(1.0, x, y);
+ let [r, g, b, _] = shown
+ .pixel(
+ (f64::from(width) * x) as u32,
+ (f64::from(height) * y) as u32,
+ )
+ .expect("inside the frame");
+ assert!(
+ file.iter()
+ .zip([r, g, b])
+ .all(|(file, shown)| file.abs_diff(shown) <= 24),
+ "{label}: at ({x}, {y}) the file is {file:?} and the monitor {:?}",
+ [r, g, b]
+ );
+ }
+ }
+ };
+
+ let plain = studio.export("no background");
+ plain.expect_colours(1.0, &picture);
+ plain.expect_colours(1.0, &[((0.5, 0.1), black), ((0.5, 0.9), black)]);
+ agree("no background", &plain, &monitor(&studio));
+
+ let set = |studio: &mut Studio, background: Background| {
+ studio.apply(Command::SetTimelineBackground {
+ timeline_id: timeline_id.clone(),
+ background,
+ });
+ };
+ let teal = [30, 160, 140];
+ set(
+ &mut studio,
+ Background::Color {
+ color: "#1ea08c".to_owned(),
+ },
+ );
+ let coloured = studio.export("a colour background");
+ coloured.expect_colours(1.0, &picture);
+ coloured.expect_colours(1.0, &[((0.5, 0.1), teal), ((0.5, 0.9), teal)]);
+ agree("a colour background", &coloured, &monitor(&studio));
+
+ // Enlarged to cover the frame, the picture's middle third fills it:
+ // the left quadrants on the left, the right ones on the right, red and
+ // green above, blue and yellow below, a little softened.
+ set(&mut studio, Background::Blur { amount: 0.3 });
+ let blurred = studio.export("a blurred background");
+ blurred.expect_colours(1.0, &picture);
+ blurred.expect_colours(
+ 1.0,
+ &[
+ ((0.15, 0.1), red),
+ ((0.85, 0.1), green),
+ ((0.15, 0.9), blue),
+ ((0.85, 0.9), yellow),
+ ],
+ );
+ agree("a blurred background", &blurred, &monitor(&studio));
+
+ // The clip's own black over the timeline's blur.
+ studio.apply(Command::UpdateClip {
+ clip_id: clip.clone(),
+ patch: ClipPatch {
+ background: Some(Some(Background::None)),
+ ..ClipPatch::default()
+ },
+ });
+ studio
+ .export("the clip's own black")
+ .expect_colours(1.0, &[((0.5, 0.1), black), ((0.5, 0.9), black)]);
+
+ // Filled, the picture is the frame: no bars to fill.
+ studio.apply(Command::FrameClip {
+ clip_id: clip,
+ framing: Framing::Fill,
+ at: 0.0,
+ });
+ let filled = studio.export("filled");
+ filled.expect_colours(
+ 1.0,
+ &[
+ ((0.4, 0.1), red),
+ ((0.6, 0.1), green),
+ ((0.4, 0.9), blue),
+ ((0.6, 0.9), yellow),
+ ],
+ );
+ agree("filled", &filled, &monitor(&studio));
+}
+
/// Three seconds of one flat colour, silent.
fn solid(path: &Path, colour: [u8; 3]) {
let options = EncodeOptions {
diff --git a/src/crates/concat-project/src/commands/mod.rs b/src/crates/concat-project/src/commands/mod.rs
index 68212129..d663d511 100644
--- a/src/crates/concat-project/src/commands/mod.rs
+++ b/src/crates/concat-project/src/commands/mod.rs
@@ -13,9 +13,10 @@ use std::sync::Arc;
use serde::{Deserialize, Serialize};
use crate::model::{
- AppliedFilter, AudioTrack, Clip, ClipKind, ColorRange, ColorSpace, Crop, CustomFont, Cutout,
- CutoutMode, Enhanced, KeyEase, KeyProperty, MediaItem, MediaKind, MediaOrigin, Project,
- ShapeStyle, SpeedPoint, Stroke, TextStyle, Timeline, Track, Transition, VideoSettings,
+ AppliedFilter, AudioTrack, Background, Clip, ClipKind, ColorRange, ColorSpace, Crop,
+ CustomFont, Cutout, CutoutMode, Enhanced, Framing, KeyEase, KeyProperty, MediaItem, MediaKind,
+ MediaOrigin, Project, ShapeStyle, SpeedPoint, Stroke, TextStyle, Timeline, Track, Transition,
+ VideoSettings,
};
mod audio;
@@ -150,6 +151,15 @@ pub struct ClipPatch {
skip_serializing_if = "Option::is_none"
)]
pub audio_stream: Option<Option<u32>>,
+ /// The clip's own background over the timeline's: absent leaves it
+ /// alone, null goes back to following the timeline's, a value - black
+ /// included - is the clip's own. See `Clip::background`.
+ #[serde(
+ default,
+ with = "double_option",
+ skip_serializing_if = "Option::is_none"
+ )]
+ pub background: Option<Option<Background>>,
}
/// `Option<Option<T>>` over JSON: absent → None, null → Some(None).
@@ -621,6 +631,22 @@ pub enum Command {
#[serde(default)]
stretch_y: Option<f64>,
},
+ /// Sizes a picture to the frame in one step: fitted whole inside it,
+ /// or covering it, centred either way and unstretched (see
+ /// [`crate::model::framing_scale`]; a turned picture is sized as turned).
+ /// A scale or a position with keys gets a key at `at` rather than a new
+ /// constant, the way every edit of a keyed property lands, so the rest
+ /// of the ride is left as it was. A no-op for a clip that is not a video
+ /// or a still, or whose media's size is unknown.
+ FrameClip {
+ /// The clip to size. An unknown id is a no-op.
+ clip_id: String,
+ /// Fit or fill.
+ framing: Framing,
+ /// Where in the clip, `0..=1`, a keyed property takes its key.
+ #[serde(default)]
+ at: f64,
+ },
/// Pulls a video clip's sound out into its own audio clip on a free
/// lane (minting one if none is free), muting the video and moving its
/// audio filters to the sound. A no-op unless the clip is an unmuted
@@ -676,6 +702,15 @@ pub enum Command {
/// The frame and the rate, together.
video: VideoSettings,
},
+ /// Sets what fills a timeline's frame behind the pictures on its lowest
+ /// picture lane; see `Timeline::background_of`. Tidied first: an amount
+ /// held to `0..=1`, a colour that does not read made black.
+ SetTimelineBackground {
+ /// The timeline. An unknown id is a no-op.
+ timeline_id: String,
+ /// The background.
+ background: Background,
+ },
/// Renames a timeline tab. Whitespace-only names are ignored so a tab
/// can never end up blank; unknown ids are tolerated.
RenameTimeline {
@@ -991,6 +1026,10 @@ impl Command {
.iter()
.flatten()
.any(|tr| bad([tr.duration]))
+ || matches!(
+ patch.background,
+ Some(Some(Background::Blur { amount })) if !amount.is_finite()
+ )
|| patch.text.iter().flatten().any(|style| {
bad([
style.font_size,
@@ -1005,6 +1044,11 @@ impl Command {
})
}
Command::SetClipSpeed { speed, .. } => bad([*speed]),
+ Command::FrameClip { at, .. } => bad([*at]),
+ Command::SetTimelineBackground {
+ background: Background::Blur { amount },
+ ..
+ } => bad([*amount]),
Command::SetClipTransform {
scale,
offset_x,
@@ -1102,7 +1146,8 @@ pub fn apply(
| Command::ClearEffectKey { .. }
| Command::ClearEffectKeys { .. }
| Command::SetClipSpeedCurve { .. }
- | Command::SetClipTransform { .. }) => properties::apply(project, mint, command),
+ | Command::SetClipTransform { .. }
+ | Command::FrameClip { .. }) => properties::apply(project, mint, command),
command @ (Command::DetachAudio { .. } | Command::ReattachAudio { .. }) => {
audio::apply(project, mint, command)
}
@@ -1111,6 +1156,7 @@ pub fn apply(
| Command::SetTrackFlag { .. }) => tracks::apply(project, mint, command),
command @ (Command::AddTimeline
| Command::SetTimelineVideo { .. }
+ | Command::SetTimelineBackground { .. }
| Command::RemoveTimeline { .. }
| Command::RenameTimeline { .. }
| Command::SelectTimeline { .. }
diff --git a/src/crates/concat-project/src/commands/properties.rs b/src/crates/concat-project/src/commands/properties.rs
index d24c8756..9417fc83 100644
--- a/src/crates/concat-project/src/commands/properties.rs
+++ b/src/crates/concat-project/src/commands/properties.rs
@@ -81,6 +81,9 @@ pub(super) fn apply(
if let Some(stream) = patch.audio_stream {
applied |= assign(&mut clip.audio_stream, stream);
}
+ if let Some(background) = patch.background {
+ applied |= assign(&mut clip.background, background.map(Background::tidy));
+ }
Ok(Outcome {
created_id: None,
applied,
@@ -344,6 +347,41 @@ pub(super) fn apply(
})
}
+ Command::FrameClip {
+ clip_id,
+ framing,
+ at,
+ } => {
+ let timeline = project.active();
+ let frame = (timeline.video.width, timeline.video.height);
+ let Some(media) = timeline
+ .clip(&clip_id)
+ .and_then(|clip| project.media_by_id(&clip.media_id))
+ .map(|item| (item.width, item.height))
+ else {
+ return Ok(Outcome::default());
+ };
+ let Some(clip) = project.active_mut().clip_mut(&clip_id) else {
+ return Ok(Outcome::default());
+ };
+ let at = at.clamp(0.0, 1.0);
+ let Some(scale) = clip.framing_scale(framing, media, frame, at) else {
+ return Ok(Outcome::default());
+ };
+ let before = clip.clone();
+ clip.write_at(KeyProperty::Scale, scale, at);
+ clip.write_at(KeyProperty::OffsetX, 0.0, at);
+ clip.write_at(KeyProperty::OffsetY, 0.0, at);
+ // A stretch is a picture pulled out of its shape, and neither a
+ // fit nor a fill is: both are the picture as it was shot.
+ clip.stretch_x = 1.0;
+ clip.stretch_y = 1.0;
+ Ok(Outcome {
+ created_id: None,
+ applied: *clip != before,
+ })
+ }
+
_ => unreachable!("commands::apply routes only this module's commands here"),
}
}
diff --git a/src/crates/concat-project/src/commands/timelines.rs b/src/crates/concat-project/src/commands/timelines.rs
index 7adb10cf..d2569d04 100644
--- a/src/crates/concat-project/src/commands/timelines.rs
+++ b/src/crates/concat-project/src/commands/timelines.rs
@@ -36,10 +36,14 @@ pub(super) fn apply(
// likeliest second timeline is another cut of the same picture,
// and the one that is not is a sheet away from being told so.
let video = project.active().video;
+ // And with its background, for the same reason: a second cut of
+ // a vertical edit wants the same bars filled the same way.
+ let background = project.active().background.clone();
project.timelines.push(Arc::new(Timeline {
id: id.clone(),
name,
video,
+ background,
tracks,
clips: Vec::new(),
extra: Default::default(),
@@ -66,6 +70,23 @@ pub(super) fn apply(
})
}
+ Command::SetTimelineBackground {
+ timeline_id,
+ background,
+ } => {
+ let applied = project
+ .timelines
+ .iter_mut()
+ .find(|timeline| timeline.id == timeline_id)
+ .is_some_and(|timeline| {
+ assign(&mut Arc::make_mut(timeline).background, background.tidy())
+ });
+ Ok(Outcome {
+ created_id: None,
+ applied,
+ })
+ }
+
Command::RemoveTimeline { timeline_id } => {
if project.timelines.len() <= 1 {
return Err(CommandError::LastTimeline);
diff --git a/src/crates/concat-project/src/lib.rs b/src/crates/concat-project/src/lib.rs
index bf488194..974fc454 100644
--- a/src/crates/concat-project/src/lib.rs
+++ b/src/crates/concat-project/src/lib.rs
@@ -283,6 +283,288 @@ mod tests {
assert!(loaded.project().media[0].color_space.is_sdr());
}
+ /// A document from before backgrounds opens with none, on the timeline
+ /// and on every clip, and a project that never chooses one writes the
+ /// document it always wrote. One that does choose one reads it back,
+ /// and a kind from a newer build opens as none rather than failing the
+ /// load.
+ #[test]
+ fn a_background_is_absent_until_chosen_and_round_trips() {
+ use crate::model::Background;
+ let (mut editor, _, clip_id) = fixture();
+ let document = editor.to_document(&settings());
+ let timeline = &document["timelines"][0];
+ assert!(timeline.get("background").is_none(), "{timeline}");
+ assert!(timeline["clips"][0].get("background").is_none());
+ let loaded = Editor::from_document(&document).expect("loads");
+ assert!(loaded.project().active().background.is_none());
+ assert_eq!(loaded.project().active().clips[0].background, None);
+
+ let timeline_id = editor.project().active_timeline_id.clone();
+ editor
+ .apply(Command::SetTimelineBackground {
+ timeline_id: timeline_id.clone(),
+ background: Background::Blur { amount: 3.0 },
+ })
+ .expect("sets");
+ assert_eq!(
+ editor.project().active().background,
+ Background::Blur { amount: 1.0 },
+ "held to 0..=1"
+ );
+ editor
+ .apply(Command::UpdateClip {
+ clip_id: clip_id.clone(),
+ patch: ClipPatch {
+ background: Some(Some(Background::Color {
+ color: "#FF8000".to_owned(),
+ })),
+ ..ClipPatch::default()
+ },
+ })
+ .expect("sets");
+ let mut document = editor.to_document(&settings());
+ assert_eq!(
+ document["timelines"][0]["background"],
+ json!({ "kind": "blur", "amount": 1.0 })
+ );
+ assert_eq!(
+ document["timelines"][0]["clips"][0]["background"],
+ json!({ "kind": "color", "color": "#ff8000" })
+ );
+ let loaded = Editor::from_document(&document).expect("loads");
+ assert_eq!(loaded.project(), editor.project());
+
+ document["timelines"][0]["background"] = json!({ "kind": "hologram", "depth": 4 });
+ document["timelines"][0]["clips"][0]["background"] = json!({ "kind": "hologram" });
+ let loaded = Editor::from_document(&document).expect("loads");
+ assert!(loaded.project().active().background.is_none());
+ assert_eq!(loaded.project().active().clips[0].background, None);
+
+ // Null on the wire is "follow the timeline's" again.
+ let follow: Command = serde_json::from_value(json!({
+ "op": "updateClip",
+ "clipId": clip_id,
+ "patch": { "background": null }
+ }))
+ .expect("parses");
+ assert!(editor.apply(follow).expect("applies").applied);
+ assert_eq!(editor.project().active().clips[0].background, None);
+ }
+
+ /// Only a picture on the lowest lane that holds pictures has a
+ /// background: its own over the timeline's, black meaning black. A
+ /// title, a picture on a lane above, and a lane of music underneath
+ /// change nothing.
+ #[test]
+ fn the_background_belongs_to_the_lowest_picture_lane() {
+ use crate::model::Background;
+ let mut editor = Editor::new();
+ let video = editor
+ .apply(media("/a.mp4", 10.0, true))
+ .expect("adds")
+ .created_id
+ .expect("id");
+ let Command::AddMedia { mut item } = media("/song.mp3", 10.0, true) else {
+ unreachable!()
+ };
+ item.kind = MediaKind::Audio;
+ item.width = None;
+ item.height = None;
+ let song = editor
+ .apply(Command::AddMedia { item })
+ .expect("adds")
+ .created_id
+ .expect("id");
+ let tracks: Vec<String> = editor
+ .project()
+ .active()
+ .tracks
+ .iter()
+ .map(|track| track.id.clone())
+ .collect();
+ let add = |editor: &mut Editor, media_id: &str, track: usize| {
+ editor
+ .apply(Command::AddClip {
+ media_id: media_id.to_owned(),
+ track_id: tracks[track].clone(),
+ start: 0.0,
+ ripple: false,
+ })
+ .expect("adds")
+ .created_id
+ .expect("id")
+ };
+ let music = add(&mut editor, &song, 0);
+ let main = add(&mut editor, &video, 1);
+ let overlay = add(&mut editor, &video, 2);
+ let timeline_id = editor.project().active_timeline_id.clone();
+ editor
+ .apply(Command::SetTimelineBackground {
+ timeline_id,
+ background: Background::Blur { amount: 0.5 },
+ })
+ .expect("sets");
+
+ let timeline = editor.project().active();
+ assert_eq!(timeline.background_lane(), Some(tracks[1].as_str()));
+ let of = |id: &str| timeline.background_of(timeline.clip(id).expect("a clip"));
+ assert_eq!(of(&main), Some(&Background::Blur { amount: 0.5 }));
+ assert_eq!(of(&overlay), None);
+ assert_eq!(of(&music), None);
+
+ editor
+ .apply(Command::UpdateClip {
+ clip_id: main.clone(),
+ patch: ClipPatch {
+ background: Some(Some(Background::None)),
+ ..ClipPatch::default()
+ },
+ })
+ .expect("sets");
+ let timeline = editor.project().active();
+ assert_eq!(
+ timeline.background_of(timeline.clip(&main).expect("a clip")),
+ None,
+ "the clip's own black wins over the timeline's blur"
+ );
+ }
+
+ /// Fit is the size every picture starts at; fill covers the frame. A
+ /// wide shot in a tall frame fills at the frame's height over the
+ /// picture's fitted height, and both centre and unstretch the picture
+ /// as one undo step.
+ #[test]
+ fn fit_and_fill_size_a_picture_to_the_frame() {
+ use crate::model::{Framing, VideoSettings};
+ let (mut editor, _, clip_id) = fixture();
+ let timeline_id = editor.project().active_timeline_id.clone();
+ editor
+ .apply(Command::SetTimelineVideo {
+ timeline_id,
+ video: VideoSettings {
+ width: 1080,
+ height: 1920,
+ ..VideoSettings::default()
+ },
+ })
+ .expect("goes vertical");
+ editor
+ .apply(Command::SetClipTransform {
+ clip_id: clip_id.clone(),
+ scale: Some(0.5),
+ offset_x: Some(0.2),
+ offset_y: None,
+ rotation: None,
+ stretch_x: Some(1.5),
+ stretch_y: None,
+ })
+ .expect("moves");
+ editor.end_gesture();
+
+ let frame = |framing| Command::FrameClip {
+ clip_id: clip_id.clone(),
+ framing,
+ at: 0.0,
+ };
+ assert!(editor.apply(frame(Framing::Fill)).expect("fills").applied);
+ let clip = &editor.project().active().clips[0];
+ // 1920x1080 fitted into 1080x1920 is 1080x607.5; covering the
+ // frame's 1920 rows takes 1920 / 607.5.
+ assert!((clip.scale - 1920.0 / 607.5).abs() < 1e-5, "{}", clip.scale);
+ assert_eq!((clip.offset_x, clip.offset_y), (0.0, 0.0));
+ assert_eq!((clip.stretch_x, clip.stretch_y), (1.0, 1.0));
+
+ editor.apply(frame(Framing::Fit)).expect("fits");
+ assert_eq!(editor.project().active().clips[0].scale, 1.0);
+
+ editor.undo();
+ editor.undo();
+ let clip = &editor.project().active().clips[0];
+ assert_eq!((clip.scale, clip.offset_x, clip.stretch_x), (0.5, 0.2, 1.5));
+ }
+
+ /// A scale with keys keeps them: the fill lands as a key where the
+ /// playhead is, and the keys either side ride on as they were.
+ #[test]
+ fn a_fill_on_a_keyed_scale_is_a_key() {
+ use crate::model::{Framing, KeyEase, KeyProperty};
+ let (mut editor, _, clip_id) = fixture();
+ for (at, value) in [(0.0, 1.0), (1.0, 2.0)] {
+ editor
+ .apply(Command::SetClipKey {
+ clip_id: clip_id.clone(),
+ property: KeyProperty::Scale,
+ at,
+ value,
+ ease: KeyEase::LINEAR,
+ })
+ .expect("keys");
+ }
+ editor
+ .apply(Command::FrameClip {
+ clip_id: clip_id.clone(),
+ framing: Framing::Fill,
+ at: 0.5,
+ })
+ .expect("fills");
+ let clip = &editor.project().active().clips[0];
+ let keys: Vec<(f64, f64)> = clip
+ .keys_on(KeyProperty::Scale)
+ .map(|key| (key.at, key.value))
+ .collect();
+ // A 16:9 picture in a 16:9 frame already covers it.
+ assert_eq!(keys, vec![(0.0, 1.0), (0.5, 1.0), (1.0, 2.0)]);
+ assert_eq!(clip.scale, 1.0, "the constant under the keys is untouched");
+ }
+
+ /// A turned picture fills further and fits smaller, and a crop is what
+ /// is sized: a picture cropped to a square fills a square frame at one.
+ #[test]
+ fn framing_follows_the_turn_and_the_crop() {
+ use crate::model::{Framing, framing_scale};
+ let sixteen_nine = (1920.0, 1080.0);
+ let square = (1000.0, 1000.0);
+ assert_eq!(
+ framing_scale(Framing::Fit, sixteen_nine, square, 0.0),
+ Some(1.0)
+ );
+ assert_eq!(
+ framing_scale(Framing::Fill, sixteen_nine, square, 0.0),
+ Some(1.777_778)
+ );
+ // On its side, the 1000x562.5 fitted picture is 562.5 wide: it
+ // fits as it was and fills at 1000 / 562.5.
+ assert_eq!(
+ framing_scale(Framing::Fit, sixteen_nine, square, 90.0),
+ Some(1.0)
+ );
+ assert_eq!(
+ framing_scale(Framing::Fill, sixteen_nine, square, 90.0),
+ Some(1.777_778)
+ );
+ let turned = framing_scale(Framing::Fit, square, square, 45.0).expect("a size");
+ assert!((turned - 1.0 / 2f64.sqrt()).abs() < 1e-6, "{turned}");
+ assert_eq!(framing_scale(Framing::Fill, (0.0, 1.0), square, 0.0), None);
+
+ let mut clip = crate::model::Clip::blank("c", "t", ClipKind::Video, "c", 0.0, 1.0);
+ clip.crop = Some(crate::model::Crop {
+ left: 0.21875,
+ top: 0.0,
+ right: 0.21875,
+ bottom: 0.0,
+ });
+ assert_eq!(
+ clip.framing_scale(Framing::Fill, (Some(1920), Some(1080)), (1000, 1000), 0.0),
+ Some(1.0)
+ );
+ clip.kind = ClipKind::Text;
+ assert_eq!(
+ clip.framing_scale(Framing::Fill, (Some(1920), Some(1080)), (1000, 1000), 0.0),
+ None
+ );
+ }
+
/// Editor with one media item and one clip at [0, 10) on track one.
fn fixture() -> (Editor, String, String) {
let mut editor = Editor::new();
diff --git a/src/crates/concat-project/src/model.rs b/src/crates/concat-project/src/model.rs
index daa3852e..a8cdf590 100644
--- a/src/crates/concat-project/src/model.rs
+++ b/src/crates/concat-project/src/model.rs
@@ -605,6 +605,140 @@ impl Crop {
}
}
+/// What fills the frame where the picture does not reach: the bars either
+/// side of a wide shot in a tall frame, or the margin round a picture
+/// scaled down. Nothing - black, as every project was before there was a
+/// choice - a blurred, enlarged copy of the clip's own frame, a colour, or
+/// a still from the bin.
+///
+/// A timeline carries one ([`Timeline::background`]) and a clip may carry
+/// its own over it ([`Clip::background`]); which clips have one at all is
+/// [`Timeline::background_of`]'s to say.
+#[derive(Clone, PartialEq, Debug, Default, Serialize, Deserialize)]
+#[serde(
+ tag = "kind",
+ rename_all = "camelCase",
+ rename_all_fields = "camelCase"
+)]
+pub enum Background {
+ /// Black: nothing drawn behind the picture.
+ #[default]
+ None,
+ /// The clip's own frame, enlarged to cover the whole frame and blurred.
+ Blur {
+ /// How soft, `0..=1`: a blur of a twelfth of the frame's shorter
+ /// side at one.
+ amount: f64,
+ },
+ /// One flat colour, `#rrggbb`.
+ Color {
+ /// The colour, as a title's colours are written.
+ color: String,
+ },
+ /// A still from the bin, enlarged to cover the frame.
+ Image {
+ /// The bin item. One that has gone, or none yet, draws nothing.
+ media_id: String,
+ },
+}
+
+impl Background {
+ /// How soft a blur starts: soft enough that the copy behind reads as
+ /// colour and light rather than as a second picture.
+ pub const DEFAULT_BLUR: f64 = 0.5;
+
+ /// True for black, the background that draws nothing.
+ pub fn is_none(&self) -> bool {
+ *self == Background::None
+ }
+
+ /// The amount held to `0..=1` and a colour that is not `#rrggbb` turned
+ /// to black: what the document reader and every command make of a value
+ /// before it is kept. A still naming no media stays one - the kind is
+ /// chosen and the still not yet - and draws nothing.
+ pub fn tidy(self) -> Background {
+ match self {
+ Background::Blur { amount } => Background::Blur {
+ amount: if amount.is_finite() {
+ amount.clamp(0.0, 1.0)
+ } else {
+ Background::DEFAULT_BLUR
+ },
+ },
+ Background::Color { color } => Background::Color {
+ color: match rgb_of(&color) {
+ Some([r, g, b]) => format!("#{r:02x}{g:02x}{b:02x}"),
+ None => "#000000".to_owned(),
+ },
+ },
+ other => other,
+ }
+ }
+
+ /// The colour's channels, for a colour background whose colour reads.
+ pub fn rgb(&self) -> Option<[u8; 3]> {
+ match self {
+ Background::Color { color } => rgb_of(color),
+ _ => None,
+ }
+ }
+}
+
+/// `#rrggbb` (or `#rrggbbaa`, its alpha ignored) as three channels.
+fn rgb_of(hex: &str) -> Option<[u8; 3]> {
+ let digits = hex.trim().strip_prefix('#')?;
+ if !(digits.len() == 6 || digits.len() == 8) || !digits.is_ascii() {
+ return None;
+ }
+ let channel = |at: usize| u8::from_str_radix(&digits[at..at + 2], 16).ok();
+ Some([channel(0)?, channel(2)?, channel(4)?])
+}
+
+/// The two ways a picture can be sized to the frame in one go.
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum Framing {
+ /// The whole picture in the frame, as large as it goes: bars where the
+ /// shapes differ. Every picture's size before it is touched.
+ Fit,
+ /// The frame covered, the picture centred and cut at the edges that
+ /// overhang it.
+ Fill,
+}
+
+/// The scale over the fitted size at which a `picture` (its pixels, after
+/// the crop) turned by `rotation` degrees fits in or covers `frame`,
+/// centred. Scale one is contain-fitted unturned, which is what a picture
+/// is drawn at before it is scaled ([`Clip::scale`]); a turn asks more of a
+/// fill and less of a fit, since a turned rectangle's corners reach
+/// further.
+pub fn framing_scale(
+ framing: Framing,
+ picture: (f64, f64),
+ frame: (f64, f64),
+ rotation: f64,
+) -> Option<f64> {
+ let (cw, ch) = picture;
+ let (width, height) = frame;
+ if !(cw > 0.0 && ch > 0.0 && width > 0.0 && height > 0.0 && rotation.is_finite()) {
+ return None;
+ }
+ let fit = (width / cw).min(height / ch);
+ let (fw, fh) = (cw * fit, ch * fit);
+ let turn = rotation.to_radians();
+ let (cos, sin) = (turn.cos().abs(), turn.sin().abs());
+ let scale = match framing {
+ // The turned picture's bounding box inside the frame.
+ Framing::Fit => (width / (fw * cos + fh * sin)).min(height / (fw * sin + fh * cos)),
+ // The frame, turned into the picture's own axes, inside the picture.
+ Framing::Fill => ((width * cos + height * sin) / fw).max((width * sin + height * cos) / fh),
+ };
+ // Rounded to a millionth, so a fit of an unturned picture is exactly
+ // the one it has always been drawn at, not a hair under.
+ let scale = (scale * 1e6).round() / 1e6;
+ Some(scale.clamp(ranges::MIN_SCALE, ranges::MAX_SCALE))
+}
+
/// Which property a user-set key belongs to.
///
/// Five of the six are the picture's; the sixth is the mix's. They are one
@@ -1104,7 +1238,7 @@ pub(crate) mod wire {
use serde::{Deserialize, Deserializer};
use serde_json::Value;
- use super::{ClipKind, ColorSpace, MediaKind, ParamKey, ShapeKind, TextAlign};
+ use super::{Background, ClipKind, ColorSpace, MediaKind, ParamKey, ShapeKind, TextAlign};
/// The entries of a list that parse, in order; not a list at all is
/// an empty one.
@@ -1192,6 +1326,16 @@ pub(crate) mod wire {
Ok(serde_json::from_value(value).unwrap_or(TextAlign::Center))
}
+ /// A background that parses, tidied, or black for one this build does
+ /// not know - a kind from a newer build opens as no background rather
+ /// than not at all.
+ pub fn background<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Background, D::Error> {
+ let value = Value::deserialize(deserializer)?;
+ Ok(serde_json::from_value::<Background>(value)
+ .map(Background::tidy)
+ .unwrap_or_default())
+ }
+
/// A shape by name, a square for one this build does not know.
pub fn shape_kind<'de, D: Deserializer<'de>>(deserializer: D) -> Result<ShapeKind, D::Error> {
let value = Value::deserialize(deserializer)?;
@@ -1365,6 +1509,14 @@ pub struct Clip {
#[serde(default, skip_serializing_if = "Option::is_none")]
#[serde(deserialize_with = "wire::maybe")]
pub shape: Option<ShapeStyle>,
+ /// What fills the frame behind this clip's picture, over the
+ /// timeline's [`Timeline::background`]: None follows the timeline's,
+ /// and `Some(Background::None)` is black under a timeline that has one.
+ /// Only a picture on the lowest picture lane has a background at all;
+ /// see [`Timeline::background_of`].
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ #[serde(deserialize_with = "wire::maybe")]
+ pub background: Option<Background>,
/// Fields this build does not know, kept so a document written by a
/// newer or a different build round-trips through this one intact.
#[serde(flatten, default, skip_serializing_if = "Map::is_empty")]
@@ -1461,6 +1613,7 @@ impl Clip {
enhanced: None,
text: None,
shape: None,
+ background: None,
extra: Map::new(),
}
}
@@ -1534,6 +1687,7 @@ impl Clip {
}
self.text = self.text.take().map(TextStyle::tidy);
self.shape = self.shape.take().map(ShapeStyle::tidy);
+ self.background = self.background.take().map(Background::tidy);
if self.muted == Some(false) {
self.muted = None;
}
@@ -1553,6 +1707,63 @@ impl Clip {
}
}
+ /// The scale at which this clip's picture, from media `width` by
+ /// `height` and through its crop, fits in or covers a `frame`, turned as
+ /// it is at `at` (`0..=1` of the clip); see [`framing_scale`]. None for a
+ /// clip that is not a picture, or a size nobody knows.
+ pub fn framing_scale(
+ &self,
+ framing: Framing,
+ media: (Option<u32>, Option<u32>),
+ frame: (u32, u32),
+ at: f64,
+ ) -> Option<f64> {
+ if !self.kind.is_visual() {
+ return None;
+ }
+ let (width, height) = (f64::from(media.0?), f64::from(media.1?));
+ let crop = self.crop.unwrap_or_default().tidy();
+ let picture = (
+ width * (1.0 - crop.left - crop.right),
+ height * (1.0 - crop.top - crop.bottom),
+ );
+ let rotation = if self.is_keyed(KeyProperty::Rotation) {
+ self.value_at(KeyProperty::Rotation, at)
+ } else {
+ self.rotation
+ };
+ framing_scale(
+ framing,
+ picture,
+ (f64::from(frame.0), f64::from(frame.1)),
+ rotation,
+ )
+ }
+
+ /// Writes a keyable property the way an edit means it: the constant
+ /// while the property has no keys, and once it rides, a key at `at`
+ /// (`0..=1` of the clip), with the ease of the key behind it. A keyed
+ /// property no longer reads its constant, so writing that would change
+ /// nothing anyone could see.
+ pub fn write_at(&mut self, property: KeyProperty, value: f64, at: f64) {
+ if self.is_keyed(property) {
+ let ease = self
+ .keys_on(property)
+ .rfind(|key| key.at < at)
+ .map_or(KeyEase::LINEAR, |key| key.ease);
+ self.set_key(property, at, value, ease);
+ return;
+ }
+ match property {
+ KeyProperty::Scale => self.scale = value,
+ KeyProperty::OffsetX => self.offset_x = value,
+ KeyProperty::OffsetY => self.offset_y = value,
+ KeyProperty::Rotation => self.rotation = value,
+ KeyProperty::Opacity => self.opacity = value,
+ KeyProperty::Volume => self.volume = value,
+ }
+ }
+
/// This property's keys, in order. Borrowed rather than collected: the
/// caller is usually asking a question about them, not keeping them.
pub fn keys_on(
@@ -1800,6 +2011,14 @@ pub struct Timeline {
/// what a second timeline is for.
#[serde(default)]
pub video: VideoSettings,
+ /// What fills the frame behind the pictures on the lowest picture lane,
+ /// where they do not reach it; see [`Timeline::background_of`]. The
+ /// timeline's rather than [`VideoSettings`]' because that is the four
+ /// numbers a frame is, copied and compared everywhere, and this names
+ /// media. Black, and left out of the document, until one is chosen.
+ #[serde(default, skip_serializing_if = "Background::is_none")]
+ #[serde(deserialize_with = "wire::background")]
+ pub background: Background,
/// The lanes, top to bottom. Never empty - `RemoveTrack` keeps a floor
/// of one.
#[serde(deserialize_with = "wire::list")]
@@ -1824,6 +2043,7 @@ impl Default for Timeline {
id: String::new(),
name: "Timeline".to_owned(),
video: VideoSettings::default(),
+ background: Background::None,
tracks: Vec::new(),
clips: Vec::new(),
extra: Map::new(),
@@ -2035,6 +2255,7 @@ impl Project {
id: "TL1".to_owned(),
name: "Timeline 1".to_owned(),
video,
+ background: Background::None,
tracks: (1..=4)
.map(|number| Track {
id: format!("T{number}"),
@@ -2162,6 +2383,46 @@ impl Timeline {
self.tracks.iter().find(|track| track.id == track_id)
}
+ /// The lane whose pictures have a background: the lowest lane holding a
+ /// video or a still - the main picture of the cut, where CapCut has its
+ /// main track. Lowest rather than simply the first lane, so a cut whose
+ /// bottom lane holds only music still has one. None for a timeline
+ /// without a picture.
+ pub fn background_lane(&self) -> Option<&str> {
+ self.tracks
+ .iter()
+ .find(|track| {
+ self.clips
+ .iter()
+ .any(|clip| clip.track_id == track.id && clip.kind.is_visual())
+ })
+ .map(|track| track.id.as_str())
+ }
+
+ /// What fills the frame behind `clip`: its own background, else the
+ /// timeline's, for a video or a still on [`Timeline::background_lane`];
+ /// None for any other clip, and for one whose background is black.
+ /// Titles, shapes and pictures on the lanes above sit over the frame the
+ /// main picture and its background have already filled, so they never
+ /// have one of their own.
+ pub fn background_of<'a>(&'a self, clip: &'a Clip) -> Option<&'a Background> {
+ self.background_on(clip, self.background_lane())
+ }
+
+ /// [`Timeline::background_of`] with the lane found once, by
+ /// [`Timeline::background_lane`], for a caller asking of every clip.
+ pub fn background_on<'a>(
+ &'a self,
+ clip: &'a Clip,
+ lane: Option<&str>,
+ ) -> Option<&'a Background> {
+ if !clip.kind.is_visual() || lane != Some(clip.track_id.as_str()) {
+ return None;
+ }
+ let background = clip.background.as_ref().unwrap_or(&self.background);
+ (!background.is_none()).then_some(background)
+ }
+
/// Where the last clip ends. Zero for an empty timeline.
pub fn duration(&self) -> f64 {
self.clips
diff --git a/src/crates/concat-render/src/gpu/tests.rs b/src/crates/concat-render/src/gpu/tests.rs
index d8a98b20..16bdb59b 100644
--- a/src/crates/concat-render/src/gpu/tests.rs
+++ b/src/crates/concat-render/src/gpu/tests.rs
@@ -15,7 +15,7 @@ use concat_core::timeline::{Blend, Transform};
use super::*;
use crate::CpuCompositor;
use crate::metrics::ssim;
-use crate::plan::{Crop, Transition, detached_clip};
+use crate::plan::{Backdrop, Crop, Transition, detached_clip};
fn gpu() -> Option<WgpuCompositor> {
let compositor = WgpuCompositor::new();
@@ -1854,3 +1854,194 @@ fn the_largest_frame_is_what_the_device_holds() {
concat_core::frame::MAX_SIDE.min(gpu.device.limits().max_texture_dimension_2d)
);
}
+
+// ── backgrounds ──
+
+/// A wide picture, red on its left half and blue on its right, the way a
+/// landscape shot sits in a vertical frame.
+fn halves(width: u32, height: u32) -> Frame {
+ let mut frame = Frame::transparent(width, height);
+ for y in 0..height {
+ for x in 0..width {
+ let colour = if x < width / 2 {
+ [255, 0, 0, 255]
+ } else {
+ [0, 0, 255, 255]
+ };
+ frame.set_pixel(x, y, colour);
+ }
+ }
+ frame
+}
+
+/// The standard normal's distribution function, by Abramowitz and
+/// Stegun's 7.1.26: to a few parts in ten million, far finer than a
+/// frame's eight bits.
+fn normal_cdf(x: f64) -> f64 {
+ let z = x.abs() / std::f64::consts::SQRT_2;
+ let t = 1.0 / (1.0 + 0.327_591_1 * z);
+ let poly = t
+ * (0.254_829_592
+ + t * (-0.284_496_736
+ + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429))));
+ let erf = 1.0 - poly * (-z * z).exp();
+ if x >= 0.0 {
+ 0.5 * (1.0 + erf)
+ } else {
+ 0.5 * (1.0 - erf)
+ }
+}
+
+/// An output level back to light, the inverse of the resolve's 1/2.4.
+fn light(level: u8) -> f64 {
+ (f64::from(level) / 255.0).powf(2.4)
+}
+
+/// A blurred background is the picture enlarged to cover the frame and
+/// blurred by the spread asked for, beneath the picture and nowhere over
+/// it: in the bars of a wide red-and-blue picture in a tall frame, the
+/// light along a row is the step from red to blue at the frame's middle,
+/// blurred by a Gaussian of `sigma` output pixels, while the picture
+/// itself comes out exactly as it does over black.
+#[test]
+fn a_blurred_background_is_the_picture_covering_the_frame_blurred() {
+ let Some(mut gpu) = gpu() else { return };
+ let blur = concat_effects::Catalogue::builtin()
+ .get("concat.gaussian-blur")
+ .expect("a built-in");
+ let (width, height) = (90, 160);
+ let sigma = 12.0;
+ let picture = layer(halves(64, 36));
+ let backdrop = picture
+ .backdrop(&Backdrop::Blur { sigma }, width, height, |radius| {
+ blur.pass(
+ &BTreeMap::from([("radius".to_owned(), f64::from(radius))]),
+ None,
+ )
+ })
+ .expect("a picture to blur");
+ assert_eq!(backdrop.effects.len(), 1, "the blur, and only the blur");
+
+ let alone = gpu.render(&plan(width, height, vec![picture.clone()]));
+ let over = gpu.render(&plan(width, height, vec![backdrop, picture]));
+
+ // 64x36 fitted into 90 across is 90x51, from row 54: the picture.
+ for y in 58..100 {
+ for x in 0..width {
+ assert_eq!(
+ over.pixel(x, y),
+ alone.pixel(x, y),
+ "the picture at {x},{y}"
+ );
+ }
+ }
+ // Above and below it, the bars: black alone, the blurred step over it.
+ let mut worst = 0.0f64;
+ for y in [10, 30, 130, 150] {
+ assert_eq!(alone.pixel(45, y), Some([0, 0, 0, 255]), "a bar alone");
+ for x in 0..width {
+ let blue = normal_cdf((f64::from(x) + 0.5 - f64::from(width) / 2.0) / f64::from(sigma));
+ let [r, g, b, a] = over.pixel(x, y).expect("in the frame");
+ assert_eq!((g, a), (0, 255), "at {x},{y}");
+ for (got, want) in [(light(r), 1.0 - blue), (light(b), blue)] {
+ worst = worst.max((got - want).abs());
+ assert!(
+ (got - want).abs() < 0.02,
+ "at {x},{y}: light {got:.4} where the blurred step is {want:.4}"
+ );
+ }
+ }
+ }
+ eprintln!("blurred background: worst {worst:.4} of light off the Gaussian step");
+}
+
+/// A colour background fills the bars with the colour, exactly, and a
+/// still covers the frame in its own shape: a tall still behind a wide
+/// picture in a wide frame fills the bars from its middle band.
+#[test]
+fn a_colour_or_a_still_fills_the_bars() {
+ let Some(mut gpu) = gpu() else { return };
+ let (width, height) = (90, 160);
+ let picture = layer(halves(64, 36));
+ let none = |_| None;
+ let colour = picture
+ .backdrop(&Backdrop::Colour([10, 200, 90]), width, height, none)
+ .expect("a colour");
+ let frame = gpu.render(&plan(width, height, vec![colour, picture.clone()]));
+ for (x, y) in [(0, 0), (89, 0), (45, 20), (0, 159), (89, 159), (45, 140)] {
+ assert_eq!(frame.pixel(x, y), Some([10, 200, 90, 255]), "at {x},{y}");
+ }
+ assert_eq!(frame.pixel(20, 80), Some([255, 0, 0, 255]));
+ assert_eq!(frame.pixel(70, 80), Some([0, 0, 255, 255]));
+
+ // A still of three bands, top to bottom, twice as tall as it is wide:
+ // covering the 90x160 frame it is 90 across and 180 down, so the frame
+ // shows the middle band whole and the outer two cut short.
+ let mut still = Frame::transparent(20, 40);
+ for y in 0..40 {
+ for x in 0..20 {
+ let band = match y {
+ 0..10 => [255, 255, 0, 255],
+ 10..30 => [0, 255, 255, 255],
+ _ => [255, 0, 255, 255],
+ };
+ still.set_pixel(x, y, band);
+ }
+ }
+ let image = picture
+ .backdrop(&Backdrop::Image(Arc::new(still)), width, height, none)
+ .expect("a still");
+ let frame = gpu.render(&plan(width, height, vec![image, picture]));
+ assert_eq!(frame.pixel(45, 40), Some([0, 255, 255, 255]));
+ assert_eq!(frame.pixel(45, 120), Some([0, 255, 255, 255]));
+ assert_eq!(frame.pixel(45, 2), Some([255, 255, 0, 255]));
+ assert_eq!(frame.pixel(45, 157), Some([255, 0, 255, 255]));
+}
+
+/// The background goes with its picture through a wipe: halfway, the half
+/// of the frame the picture has gone from is black, bars and all, and the
+/// edge crosses the bars where it crosses the picture.
+#[test]
+fn a_background_is_wiped_with_its_picture() {
+ let Some(mut gpu) = gpu() else { return };
+ let (width, height) = (90, 160);
+ let mut picture = layer(halves(64, 36));
+ picture.transitions = vec![Transition::Wipe {
+ uncovered: 0.5,
+ from_right: false,
+ }];
+ let colour = picture
+ .backdrop(&Backdrop::Colour([0, 255, 0]), width, height, |_| None)
+ .expect("a colour");
+ let frame = gpu.render(&plan(width, height, vec![colour, picture]));
+ assert_eq!(
+ frame.pixel(20, 80),
+ Some([255, 0, 0, 255]),
+ "the picture's kept half"
+ );
+ assert_eq!(
+ frame.pixel(70, 80),
+ Some([0, 0, 0, 255]),
+ "the picture's wiped half"
+ );
+ assert_eq!(
+ frame.pixel(20, 20),
+ Some([0, 255, 0, 255]),
+ "the bar's kept half"
+ );
+ assert_eq!(
+ frame.pixel(70, 20),
+ Some([0, 0, 0, 255]),
+ "the bar's wiped half"
+ );
+ assert_eq!(
+ frame.pixel(43, 20),
+ Some([0, 255, 0, 255]),
+ "the edge is the picture's"
+ );
+ assert_eq!(
+ frame.pixel(47, 20),
+ Some([0, 0, 0, 255]),
+ "the edge is the picture's"
+ );
+}
diff --git a/src/crates/concat-render/src/lib.rs b/src/crates/concat-render/src/lib.rs
index 109e09e4..153a2517 100644
--- a/src/crates/concat-render/src/lib.rs
+++ b/src/crates/concat-render/src/lib.rs
@@ -31,8 +31,8 @@ pub use compositor::Compositor;
pub use gpu::WgpuCompositor;
pub use metrics::ssim;
pub use plan::{
- Crop, FramePlan, Geometry, PlannedLayer, PlannedTreatment, Shading, Transition, detached_clip,
- plan_frame,
+ Backdrop, Crop, FramePlan, Geometry, PlannedLayer, PlannedTreatment, Shading, Transition,
+ detached_clip, plan_frame,
};
#[cfg(test)]
pub(crate) use reference::CpuCompositor;
diff --git a/src/crates/concat-render/src/plan.rs b/src/crates/concat-render/src/plan.rs
index 0337cfd1..68c8ac84 100644
--- a/src/crates/concat-render/src/plan.rs
+++ b/src/crates/concat-render/src/plan.rs
@@ -289,6 +289,199 @@ impl PlannedLayer {
}
}
+/// What fills the frame behind a picture where the picture does not reach
+/// it - the bars beside a wide shot in a tall frame - as a compositor draws
+/// it: one more layer, beneath the picture's own, built by
+/// [`PlannedLayer::backdrop`]. Nothing here is new to the compositors, so
+/// the monitor and the export cannot draw it two ways.
+#[derive(Clone, Debug)]
+pub enum Backdrop {
+ /// The picture itself, run through its effects, enlarged to cover the
+ /// frame and blurred: a Gaussian of `sigma` output pixels.
+ Blur {
+ /// The blur's spread, in pixels of the output frame.
+ sigma: f32,
+ },
+ /// One opaque colour.
+ Colour([u8; 3]),
+ /// A still, enlarged to cover the frame.
+ Image(Arc<Frame>),
+}
+
+impl PlannedLayer {
+ /// True when the picture is certain to hide everything beneath it: drawn
+ /// whole and solid, unturned, and placed over the entire frame. What
+ /// spares a picture that already fills the frame a background nobody
+ /// would see. `opaque` is the caller's word that the pixels themselves
+ /// have no holes - a still may, and so may a cutout or any effect,
+ /// which is why a layer with effects is never taken to cover.
+ pub fn covers(&self, width: u32, height: u32, opaque: bool) -> bool {
+ let Some(source) = &self.source else {
+ return false;
+ };
+ if !opaque
+ || self.weight() < 1.0
+ || !self.effects.is_empty()
+ || self.mask.is_some()
+ || !self.transitions.is_empty()
+ || self.blend != Blend::Normal
+ {
+ return false;
+ }
+ let geometry = self.geometry(source, width, height);
+ if geometry.rotation != 0.0 {
+ return false;
+ }
+ let half = (
+ geometry.fitted.0 as f32 * geometry.scale.0 / 2.0,
+ geometry.fitted.1 as f32 * geometry.scale.1 / 2.0,
+ );
+ // Half a pixel either way: a picture whose edge lands inside the
+ // last pixel still covers its centre, which is all a sampler reads.
+ geometry.centre.0 - half.0 <= 0.5
+ && geometry.centre.0 + half.0 >= width as f32 - 0.5
+ && geometry.centre.1 - half.1 <= 0.5
+ && geometry.centre.1 + half.1 >= height as f32 - 0.5
+ }
+
+ /// The layer `backdrop` draws as under this one, in a `width` by
+ /// `height` frame, or None where it has nothing to draw with - a blur
+ /// of a layer with no picture yet.
+ ///
+ /// It covers the frame, centred, whatever the picture's own scale,
+ /// position and turn: the background stays put while the picture moves
+ /// over it. It belongs to the picture all the same: it is on the
+ /// picture's track, as strong as the picture is at this frame, and goes
+ /// through the picture's fades and wipes, so a dissolve or a fade to
+ /// black takes the background with the picture rather than leaving it
+ /// behind. `blur` makes the Gaussian pass for a radius in the picture's
+ /// own pixels; the plan is a description and knows no package.
+ pub fn backdrop(
+ &self,
+ backdrop: &Backdrop,
+ width: u32,
+ height: u32,
+ blur: impl FnOnce(f32) -> Option<ShaderPass>,
+ ) -> Option<PlannedLayer> {
+ let mut layer = PlannedLayer {
+ transform: Transform::default(),
+ mask: None,
+ blend: Blend::Normal,
+ ..self.clone()
+ };
+ let (picture, effects) = match backdrop {
+ Backdrop::Blur { sigma } => {
+ let source = self.source.clone()?;
+ let cover = cover_of(&layer, &source, width, height);
+ // The spread asked for on screen, in the pixels the blur
+ // runs at: the picture's fitted size, before the cover
+ // enlarges it.
+ let radius = sigma / cover;
+ let mut effects = self.effects.clone();
+ if radius >= 0.5 {
+ effects.extend(blur(radius));
+ }
+ (source, effects)
+ }
+ Backdrop::Colour([r, g, b]) => {
+ let mut solid = Frame::black(16, 16);
+ solid.fill([*r, *g, *b, 255]);
+ layer.crop = Crop::NONE;
+ layer.flip_h = false;
+ layer.flip_v = false;
+ (Arc::new(solid), Vec::new())
+ }
+ Backdrop::Image(still) => {
+ layer.crop = Crop::NONE;
+ layer.flip_h = false;
+ layer.flip_v = false;
+ (Arc::clone(still), Vec::new())
+ }
+ };
+ let cover = cover_of(&layer, &picture, width, height);
+ // A colour stretches to the frame's shape; a picture keeps its own.
+ let stretch = match backdrop {
+ Backdrop::Colour(_) => {
+ let fitted = layer.geometry(&picture, width, height).fitted;
+ (
+ f64::from(width) / f64::from(fitted.0) / f64::from(cover),
+ f64::from(height) / f64::from(fitted.1) / f64::from(cover),
+ )
+ }
+ _ => (1.0, 1.0),
+ };
+ // Two pixels over on the shorter side: the fit centres by whole
+ // pixels, and a cover that met the edge exactly could leave a hair
+ // of black along it.
+ let over = 1.0 + 2.0 / width.min(height).max(1) as f64;
+ layer.transform = Transform {
+ scale: f64::from(cover) * over,
+ stretch_x: stretch.0,
+ stretch_y: stretch.1,
+ ..Transform::default()
+ };
+ layer.transitions = self.backdrop_transitions(&layer, &picture, width, height);
+ layer.source = Some(picture);
+ layer.effects = effects;
+ Some(layer)
+ }
+
+ /// This layer's transitions as they fall on `backdrop` drawn from
+ /// `picture`: the fades as they are, and each wipe's edge moved to where
+ /// it crosses the frame over the picture, since the background is wider
+ /// than the picture and its own `u` runs across more of the frame.
+ fn backdrop_transitions(
+ &self,
+ backdrop: &PlannedLayer,
+ picture: &Frame,
+ width: u32,
+ height: u32,
+ ) -> Vec<Transition> {
+ let wiped = self
+ .transitions
+ .iter()
+ .any(|transition| matches!(transition, Transition::Wipe { .. }));
+ let Some(source) = self.source.as_ref().filter(|_| wiped) else {
+ return self.transitions.clone();
+ };
+ let span = |layer: &PlannedLayer, frame: &Frame| {
+ let geometry = layer.geometry(frame, width, height);
+ let wide = geometry.fitted.0 as f32 * geometry.scale.0;
+ (geometry.centre.0 - wide / 2.0, wide.max(1e-3))
+ };
+ let (own_left, own_wide) = span(self, source);
+ let (left, wide) = span(backdrop, picture);
+ let across = |u: f32| ((own_left + u * own_wide - left) / wide).clamp(0.0, 1.0);
+ self.transitions
+ .iter()
+ .map(|transition| match *transition {
+ Transition::Wipe {
+ uncovered,
+ from_right: false,
+ } => Transition::Wipe {
+ uncovered: across(uncovered),
+ from_right: false,
+ },
+ Transition::Wipe {
+ uncovered,
+ from_right: true,
+ } => Transition::Wipe {
+ uncovered: 1.0 - across(1.0 - uncovered),
+ from_right: true,
+ },
+ fade => fade,
+ })
+ .collect()
+ }
+}
+
+/// How much larger than its fitted size `picture` must be drawn, through
+/// `layer`'s crop, to cover a `width` by `height` frame.
+fn cover_of(layer: &PlannedLayer, picture: &Frame, width: u32, height: u32) -> f32 {
+ let fitted = Geometry::of(layer, picture.width(), picture.height(), width, height).fitted;
+ (width as f32 / fitted.0 as f32).max(height as f32 / fitted.1 as f32)
+}
+
/// Where a layer's picture comes from and where it lands: the arithmetic
/// both compositors share, so a picture cannot be placed two ways.
///
@@ -717,6 +910,28 @@ mod tests {
assert_eq!(cropped.uv_of(1.0, 1.0, false, false), (0.75, 1.0));
}
+ /// A picture covers the frame only drawn whole and solid over every
+ /// pixel of it: a wide picture in a wide frame does, in a tall one only
+ /// once it is scaled to fill, and never at half strength, turned, with
+ /// effects, or when its pixels may have holes.
+ #[test]
+ fn a_picture_covers_the_frame_only_solid_and_whole() {
+ let mut layer = PlannedLayer::picture(a_clip(), Arc::new(Frame::black(64, 36)));
+ assert!(layer.covers(64, 36, true));
+ assert!(!layer.covers(64, 36, false), "a still may have holes");
+ assert!(!layer.covers(36, 64, true));
+ layer.transform.scale = 3.2;
+ assert!(layer.covers(36, 64, true), "filled");
+ layer.transform.offset_y = 0.1;
+ assert!(!layer.covers(36, 64, true), "moved off an edge");
+ layer.transform.offset_y = 0.0;
+ layer.transform.rotation = 2.0;
+ assert!(!layer.covers(36, 64, true));
+ layer.transform.rotation = 0.0;
+ layer.opacity = 0.5;
+ assert!(!layer.covers(36, 64, true));
+ }
+
/// Fades compose into one map of the colour and wipes into one edge a
/// side, so two fades to black at half are three quarters black and a
/// wipe keeps only what its edge has uncovered.