The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
feat(render): a background behind the main picture, and Fit and Fill (#232, #255)
Jareer03dac1f98de5
diff --git a/src/crates/concat-export/src/flatten.rs b/src/crates/concat-export/src/flatten.rsindex 3c2416a0..0157f8c2 100644--- a/src/crates/concat-export/src/flatten.rs+++ b/src/crates/concat-export/src/flatten.rs@@ -15,10 +15,12 @@use std::path::Path;-use concat_project::model::{ClipKind as ModelClipKind, KeyProperty, Project, Timeline};+use concat_project::model::{+ Background, ClipKind as ModelClipKind, KeyProperty, MediaKind, Project, Timeline,+};use crate::chains::audio_filter_chain;-use crate::{ClipKind, ExportClip, ExportKey, TransitionSpec};+use crate::{ClipKind, ExportBackdrop, ExportClip, ExportKey, TransitionSpec};/// Flattens one timeline of `project` for the exporter - the active one/// when `timeline_id` is `None`. Text clips are excluded (they rasterise@@ -39,6 +41,8 @@ pub fn flatten_timeline_in(let Some(timeline) = pick_timeline(project, timeline_id) else {return Vec::new();};+ // Found once: asked of every clip, it is a walk of every clip.+ let lane = timeline.background_lane();timeline.clips@@ -133,6 +137,9 @@ pub fn flatten_timeline_in(media_height: media.height,media_duration: media.duration,has_audio: Some(media.has_audio),+ backdrop: timeline+ .background_on(clip, lane)+ .and_then(|background| backdrop_of(project, background)),cutout: clip.cutout.clone(),mask_dir: match (&clip.cutout, project_dir) {(Some(cutout), Some(dir)) => {@@ -148,6 +155,26 @@ pub fn flatten_timeline_in(.collect()}+/// A background as it is drawn: a blur and a colour as they are, a still+/// found in the bin. None for black, and for a still that is no longer in+/// the bin or is not a still - a background that cannot be drawn is no+/// background, never a failed frame.+fn backdrop_of(project: &Project, background: &Background) -> Option<ExportBackdrop> {+ match background {+ Background::None => None,+ Background::Blur { amount } => Some(ExportBackdrop::Blur { amount: *amount }),+ Background::Color { .. } => background.rgb().map(|rgb| ExportBackdrop::Colour { rgb }),+ Background::Image { media_id } => project+ .media_by_id(media_id)+ .filter(|item| item.kind == MediaKind::Image)+ .map(|item| ExportBackdrop::Image {+ path: item.path.clone(),+ width: item.width,+ height: item.height,+ }),+ }+}+/// The timeline to flatten: the named one, or the document's active one,/// falling back to the first - the same preference the UI's tab strip has.fn pick_timeline<'a>(project: &'a Project, timeline_id: Option<&str>) -> Option<&'a Timeline> {diff --git a/src/crates/concat-export/src/lib.rs b/src/crates/concat-export/src/lib.rsindex 1e92e548..cf6d6964 100644--- a/src/crates/concat-export/src/lib.rs+++ b/src/crates/concat-export/src/lib.rs@@ -226,6 +226,53 @@ pub struct ExportClip {/// it back through `reveal_order()`.#[serde(skip)]pub reveal_map: Option<Arc<RevealMap>>,+ /// What fills the frame behind the picture where it does not reach, as+ /// the flattener resolved it - the clip's own background or its+ /// timeline's, for a picture on the lowest picture lane; absent for+ /// black and for every other clip. See `Timeline::background_of`.+ #[serde(default)]+ pub backdrop: Option<ExportBackdrop>,+}++/// A clip's background, resolved for drawing: the document's+/// `Background` with its colour read and its still found in the bin.+#[derive(Deserialize, Clone, PartialEq, Debug)]+#[serde(+ tag = "kind",+ rename_all = "camelCase",+ rename_all_fields = "camelCase"+)]+pub enum ExportBackdrop {+ /// The clip's own picture, enlarged to cover the frame and blurred.+ Blur {+ /// How soft, `0..=1`: a Gaussian of a twelfth of the frame's+ /// shorter side at one.+ amount: f64,+ },+ /// One opaque colour.+ Colour {+ /// Its channels.+ rgb: [u8; 3],+ },+ /// A still, enlarged to cover the frame.+ Image {+ /// The file.+ path: String,+ /// Its pixel width, where the bin knows it: what it is decoded at.+ width: Option<u32>,+ /// Its pixel height.+ height: Option<u32>,+ },+}++impl ExportBackdrop {+ /// The blur's spread, in pixels of a `width` by `height` frame, at+ /// `amount`: a twelfth of the shorter side at the most, so the same+ /// amount is the same softness at every size the frame is drawn at -+ /// the monitor's quarter and the export's whole alike.+ pub fn sigma(amount: f64, width: u32, height: u32) -> f32 {+ (amount.clamp(0.0, 1.0) * f64::from(width.min(height)) / 12.0) as f32+ }}impl ExportClip {@@ -277,6 +324,7 @@ impl ExportClip {mask_dir: String::new(),highlighted: false,reveal_map: None,+ backdrop: None,}}}@@ -1308,6 +1356,89 @@ fn shapes_at(}}+/// `layers` with every picture's background beneath it, in a `width` by+/// `height` frame: the one place the export and the monitor both put+/// backgrounds in, so the two cannot draw them two ways. A picture that+/// already hides the whole frame gets none - `opaque` says whose pixels+/// have no holes to see one through - and a still that cannot be read is+/// no background rather than a failed frame.+fn with_backdrops(+ layers: Vec<PlannedLayer>,+ backdrops: &HashMap<ClipId, ExportBackdrop>,+ opaque: impl Fn(ClipId) -> bool,+ pool: &concat_media::ReaderPool,+ (width, height): (u32, u32),+) -> Vec<PlannedLayer> {+ if backdrops.is_empty() {+ return layers;+ }+ let mut out = Vec::with_capacity(layers.len() + 1);+ for layer in layers {+ let drawn = backdrops+ .get(&layer.clip)+ .filter(|_| !layer.covers(width, height, opaque(layer.clip)))+ .and_then(|backdrop| backdrop_of(backdrop, pool, (width, height)))+ .and_then(|backdrop| layer.backdrop(&backdrop, width, height, blur_pass));+ out.extend(drawn);+ out.push(layer);+ }+ out+}++/// A clip's background as the frame plan draws it, at a `width` by+/// `height` frame: the blur's spread in that frame's pixels, a still read+/// through the pool at a size that covers it.+fn backdrop_of(+ backdrop: &ExportBackdrop,+ pool: &concat_media::ReaderPool,+ (width, height): (u32, u32),+) -> Option<concat_render::Backdrop> {+ match backdrop {+ ExportBackdrop::Blur { amount } => Some(concat_render::Backdrop::Blur {+ sigma: ExportBackdrop::sigma(*amount, width, height),+ }),+ ExportBackdrop::Colour { rgb } => Some(concat_render::Backdrop::Colour(*rgb)),+ ExportBackdrop::Image {+ path,+ width: still_width,+ height: still_height,+ } => {+ // Read at the size that covers the frame, its own shape kept;+ // the frame's when the bin never learnt the still's.+ let size = match (still_width, still_height) {+ (Some(w), Some(h)) if *w > 0 && *h > 0 => {+ let cover =+ (f64::from(width) / f64::from(*w)).max(f64::from(height) / f64::from(*h));+ (+ ((f64::from(*w) * cover).round() as u32).max(2),+ ((f64::from(*h) * cover).round() as u32).max(2),+ )+ }+ _ => (width, height),+ };+ let request = concat_media::FrameRequest::new(path, Rational::ZERO, size.0, size.1)+ .as_still(true)+ .at_any_size(true);+ match pool.frame(&request) {+ Ok(still) => Some(concat_render::Backdrop::Image(still)),+ Err(error) => {+ log::debug!("background image {path}: {error}");+ None+ }+ }+ }+ }+}++/// The Gaussian blur's pass at `radius` pixels: the package every blurred+/// background is drawn with, so it blurs exactly as the effect does.+fn blur_pass(radius: f32) -> Option<ShaderPass> {+ Catalogue::builtin().get("concat.gaussian-blur")?.pass(+ &BTreeMap::from([("radius".to_owned(), f64::from(radius))]),+ None,+ )+}+/// A stack already drawn, as a layer over the whole frame on the lowest/// track: texel for pixel, since it is the frame's own size.fn ground_layer(ground: Frame) -> PlannedLayer {@@ -1472,6 +1603,7 @@ fn render_picture(reveal_maps,cutouts,highlight: _,+ backdrops,} = build_timeline(request, rate, visible, transitions);let options = EncodeOptions {@@ -1640,7 +1772,13 @@ fn render_picture(time,width: request.width,height: request.height,- layers,+ layers: with_backdrops(+ layers,+ &backdrops,+ |clip| !stills.contains(&clip) && !cutouts.contains_key(&clip),+ &sought,+ (request.width, request.height),+ ),treatments: Vec::new(),output: output_of(request.color_space),};@@ -2077,6 +2215,7 @@ pub fn preview_sources_of(reveal_maps,cutouts,highlight,+ backdrops,} = &plan.built;let highlight = *highlight;let time = quantise(seconds, rate);@@ -2138,7 +2277,13 @@ pub fn preview_sources_of(time,width: plan.width,height: plan.height,- layers,+ layers: with_backdrops(+ layers,+ backdrops,+ |clip| !stills.contains(&clip) && !cutouts.contains_key(&clip),+ pool,+ (plan.width, plan.height),+ ),treatments: Vec::new(),output: plan.output,};diff --git a/src/crates/concat-export/src/resolve.rs b/src/crates/concat-export/src/resolve.rsindex ac8f92ed..31013621 100644--- a/src/crates/concat-export/src/resolve.rs+++ b/src/crates/concat-export/src/resolve.rs@@ -115,6 +115,9 @@ pub(crate) struct BuiltTimeline {pub(crate) transitions: Vec<TransitionSpan>,/// The clips whose background a mask takes away.pub(crate) cutouts: HashMap<ClipId, CutoutJob>,+ /// What fills the frame behind each picture that has a background; see+ /// [`crate::with_backdrops`].+ pub(crate) backdrops: HashMap<ClipId, crate::ExportBackdrop>,}/// A packaged transition over a cut. The incoming clip has been overlapped@@ -212,6 +215,7 @@ pub(crate) fn build_timeline(let mut chains: HashMap<ClipId, Vec<AppliedFilter>> = HashMap::new();let mut reveal_maps: HashMap<ClipId, Arc<RevealMap>> = HashMap::new();let mut cutouts: HashMap<ClipId, CutoutJob> = HashMap::new();+ let mut backdrops: HashMap<ClipId, crate::ExportBackdrop> = HashMap::new();let mut highlight: Option<ClipId> = None;let lanes = visible.iter().map(|clip| clip.track).max().unwrap_or(0) + 1;@@ -303,6 +307,9 @@ pub(crate) fn build_timeline(if clip.highlighted {highlight = Some(id);}+ if let Some(backdrop) = &clip.backdrop {+ backdrops.insert(id, backdrop.clone());+ }}}@@ -320,6 +327,7 @@ pub(crate) fn build_timeline(reveal_maps,cutouts,highlight,+ backdrops,}}diff --git a/src/crates/concat-host/tests/export.rs b/src/crates/concat-host/tests/export.rsindex e99c8bc7..9b78fb12 100644--- a/src/crates/concat-host/tests/export.rs+++ b/src/crates/concat-host/tests/export.rs@@ -860,6 +860,145 @@ fn crop_and_flips_reach_the_picture() {);}+/// A background fills the bars of a wide picture in a tall frame - a+/// colour, or the picture itself enlarged and blurred - and the monitor+/// shows what the export writes: the paused monitor's frame, drawn from the+/// same flattened clips, agrees with the file at every point checked. A+/// fill then takes the bars away altogether.+#[test]+fn a_background_fills_the_bars_in_the_export_and_the_monitor() {+ use concat_project::model::{Background, Framing};+ let scratch = Scratch::new("background");+ let path = scratch.path().join("quadrants.mp4");+ quadrants(&path);+ let [red, green, blue, yellow] = QUADRANTS;+ let black = [0, 0, 0];+ let (width, height) = (90, 160);++ let mut studio = Studio::new(scratch.path(), "Background", video(width, height, 30, 1));+ let media = studio.import(&path);+ let clip = studio+ .apply(Command::AddClipAtFirstFree {+ media_id: media,+ start: 0.0,+ })+ .expect("the clip has an id");+ let timeline_id = studio.project().active_timeline_id.clone();+ // The picture is 90x51 across the middle of the frame: its quadrants+ // either side of (0.5, 0.5), the bars above 0.34 and below 0.66.+ let picture = [+ ((0.25, 0.42), red),+ ((0.75, 0.42), green),+ ((0.25, 0.58), blue),+ ((0.75, 0.58), yellow),+ ];+ let monitor = |studio: &Studio| {+ let request = concat_export::PreviewFrameRequest {+ time: 1.0,+ width,+ height,+ rate_num: 30,+ rate_den: 1,+ clips: studio.session.flattened_clips(),+ color_space: Default::default(),+ };+ let pixels =+ concat_export::preview_frame(&concat_media::ReaderPool::with_defaults(), &request)+ .expect("the monitor draws the frame");+ Frame::from_rgba(width, height, pixels).expect("a whole frame")+ };+ let agree = |label: &str, exported: &Exported, shown: &Frame| {+ for y in [0.1, 0.25, 0.42, 0.58, 0.75, 0.9] {+ for x in [0.1, 0.25, 0.4, 0.6, 0.75, 0.9] {+ let file = exported.colour_at_point(1.0, x, y);+ let [r, g, b, _] = shown+ .pixel(+ (f64::from(width) * x) as u32,+ (f64::from(height) * y) as u32,+ )+ .expect("inside the frame");+ assert!(+ file.iter()+ .zip([r, g, b])+ .all(|(file, shown)| file.abs_diff(shown) <= 24),+ "{label}: at ({x}, {y}) the file is {file:?} and the monitor {:?}",+ [r, g, b]+ );+ }+ }+ };++ let plain = studio.export("no background");+ plain.expect_colours(1.0, &picture);+ plain.expect_colours(1.0, &[((0.5, 0.1), black), ((0.5, 0.9), black)]);+ agree("no background", &plain, &monitor(&studio));++ let set = |studio: &mut Studio, background: Background| {+ studio.apply(Command::SetTimelineBackground {+ timeline_id: timeline_id.clone(),+ background,+ });+ };+ let teal = [30, 160, 140];+ set(+ &mut studio,+ Background::Color {+ color: "#1ea08c".to_owned(),+ },+ );+ let coloured = studio.export("a colour background");+ coloured.expect_colours(1.0, &picture);+ coloured.expect_colours(1.0, &[((0.5, 0.1), teal), ((0.5, 0.9), teal)]);+ agree("a colour background", &coloured, &monitor(&studio));++ // Enlarged to cover the frame, the picture's middle third fills it:+ // the left quadrants on the left, the right ones on the right, red and+ // green above, blue and yellow below, a little softened.+ set(&mut studio, Background::Blur { amount: 0.3 });+ let blurred = studio.export("a blurred background");+ blurred.expect_colours(1.0, &picture);+ blurred.expect_colours(+ 1.0,+ &[+ ((0.15, 0.1), red),+ ((0.85, 0.1), green),+ ((0.15, 0.9), blue),+ ((0.85, 0.9), yellow),+ ],+ );+ agree("a blurred background", &blurred, &monitor(&studio));++ // The clip's own black over the timeline's blur.+ studio.apply(Command::UpdateClip {+ clip_id: clip.clone(),+ patch: ClipPatch {+ background: Some(Some(Background::None)),+ ..ClipPatch::default()+ },+ });+ studio+ .export("the clip's own black")+ .expect_colours(1.0, &[((0.5, 0.1), black), ((0.5, 0.9), black)]);++ // Filled, the picture is the frame: no bars to fill.+ studio.apply(Command::FrameClip {+ clip_id: clip,+ framing: Framing::Fill,+ at: 0.0,+ });+ let filled = studio.export("filled");+ filled.expect_colours(+ 1.0,+ &[+ ((0.4, 0.1), red),+ ((0.6, 0.1), green),+ ((0.4, 0.9), blue),+ ((0.6, 0.9), yellow),+ ],+ );+ agree("filled", &filled, &monitor(&studio));+}+/// Three seconds of one flat colour, silent.fn solid(path: &Path, colour: [u8; 3]) {let options = EncodeOptions {diff --git a/src/crates/concat-project/src/commands/mod.rs b/src/crates/concat-project/src/commands/mod.rsindex 68212129..d663d511 100644--- a/src/crates/concat-project/src/commands/mod.rs+++ b/src/crates/concat-project/src/commands/mod.rs@@ -13,9 +13,10 @@ use std::sync::Arc;use serde::{Deserialize, Serialize};use crate::model::{- AppliedFilter, AudioTrack, Clip, ClipKind, ColorRange, ColorSpace, Crop, CustomFont, Cutout,- CutoutMode, Enhanced, KeyEase, KeyProperty, MediaItem, MediaKind, MediaOrigin, Project,- ShapeStyle, SpeedPoint, Stroke, TextStyle, Timeline, Track, Transition, VideoSettings,+ AppliedFilter, AudioTrack, Background, Clip, ClipKind, ColorRange, ColorSpace, Crop,+ CustomFont, Cutout, CutoutMode, Enhanced, Framing, KeyEase, KeyProperty, MediaItem, MediaKind,+ MediaOrigin, Project, ShapeStyle, SpeedPoint, Stroke, TextStyle, Timeline, Track, Transition,+ VideoSettings,};mod audio;@@ -150,6 +151,15 @@ pub struct ClipPatch {skip_serializing_if = "Option::is_none")]pub audio_stream: Option<Option<u32>>,+ /// The clip's own background over the timeline's: absent leaves it+ /// alone, null goes back to following the timeline's, a value - black+ /// included - is the clip's own. See `Clip::background`.+ #[serde(+ default,+ with = "double_option",+ skip_serializing_if = "Option::is_none"+ )]+ pub background: Option<Option<Background>>,}/// `Option<Option<T>>` over JSON: absent → None, null → Some(None).@@ -621,6 +631,22 @@ pub enum Command {#[serde(default)]stretch_y: Option<f64>,},+ /// Sizes a picture to the frame in one step: fitted whole inside it,+ /// or covering it, centred either way and unstretched (see+ /// [`crate::model::framing_scale`]; a turned picture is sized as turned).+ /// A scale or a position with keys gets a key at `at` rather than a new+ /// constant, the way every edit of a keyed property lands, so the rest+ /// of the ride is left as it was. A no-op for a clip that is not a video+ /// or a still, or whose media's size is unknown.+ FrameClip {+ /// The clip to size. An unknown id is a no-op.+ clip_id: String,+ /// Fit or fill.+ framing: Framing,+ /// Where in the clip, `0..=1`, a keyed property takes its key.+ #[serde(default)]+ at: f64,+ },/// Pulls a video clip's sound out into its own audio clip on a free/// lane (minting one if none is free), muting the video and moving its/// audio filters to the sound. A no-op unless the clip is an unmuted@@ -676,6 +702,15 @@ pub enum Command {/// The frame and the rate, together.video: VideoSettings,},+ /// Sets what fills a timeline's frame behind the pictures on its lowest+ /// picture lane; see `Timeline::background_of`. Tidied first: an amount+ /// held to `0..=1`, a colour that does not read made black.+ SetTimelineBackground {+ /// The timeline. An unknown id is a no-op.+ timeline_id: String,+ /// The background.+ background: Background,+ },/// Renames a timeline tab. Whitespace-only names are ignored so a tab/// can never end up blank; unknown ids are tolerated.RenameTimeline {@@ -991,6 +1026,10 @@ impl Command {.iter().flatten().any(|tr| bad([tr.duration]))+ || matches!(+ patch.background,+ Some(Some(Background::Blur { amount })) if !amount.is_finite()+ )|| patch.text.iter().flatten().any(|style| {bad([style.font_size,@@ -1005,6 +1044,11 @@ impl Command {})}Command::SetClipSpeed { speed, .. } => bad([*speed]),+ Command::FrameClip { at, .. } => bad([*at]),+ Command::SetTimelineBackground {+ background: Background::Blur { amount },+ ..+ } => bad([*amount]),Command::SetClipTransform {scale,offset_x,@@ -1102,7 +1146,8 @@ pub fn apply(| Command::ClearEffectKey { .. }| Command::ClearEffectKeys { .. }| Command::SetClipSpeedCurve { .. }- | Command::SetClipTransform { .. }) => properties::apply(project, mint, command),+ | Command::SetClipTransform { .. }+ | Command::FrameClip { .. }) => properties::apply(project, mint, command),command @ (Command::DetachAudio { .. } | Command::ReattachAudio { .. }) => {audio::apply(project, mint, command)}@@ -1111,6 +1156,7 @@ pub fn apply(| Command::SetTrackFlag { .. }) => tracks::apply(project, mint, command),command @ (Command::AddTimeline| Command::SetTimelineVideo { .. }+ | Command::SetTimelineBackground { .. }| Command::RemoveTimeline { .. }| Command::RenameTimeline { .. }| Command::SelectTimeline { .. }diff --git a/src/crates/concat-project/src/commands/properties.rs b/src/crates/concat-project/src/commands/properties.rsindex d24c8756..9417fc83 100644--- a/src/crates/concat-project/src/commands/properties.rs+++ b/src/crates/concat-project/src/commands/properties.rs@@ -81,6 +81,9 @@ pub(super) fn apply(if let Some(stream) = patch.audio_stream {applied |= assign(&mut clip.audio_stream, stream);}+ if let Some(background) = patch.background {+ applied |= assign(&mut clip.background, background.map(Background::tidy));+ }Ok(Outcome {created_id: None,applied,@@ -344,6 +347,41 @@ pub(super) fn apply(})}+ Command::FrameClip {+ clip_id,+ framing,+ at,+ } => {+ let timeline = project.active();+ let frame = (timeline.video.width, timeline.video.height);+ let Some(media) = timeline+ .clip(&clip_id)+ .and_then(|clip| project.media_by_id(&clip.media_id))+ .map(|item| (item.width, item.height))+ else {+ return Ok(Outcome::default());+ };+ let Some(clip) = project.active_mut().clip_mut(&clip_id) else {+ return Ok(Outcome::default());+ };+ let at = at.clamp(0.0, 1.0);+ let Some(scale) = clip.framing_scale(framing, media, frame, at) else {+ return Ok(Outcome::default());+ };+ let before = clip.clone();+ clip.write_at(KeyProperty::Scale, scale, at);+ clip.write_at(KeyProperty::OffsetX, 0.0, at);+ clip.write_at(KeyProperty::OffsetY, 0.0, at);+ // A stretch is a picture pulled out of its shape, and neither a+ // fit nor a fill is: both are the picture as it was shot.+ clip.stretch_x = 1.0;+ clip.stretch_y = 1.0;+ Ok(Outcome {+ created_id: None,+ applied: *clip != before,+ })+ }+_ => unreachable!("commands::apply routes only this module's commands here"),}}diff --git a/src/crates/concat-project/src/commands/timelines.rs b/src/crates/concat-project/src/commands/timelines.rsindex 7adb10cf..d2569d04 100644--- a/src/crates/concat-project/src/commands/timelines.rs+++ b/src/crates/concat-project/src/commands/timelines.rs@@ -36,10 +36,14 @@ pub(super) fn apply(// likeliest second timeline is another cut of the same picture,// and the one that is not is a sheet away from being told so.let video = project.active().video;+ // And with its background, for the same reason: a second cut of+ // a vertical edit wants the same bars filled the same way.+ let background = project.active().background.clone();project.timelines.push(Arc::new(Timeline {id: id.clone(),name,video,+ background,tracks,clips: Vec::new(),extra: Default::default(),@@ -66,6 +70,23 @@ pub(super) fn apply(})}+ Command::SetTimelineBackground {+ timeline_id,+ background,+ } => {+ let applied = project+ .timelines+ .iter_mut()+ .find(|timeline| timeline.id == timeline_id)+ .is_some_and(|timeline| {+ assign(&mut Arc::make_mut(timeline).background, background.tidy())+ });+ Ok(Outcome {+ created_id: None,+ applied,+ })+ }+Command::RemoveTimeline { timeline_id } => {if project.timelines.len() <= 1 {return Err(CommandError::LastTimeline);diff --git a/src/crates/concat-project/src/lib.rs b/src/crates/concat-project/src/lib.rsindex bf488194..974fc454 100644--- a/src/crates/concat-project/src/lib.rs+++ b/src/crates/concat-project/src/lib.rs@@ -283,6 +283,288 @@ mod tests {assert!(loaded.project().media[0].color_space.is_sdr());}+ /// A document from before backgrounds opens with none, on the timeline+ /// and on every clip, and a project that never chooses one writes the+ /// document it always wrote. One that does choose one reads it back,+ /// and a kind from a newer build opens as none rather than failing the+ /// load.+ #[test]+ fn a_background_is_absent_until_chosen_and_round_trips() {+ use crate::model::Background;+ let (mut editor, _, clip_id) = fixture();+ let document = editor.to_document(&settings());+ let timeline = &document["timelines"][0];+ assert!(timeline.get("background").is_none(), "{timeline}");+ assert!(timeline["clips"][0].get("background").is_none());+ let loaded = Editor::from_document(&document).expect("loads");+ assert!(loaded.project().active().background.is_none());+ assert_eq!(loaded.project().active().clips[0].background, None);++ let timeline_id = editor.project().active_timeline_id.clone();+ editor+ .apply(Command::SetTimelineBackground {+ timeline_id: timeline_id.clone(),+ background: Background::Blur { amount: 3.0 },+ })+ .expect("sets");+ assert_eq!(+ editor.project().active().background,+ Background::Blur { amount: 1.0 },+ "held to 0..=1"+ );+ editor+ .apply(Command::UpdateClip {+ clip_id: clip_id.clone(),+ patch: ClipPatch {+ background: Some(Some(Background::Color {+ color: "#FF8000".to_owned(),+ })),+ ..ClipPatch::default()+ },+ })+ .expect("sets");+ let mut document = editor.to_document(&settings());+ assert_eq!(+ document["timelines"][0]["background"],+ json!({ "kind": "blur", "amount": 1.0 })+ );+ assert_eq!(+ document["timelines"][0]["clips"][0]["background"],+ json!({ "kind": "color", "color": "#ff8000" })+ );+ let loaded = Editor::from_document(&document).expect("loads");+ assert_eq!(loaded.project(), editor.project());++ document["timelines"][0]["background"] = json!({ "kind": "hologram", "depth": 4 });+ document["timelines"][0]["clips"][0]["background"] = json!({ "kind": "hologram" });+ let loaded = Editor::from_document(&document).expect("loads");+ assert!(loaded.project().active().background.is_none());+ assert_eq!(loaded.project().active().clips[0].background, None);++ // Null on the wire is "follow the timeline's" again.+ let follow: Command = serde_json::from_value(json!({+ "op": "updateClip",+ "clipId": clip_id,+ "patch": { "background": null }+ }))+ .expect("parses");+ assert!(editor.apply(follow).expect("applies").applied);+ assert_eq!(editor.project().active().clips[0].background, None);+ }++ /// Only a picture on the lowest lane that holds pictures has a+ /// background: its own over the timeline's, black meaning black. A+ /// title, a picture on a lane above, and a lane of music underneath+ /// change nothing.+ #[test]+ fn the_background_belongs_to_the_lowest_picture_lane() {+ use crate::model::Background;+ let mut editor = Editor::new();+ let video = editor+ .apply(media("/a.mp4", 10.0, true))+ .expect("adds")+ .created_id+ .expect("id");+ let Command::AddMedia { mut item } = media("/song.mp3", 10.0, true) else {+ unreachable!()+ };+ item.kind = MediaKind::Audio;+ item.width = None;+ item.height = None;+ let song = editor+ .apply(Command::AddMedia { item })+ .expect("adds")+ .created_id+ .expect("id");+ let tracks: Vec<String> = editor+ .project()+ .active()+ .tracks+ .iter()+ .map(|track| track.id.clone())+ .collect();+ let add = |editor: &mut Editor, media_id: &str, track: usize| {+ editor+ .apply(Command::AddClip {+ media_id: media_id.to_owned(),+ track_id: tracks[track].clone(),+ start: 0.0,+ ripple: false,+ })+ .expect("adds")+ .created_id+ .expect("id")+ };+ let music = add(&mut editor, &song, 0);+ let main = add(&mut editor, &video, 1);+ let overlay = add(&mut editor, &video, 2);+ let timeline_id = editor.project().active_timeline_id.clone();+ editor+ .apply(Command::SetTimelineBackground {+ timeline_id,+ background: Background::Blur { amount: 0.5 },+ })+ .expect("sets");++ let timeline = editor.project().active();+ assert_eq!(timeline.background_lane(), Some(tracks[1].as_str()));+ let of = |id: &str| timeline.background_of(timeline.clip(id).expect("a clip"));+ assert_eq!(of(&main), Some(&Background::Blur { amount: 0.5 }));+ assert_eq!(of(&overlay), None);+ assert_eq!(of(&music), None);++ editor+ .apply(Command::UpdateClip {+ clip_id: main.clone(),+ patch: ClipPatch {+ background: Some(Some(Background::None)),+ ..ClipPatch::default()+ },+ })+ .expect("sets");+ let timeline = editor.project().active();+ assert_eq!(+ timeline.background_of(timeline.clip(&main).expect("a clip")),+ None,+ "the clip's own black wins over the timeline's blur"+ );+ }++ /// Fit is the size every picture starts at; fill covers the frame. A+ /// wide shot in a tall frame fills at the frame's height over the+ /// picture's fitted height, and both centre and unstretch the picture+ /// as one undo step.+ #[test]+ fn fit_and_fill_size_a_picture_to_the_frame() {+ use crate::model::{Framing, VideoSettings};+ let (mut editor, _, clip_id) = fixture();+ let timeline_id = editor.project().active_timeline_id.clone();+ editor+ .apply(Command::SetTimelineVideo {+ timeline_id,+ video: VideoSettings {+ width: 1080,+ height: 1920,+ ..VideoSettings::default()+ },+ })+ .expect("goes vertical");+ editor+ .apply(Command::SetClipTransform {+ clip_id: clip_id.clone(),+ scale: Some(0.5),+ offset_x: Some(0.2),+ offset_y: None,+ rotation: None,+ stretch_x: Some(1.5),+ stretch_y: None,+ })+ .expect("moves");+ editor.end_gesture();++ let frame = |framing| Command::FrameClip {+ clip_id: clip_id.clone(),+ framing,+ at: 0.0,+ };+ assert!(editor.apply(frame(Framing::Fill)).expect("fills").applied);+ let clip = &editor.project().active().clips[0];+ // 1920x1080 fitted into 1080x1920 is 1080x607.5; covering the+ // frame's 1920 rows takes 1920 / 607.5.+ assert!((clip.scale - 1920.0 / 607.5).abs() < 1e-5, "{}", clip.scale);+ assert_eq!((clip.offset_x, clip.offset_y), (0.0, 0.0));+ assert_eq!((clip.stretch_x, clip.stretch_y), (1.0, 1.0));++ editor.apply(frame(Framing::Fit)).expect("fits");+ assert_eq!(editor.project().active().clips[0].scale, 1.0);++ editor.undo();+ editor.undo();+ let clip = &editor.project().active().clips[0];+ assert_eq!((clip.scale, clip.offset_x, clip.stretch_x), (0.5, 0.2, 1.5));+ }++ /// A scale with keys keeps them: the fill lands as a key where the+ /// playhead is, and the keys either side ride on as they were.+ #[test]+ fn a_fill_on_a_keyed_scale_is_a_key() {+ use crate::model::{Framing, KeyEase, KeyProperty};+ let (mut editor, _, clip_id) = fixture();+ for (at, value) in [(0.0, 1.0), (1.0, 2.0)] {+ editor+ .apply(Command::SetClipKey {+ clip_id: clip_id.clone(),+ property: KeyProperty::Scale,+ at,+ value,+ ease: KeyEase::LINEAR,+ })+ .expect("keys");+ }+ editor+ .apply(Command::FrameClip {+ clip_id: clip_id.clone(),+ framing: Framing::Fill,+ at: 0.5,+ })+ .expect("fills");+ let clip = &editor.project().active().clips[0];+ let keys: Vec<(f64, f64)> = clip+ .keys_on(KeyProperty::Scale)+ .map(|key| (key.at, key.value))+ .collect();+ // A 16:9 picture in a 16:9 frame already covers it.+ assert_eq!(keys, vec![(0.0, 1.0), (0.5, 1.0), (1.0, 2.0)]);+ assert_eq!(clip.scale, 1.0, "the constant under the keys is untouched");+ }++ /// A turned picture fills further and fits smaller, and a crop is what+ /// is sized: a picture cropped to a square fills a square frame at one.+ #[test]+ fn framing_follows_the_turn_and_the_crop() {+ use crate::model::{Framing, framing_scale};+ let sixteen_nine = (1920.0, 1080.0);+ let square = (1000.0, 1000.0);+ assert_eq!(+ framing_scale(Framing::Fit, sixteen_nine, square, 0.0),+ Some(1.0)+ );+ assert_eq!(+ framing_scale(Framing::Fill, sixteen_nine, square, 0.0),+ Some(1.777_778)+ );+ // On its side, the 1000x562.5 fitted picture is 562.5 wide: it+ // fits as it was and fills at 1000 / 562.5.+ assert_eq!(+ framing_scale(Framing::Fit, sixteen_nine, square, 90.0),+ Some(1.0)+ );+ assert_eq!(+ framing_scale(Framing::Fill, sixteen_nine, square, 90.0),+ Some(1.777_778)+ );+ let turned = framing_scale(Framing::Fit, square, square, 45.0).expect("a size");+ assert!((turned - 1.0 / 2f64.sqrt()).abs() < 1e-6, "{turned}");+ assert_eq!(framing_scale(Framing::Fill, (0.0, 1.0), square, 0.0), None);++ let mut clip = crate::model::Clip::blank("c", "t", ClipKind::Video, "c", 0.0, 1.0);+ clip.crop = Some(crate::model::Crop {+ left: 0.21875,+ top: 0.0,+ right: 0.21875,+ bottom: 0.0,+ });+ assert_eq!(+ clip.framing_scale(Framing::Fill, (Some(1920), Some(1080)), (1000, 1000), 0.0),+ Some(1.0)+ );+ clip.kind = ClipKind::Text;+ assert_eq!(+ clip.framing_scale(Framing::Fill, (Some(1920), Some(1080)), (1000, 1000), 0.0),+ None+ );+ }+/// Editor with one media item and one clip at [0, 10) on track one.fn fixture() -> (Editor, String, String) {let mut editor = Editor::new();diff --git a/src/crates/concat-project/src/model.rs b/src/crates/concat-project/src/model.rsindex daa3852e..a8cdf590 100644--- a/src/crates/concat-project/src/model.rs+++ b/src/crates/concat-project/src/model.rs@@ -605,6 +605,140 @@ impl Crop {}}+/// What fills the frame where the picture does not reach: the bars either+/// side of a wide shot in a tall frame, or the margin round a picture+/// scaled down. Nothing - black, as every project was before there was a+/// choice - a blurred, enlarged copy of the clip's own frame, a colour, or+/// a still from the bin.+///+/// A timeline carries one ([`Timeline::background`]) and a clip may carry+/// its own over it ([`Clip::background`]); which clips have one at all is+/// [`Timeline::background_of`]'s to say.+#[derive(Clone, PartialEq, Debug, Default, Serialize, Deserialize)]+#[serde(+ tag = "kind",+ rename_all = "camelCase",+ rename_all_fields = "camelCase"+)]+pub enum Background {+ /// Black: nothing drawn behind the picture.+ #[default]+ None,+ /// The clip's own frame, enlarged to cover the whole frame and blurred.+ Blur {+ /// How soft, `0..=1`: a blur of a twelfth of the frame's shorter+ /// side at one.+ amount: f64,+ },+ /// One flat colour, `#rrggbb`.+ Color {+ /// The colour, as a title's colours are written.+ color: String,+ },+ /// A still from the bin, enlarged to cover the frame.+ Image {+ /// The bin item. One that has gone, or none yet, draws nothing.+ media_id: String,+ },+}++impl Background {+ /// How soft a blur starts: soft enough that the copy behind reads as+ /// colour and light rather than as a second picture.+ pub const DEFAULT_BLUR: f64 = 0.5;++ /// True for black, the background that draws nothing.+ pub fn is_none(&self) -> bool {+ *self == Background::None+ }++ /// The amount held to `0..=1` and a colour that is not `#rrggbb` turned+ /// to black: what the document reader and every command make of a value+ /// before it is kept. A still naming no media stays one - the kind is+ /// chosen and the still not yet - and draws nothing.+ pub fn tidy(self) -> Background {+ match self {+ Background::Blur { amount } => Background::Blur {+ amount: if amount.is_finite() {+ amount.clamp(0.0, 1.0)+ } else {+ Background::DEFAULT_BLUR+ },+ },+ Background::Color { color } => Background::Color {+ color: match rgb_of(&color) {+ Some([r, g, b]) => format!("#{r:02x}{g:02x}{b:02x}"),+ None => "#000000".to_owned(),+ },+ },+ other => other,+ }+ }++ /// The colour's channels, for a colour background whose colour reads.+ pub fn rgb(&self) -> Option<[u8; 3]> {+ match self {+ Background::Color { color } => rgb_of(color),+ _ => None,+ }+ }+}++/// `#rrggbb` (or `#rrggbbaa`, its alpha ignored) as three channels.+fn rgb_of(hex: &str) -> Option<[u8; 3]> {+ let digits = hex.trim().strip_prefix('#')?;+ if !(digits.len() == 6 || digits.len() == 8) || !digits.is_ascii() {+ return None;+ }+ let channel = |at: usize| u8::from_str_radix(&digits[at..at + 2], 16).ok();+ Some([channel(0)?, channel(2)?, channel(4)?])+}++/// The two ways a picture can be sized to the frame in one go.+#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)]+#[serde(rename_all = "camelCase")]+pub enum Framing {+ /// The whole picture in the frame, as large as it goes: bars where the+ /// shapes differ. Every picture's size before it is touched.+ Fit,+ /// The frame covered, the picture centred and cut at the edges that+ /// overhang it.+ Fill,+}++/// The scale over the fitted size at which a `picture` (its pixels, after+/// the crop) turned by `rotation` degrees fits in or covers `frame`,+/// centred. Scale one is contain-fitted unturned, which is what a picture+/// is drawn at before it is scaled ([`Clip::scale`]); a turn asks more of a+/// fill and less of a fit, since a turned rectangle's corners reach+/// further.+pub fn framing_scale(+ framing: Framing,+ picture: (f64, f64),+ frame: (f64, f64),+ rotation: f64,+) -> Option<f64> {+ let (cw, ch) = picture;+ let (width, height) = frame;+ if !(cw > 0.0 && ch > 0.0 && width > 0.0 && height > 0.0 && rotation.is_finite()) {+ return None;+ }+ let fit = (width / cw).min(height / ch);+ let (fw, fh) = (cw * fit, ch * fit);+ let turn = rotation.to_radians();+ let (cos, sin) = (turn.cos().abs(), turn.sin().abs());+ let scale = match framing {+ // The turned picture's bounding box inside the frame.+ Framing::Fit => (width / (fw * cos + fh * sin)).min(height / (fw * sin + fh * cos)),+ // The frame, turned into the picture's own axes, inside the picture.+ Framing::Fill => ((width * cos + height * sin) / fw).max((width * sin + height * cos) / fh),+ };+ // Rounded to a millionth, so a fit of an unturned picture is exactly+ // the one it has always been drawn at, not a hair under.+ let scale = (scale * 1e6).round() / 1e6;+ Some(scale.clamp(ranges::MIN_SCALE, ranges::MAX_SCALE))+}+/// Which property a user-set key belongs to.////// Five of the six are the picture's; the sixth is the mix's. They are one@@ -1104,7 +1238,7 @@ pub(crate) mod wire {use serde::{Deserialize, Deserializer};use serde_json::Value;- use super::{ClipKind, ColorSpace, MediaKind, ParamKey, ShapeKind, TextAlign};+ use super::{Background, ClipKind, ColorSpace, MediaKind, ParamKey, ShapeKind, TextAlign};/// The entries of a list that parse, in order; not a list at all is/// an empty one.@@ -1192,6 +1326,16 @@ pub(crate) mod wire {Ok(serde_json::from_value(value).unwrap_or(TextAlign::Center))}+ /// A background that parses, tidied, or black for one this build does+ /// not know - a kind from a newer build opens as no background rather+ /// than not at all.+ pub fn background<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Background, D::Error> {+ let value = Value::deserialize(deserializer)?;+ Ok(serde_json::from_value::<Background>(value)+ .map(Background::tidy)+ .unwrap_or_default())+ }+/// A shape by name, a square for one this build does not know.pub fn shape_kind<'de, D: Deserializer<'de>>(deserializer: D) -> Result<ShapeKind, D::Error> {let value = Value::deserialize(deserializer)?;@@ -1365,6 +1509,14 @@ pub struct Clip {#[serde(default, skip_serializing_if = "Option::is_none")]#[serde(deserialize_with = "wire::maybe")]pub shape: Option<ShapeStyle>,+ /// What fills the frame behind this clip's picture, over the+ /// timeline's [`Timeline::background`]: None follows the timeline's,+ /// and `Some(Background::None)` is black under a timeline that has one.+ /// Only a picture on the lowest picture lane has a background at all;+ /// see [`Timeline::background_of`].+ #[serde(default, skip_serializing_if = "Option::is_none")]+ #[serde(deserialize_with = "wire::maybe")]+ pub background: Option<Background>,/// Fields this build does not know, kept so a document written by a/// newer or a different build round-trips through this one intact.#[serde(flatten, default, skip_serializing_if = "Map::is_empty")]@@ -1461,6 +1613,7 @@ impl Clip {enhanced: None,text: None,shape: None,+ background: None,extra: Map::new(),}}@@ -1534,6 +1687,7 @@ impl Clip {}self.text = self.text.take().map(TextStyle::tidy);self.shape = self.shape.take().map(ShapeStyle::tidy);+ self.background = self.background.take().map(Background::tidy);if self.muted == Some(false) {self.muted = None;}@@ -1553,6 +1707,63 @@ impl Clip {}}+ /// The scale at which this clip's picture, from media `width` by+ /// `height` and through its crop, fits in or covers a `frame`, turned as+ /// it is at `at` (`0..=1` of the clip); see [`framing_scale`]. None for a+ /// clip that is not a picture, or a size nobody knows.+ pub fn framing_scale(+ &self,+ framing: Framing,+ media: (Option<u32>, Option<u32>),+ frame: (u32, u32),+ at: f64,+ ) -> Option<f64> {+ if !self.kind.is_visual() {+ return None;+ }+ let (width, height) = (f64::from(media.0?), f64::from(media.1?));+ let crop = self.crop.unwrap_or_default().tidy();+ let picture = (+ width * (1.0 - crop.left - crop.right),+ height * (1.0 - crop.top - crop.bottom),+ );+ let rotation = if self.is_keyed(KeyProperty::Rotation) {+ self.value_at(KeyProperty::Rotation, at)+ } else {+ self.rotation+ };+ framing_scale(+ framing,+ picture,+ (f64::from(frame.0), f64::from(frame.1)),+ rotation,+ )+ }++ /// Writes a keyable property the way an edit means it: the constant+ /// while the property has no keys, and once it rides, a key at `at`+ /// (`0..=1` of the clip), with the ease of the key behind it. A keyed+ /// property no longer reads its constant, so writing that would change+ /// nothing anyone could see.+ pub fn write_at(&mut self, property: KeyProperty, value: f64, at: f64) {+ if self.is_keyed(property) {+ let ease = self+ .keys_on(property)+ .rfind(|key| key.at < at)+ .map_or(KeyEase::LINEAR, |key| key.ease);+ self.set_key(property, at, value, ease);+ return;+ }+ match property {+ KeyProperty::Scale => self.scale = value,+ KeyProperty::OffsetX => self.offset_x = value,+ KeyProperty::OffsetY => self.offset_y = value,+ KeyProperty::Rotation => self.rotation = value,+ KeyProperty::Opacity => self.opacity = value,+ KeyProperty::Volume => self.volume = value,+ }+ }+/// This property's keys, in order. Borrowed rather than collected: the/// caller is usually asking a question about them, not keeping them.pub fn keys_on(@@ -1800,6 +2011,14 @@ pub struct Timeline {/// what a second timeline is for.#[serde(default)]pub video: VideoSettings,+ /// What fills the frame behind the pictures on the lowest picture lane,+ /// where they do not reach it; see [`Timeline::background_of`]. The+ /// timeline's rather than [`VideoSettings`]' because that is the four+ /// numbers a frame is, copied and compared everywhere, and this names+ /// media. Black, and left out of the document, until one is chosen.+ #[serde(default, skip_serializing_if = "Background::is_none")]+ #[serde(deserialize_with = "wire::background")]+ pub background: Background,/// The lanes, top to bottom. Never empty - `RemoveTrack` keeps a floor/// of one.#[serde(deserialize_with = "wire::list")]@@ -1824,6 +2043,7 @@ impl Default for Timeline {id: String::new(),name: "Timeline".to_owned(),video: VideoSettings::default(),+ background: Background::None,tracks: Vec::new(),clips: Vec::new(),extra: Map::new(),@@ -2035,6 +2255,7 @@ impl Project {id: "TL1".to_owned(),name: "Timeline 1".to_owned(),video,+ background: Background::None,tracks: (1..=4).map(|number| Track {id: format!("T{number}"),@@ -2162,6 +2383,46 @@ impl Timeline {self.tracks.iter().find(|track| track.id == track_id)}+ /// The lane whose pictures have a background: the lowest lane holding a+ /// video or a still - the main picture of the cut, where CapCut has its+ /// main track. Lowest rather than simply the first lane, so a cut whose+ /// bottom lane holds only music still has one. None for a timeline+ /// without a picture.+ pub fn background_lane(&self) -> Option<&str> {+ self.tracks+ .iter()+ .find(|track| {+ self.clips+ .iter()+ .any(|clip| clip.track_id == track.id && clip.kind.is_visual())+ })+ .map(|track| track.id.as_str())+ }++ /// What fills the frame behind `clip`: its own background, else the+ /// timeline's, for a video or a still on [`Timeline::background_lane`];+ /// None for any other clip, and for one whose background is black.+ /// Titles, shapes and pictures on the lanes above sit over the frame the+ /// main picture and its background have already filled, so they never+ /// have one of their own.+ pub fn background_of<'a>(&'a self, clip: &'a Clip) -> Option<&'a Background> {+ self.background_on(clip, self.background_lane())+ }++ /// [`Timeline::background_of`] with the lane found once, by+ /// [`Timeline::background_lane`], for a caller asking of every clip.+ pub fn background_on<'a>(+ &'a self,+ clip: &'a Clip,+ lane: Option<&str>,+ ) -> Option<&'a Background> {+ if !clip.kind.is_visual() || lane != Some(clip.track_id.as_str()) {+ return None;+ }+ let background = clip.background.as_ref().unwrap_or(&self.background);+ (!background.is_none()).then_some(background)+ }+/// Where the last clip ends. Zero for an empty timeline.pub fn duration(&self) -> f64 {self.clipsdiff --git a/src/crates/concat-render/src/gpu/tests.rs b/src/crates/concat-render/src/gpu/tests.rsindex d8a98b20..16bdb59b 100644--- a/src/crates/concat-render/src/gpu/tests.rs+++ b/src/crates/concat-render/src/gpu/tests.rs@@ -15,7 +15,7 @@ use concat_core::timeline::{Blend, Transform};use super::*;use crate::CpuCompositor;use crate::metrics::ssim;-use crate::plan::{Crop, Transition, detached_clip};+use crate::plan::{Backdrop, Crop, Transition, detached_clip};fn gpu() -> Option<WgpuCompositor> {let compositor = WgpuCompositor::new();@@ -1854,3 +1854,194 @@ fn the_largest_frame_is_what_the_device_holds() {concat_core::frame::MAX_SIDE.min(gpu.device.limits().max_texture_dimension_2d));}++// ── backgrounds ──++/// A wide picture, red on its left half and blue on its right, the way a+/// landscape shot sits in a vertical frame.+fn halves(width: u32, height: u32) -> Frame {+ let mut frame = Frame::transparent(width, height);+ for y in 0..height {+ for x in 0..width {+ let colour = if x < width / 2 {+ [255, 0, 0, 255]+ } else {+ [0, 0, 255, 255]+ };+ frame.set_pixel(x, y, colour);+ }+ }+ frame+}++/// The standard normal's distribution function, by Abramowitz and+/// Stegun's 7.1.26: to a few parts in ten million, far finer than a+/// frame's eight bits.+fn normal_cdf(x: f64) -> f64 {+ let z = x.abs() / std::f64::consts::SQRT_2;+ let t = 1.0 / (1.0 + 0.327_591_1 * z);+ let poly = t+ * (0.254_829_592+ + t * (-0.284_496_736+ + t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429))));+ let erf = 1.0 - poly * (-z * z).exp();+ if x >= 0.0 {+ 0.5 * (1.0 + erf)+ } else {+ 0.5 * (1.0 - erf)+ }+}++/// An output level back to light, the inverse of the resolve's 1/2.4.+fn light(level: u8) -> f64 {+ (f64::from(level) / 255.0).powf(2.4)+}++/// A blurred background is the picture enlarged to cover the frame and+/// blurred by the spread asked for, beneath the picture and nowhere over+/// it: in the bars of a wide red-and-blue picture in a tall frame, the+/// light along a row is the step from red to blue at the frame's middle,+/// blurred by a Gaussian of `sigma` output pixels, while the picture+/// itself comes out exactly as it does over black.+#[test]+fn a_blurred_background_is_the_picture_covering_the_frame_blurred() {+ let Some(mut gpu) = gpu() else { return };+ let blur = concat_effects::Catalogue::builtin()+ .get("concat.gaussian-blur")+ .expect("a built-in");+ let (width, height) = (90, 160);+ let sigma = 12.0;+ let picture = layer(halves(64, 36));+ let backdrop = picture+ .backdrop(&Backdrop::Blur { sigma }, width, height, |radius| {+ blur.pass(+ &BTreeMap::from([("radius".to_owned(), f64::from(radius))]),+ None,+ )+ })+ .expect("a picture to blur");+ assert_eq!(backdrop.effects.len(), 1, "the blur, and only the blur");++ let alone = gpu.render(&plan(width, height, vec![picture.clone()]));+ let over = gpu.render(&plan(width, height, vec![backdrop, picture]));++ // 64x36 fitted into 90 across is 90x51, from row 54: the picture.+ for y in 58..100 {+ for x in 0..width {+ assert_eq!(+ over.pixel(x, y),+ alone.pixel(x, y),+ "the picture at {x},{y}"+ );+ }+ }+ // Above and below it, the bars: black alone, the blurred step over it.+ let mut worst = 0.0f64;+ for y in [10, 30, 130, 150] {+ assert_eq!(alone.pixel(45, y), Some([0, 0, 0, 255]), "a bar alone");+ for x in 0..width {+ let blue = normal_cdf((f64::from(x) + 0.5 - f64::from(width) / 2.0) / f64::from(sigma));+ let [r, g, b, a] = over.pixel(x, y).expect("in the frame");+ assert_eq!((g, a), (0, 255), "at {x},{y}");+ for (got, want) in [(light(r), 1.0 - blue), (light(b), blue)] {+ worst = worst.max((got - want).abs());+ assert!(+ (got - want).abs() < 0.02,+ "at {x},{y}: light {got:.4} where the blurred step is {want:.4}"+ );+ }+ }+ }+ eprintln!("blurred background: worst {worst:.4} of light off the Gaussian step");+}++/// A colour background fills the bars with the colour, exactly, and a+/// still covers the frame in its own shape: a tall still behind a wide+/// picture in a wide frame fills the bars from its middle band.+#[test]+fn a_colour_or_a_still_fills_the_bars() {+ let Some(mut gpu) = gpu() else { return };+ let (width, height) = (90, 160);+ let picture = layer(halves(64, 36));+ let none = |_| None;+ let colour = picture+ .backdrop(&Backdrop::Colour([10, 200, 90]), width, height, none)+ .expect("a colour");+ let frame = gpu.render(&plan(width, height, vec![colour, picture.clone()]));+ for (x, y) in [(0, 0), (89, 0), (45, 20), (0, 159), (89, 159), (45, 140)] {+ assert_eq!(frame.pixel(x, y), Some([10, 200, 90, 255]), "at {x},{y}");+ }+ assert_eq!(frame.pixel(20, 80), Some([255, 0, 0, 255]));+ assert_eq!(frame.pixel(70, 80), Some([0, 0, 255, 255]));++ // A still of three bands, top to bottom, twice as tall as it is wide:+ // covering the 90x160 frame it is 90 across and 180 down, so the frame+ // shows the middle band whole and the outer two cut short.+ let mut still = Frame::transparent(20, 40);+ for y in 0..40 {+ for x in 0..20 {+ let band = match y {+ 0..10 => [255, 255, 0, 255],+ 10..30 => [0, 255, 255, 255],+ _ => [255, 0, 255, 255],+ };+ still.set_pixel(x, y, band);+ }+ }+ let image = picture+ .backdrop(&Backdrop::Image(Arc::new(still)), width, height, none)+ .expect("a still");+ let frame = gpu.render(&plan(width, height, vec![image, picture]));+ assert_eq!(frame.pixel(45, 40), Some([0, 255, 255, 255]));+ assert_eq!(frame.pixel(45, 120), Some([0, 255, 255, 255]));+ assert_eq!(frame.pixel(45, 2), Some([255, 255, 0, 255]));+ assert_eq!(frame.pixel(45, 157), Some([255, 0, 255, 255]));+}++/// The background goes with its picture through a wipe: halfway, the half+/// of the frame the picture has gone from is black, bars and all, and the+/// edge crosses the bars where it crosses the picture.+#[test]+fn a_background_is_wiped_with_its_picture() {+ let Some(mut gpu) = gpu() else { return };+ let (width, height) = (90, 160);+ let mut picture = layer(halves(64, 36));+ picture.transitions = vec![Transition::Wipe {+ uncovered: 0.5,+ from_right: false,+ }];+ let colour = picture+ .backdrop(&Backdrop::Colour([0, 255, 0]), width, height, |_| None)+ .expect("a colour");+ let frame = gpu.render(&plan(width, height, vec![colour, picture]));+ assert_eq!(+ frame.pixel(20, 80),+ Some([255, 0, 0, 255]),+ "the picture's kept half"+ );+ assert_eq!(+ frame.pixel(70, 80),+ Some([0, 0, 0, 255]),+ "the picture's wiped half"+ );+ assert_eq!(+ frame.pixel(20, 20),+ Some([0, 255, 0, 255]),+ "the bar's kept half"+ );+ assert_eq!(+ frame.pixel(70, 20),+ Some([0, 0, 0, 255]),+ "the bar's wiped half"+ );+ assert_eq!(+ frame.pixel(43, 20),+ Some([0, 255, 0, 255]),+ "the edge is the picture's"+ );+ assert_eq!(+ frame.pixel(47, 20),+ Some([0, 0, 0, 255]),+ "the edge is the picture's"+ );+}diff --git a/src/crates/concat-render/src/lib.rs b/src/crates/concat-render/src/lib.rsindex 109e09e4..153a2517 100644--- a/src/crates/concat-render/src/lib.rs+++ b/src/crates/concat-render/src/lib.rs@@ -31,8 +31,8 @@ pub use compositor::Compositor;pub use gpu::WgpuCompositor;pub use metrics::ssim;pub use plan::{- Crop, FramePlan, Geometry, PlannedLayer, PlannedTreatment, Shading, Transition, detached_clip,- plan_frame,+ Backdrop, Crop, FramePlan, Geometry, PlannedLayer, PlannedTreatment, Shading, Transition,+ detached_clip, plan_frame,};#[cfg(test)]pub(crate) use reference::CpuCompositor;diff --git a/src/crates/concat-render/src/plan.rs b/src/crates/concat-render/src/plan.rsindex 0337cfd1..68c8ac84 100644--- a/src/crates/concat-render/src/plan.rs+++ b/src/crates/concat-render/src/plan.rs@@ -289,6 +289,199 @@ impl PlannedLayer {}}+/// What fills the frame behind a picture where the picture does not reach+/// it - the bars beside a wide shot in a tall frame - as a compositor draws+/// it: one more layer, beneath the picture's own, built by+/// [`PlannedLayer::backdrop`]. Nothing here is new to the compositors, so+/// the monitor and the export cannot draw it two ways.+#[derive(Clone, Debug)]+pub enum Backdrop {+ /// The picture itself, run through its effects, enlarged to cover the+ /// frame and blurred: a Gaussian of `sigma` output pixels.+ Blur {+ /// The blur's spread, in pixels of the output frame.+ sigma: f32,+ },+ /// One opaque colour.+ Colour([u8; 3]),+ /// A still, enlarged to cover the frame.+ Image(Arc<Frame>),+}++impl PlannedLayer {+ /// True when the picture is certain to hide everything beneath it: drawn+ /// whole and solid, unturned, and placed over the entire frame. What+ /// spares a picture that already fills the frame a background nobody+ /// would see. `opaque` is the caller's word that the pixels themselves+ /// have no holes - a still may, and so may a cutout or any effect,+ /// which is why a layer with effects is never taken to cover.+ pub fn covers(&self, width: u32, height: u32, opaque: bool) -> bool {+ let Some(source) = &self.source else {+ return false;+ };+ if !opaque+ || self.weight() < 1.0+ || !self.effects.is_empty()+ || self.mask.is_some()+ || !self.transitions.is_empty()+ || self.blend != Blend::Normal+ {+ return false;+ }+ let geometry = self.geometry(source, width, height);+ if geometry.rotation != 0.0 {+ return false;+ }+ let half = (+ geometry.fitted.0 as f32 * geometry.scale.0 / 2.0,+ geometry.fitted.1 as f32 * geometry.scale.1 / 2.0,+ );+ // Half a pixel either way: a picture whose edge lands inside the+ // last pixel still covers its centre, which is all a sampler reads.+ geometry.centre.0 - half.0 <= 0.5+ && geometry.centre.0 + half.0 >= width as f32 - 0.5+ && geometry.centre.1 - half.1 <= 0.5+ && geometry.centre.1 + half.1 >= height as f32 - 0.5+ }++ /// The layer `backdrop` draws as under this one, in a `width` by+ /// `height` frame, or None where it has nothing to draw with - a blur+ /// of a layer with no picture yet.+ ///+ /// It covers the frame, centred, whatever the picture's own scale,+ /// position and turn: the background stays put while the picture moves+ /// over it. It belongs to the picture all the same: it is on the+ /// picture's track, as strong as the picture is at this frame, and goes+ /// through the picture's fades and wipes, so a dissolve or a fade to+ /// black takes the background with the picture rather than leaving it+ /// behind. `blur` makes the Gaussian pass for a radius in the picture's+ /// own pixels; the plan is a description and knows no package.+ pub fn backdrop(+ &self,+ backdrop: &Backdrop,+ width: u32,+ height: u32,+ blur: impl FnOnce(f32) -> Option<ShaderPass>,+ ) -> Option<PlannedLayer> {+ let mut layer = PlannedLayer {+ transform: Transform::default(),+ mask: None,+ blend: Blend::Normal,+ ..self.clone()+ };+ let (picture, effects) = match backdrop {+ Backdrop::Blur { sigma } => {+ let source = self.source.clone()?;+ let cover = cover_of(&layer, &source, width, height);+ // The spread asked for on screen, in the pixels the blur+ // runs at: the picture's fitted size, before the cover+ // enlarges it.+ let radius = sigma / cover;+ let mut effects = self.effects.clone();+ if radius >= 0.5 {+ effects.extend(blur(radius));+ }+ (source, effects)+ }+ Backdrop::Colour([r, g, b]) => {+ let mut solid = Frame::black(16, 16);+ solid.fill([*r, *g, *b, 255]);+ layer.crop = Crop::NONE;+ layer.flip_h = false;+ layer.flip_v = false;+ (Arc::new(solid), Vec::new())+ }+ Backdrop::Image(still) => {+ layer.crop = Crop::NONE;+ layer.flip_h = false;+ layer.flip_v = false;+ (Arc::clone(still), Vec::new())+ }+ };+ let cover = cover_of(&layer, &picture, width, height);+ // A colour stretches to the frame's shape; a picture keeps its own.+ let stretch = match backdrop {+ Backdrop::Colour(_) => {+ let fitted = layer.geometry(&picture, width, height).fitted;+ (+ f64::from(width) / f64::from(fitted.0) / f64::from(cover),+ f64::from(height) / f64::from(fitted.1) / f64::from(cover),+ )+ }+ _ => (1.0, 1.0),+ };+ // Two pixels over on the shorter side: the fit centres by whole+ // pixels, and a cover that met the edge exactly could leave a hair+ // of black along it.+ let over = 1.0 + 2.0 / width.min(height).max(1) as f64;+ layer.transform = Transform {+ scale: f64::from(cover) * over,+ stretch_x: stretch.0,+ stretch_y: stretch.1,+ ..Transform::default()+ };+ layer.transitions = self.backdrop_transitions(&layer, &picture, width, height);+ layer.source = Some(picture);+ layer.effects = effects;+ Some(layer)+ }++ /// This layer's transitions as they fall on `backdrop` drawn from+ /// `picture`: the fades as they are, and each wipe's edge moved to where+ /// it crosses the frame over the picture, since the background is wider+ /// than the picture and its own `u` runs across more of the frame.+ fn backdrop_transitions(+ &self,+ backdrop: &PlannedLayer,+ picture: &Frame,+ width: u32,+ height: u32,+ ) -> Vec<Transition> {+ let wiped = self+ .transitions+ .iter()+ .any(|transition| matches!(transition, Transition::Wipe { .. }));+ let Some(source) = self.source.as_ref().filter(|_| wiped) else {+ return self.transitions.clone();+ };+ let span = |layer: &PlannedLayer, frame: &Frame| {+ let geometry = layer.geometry(frame, width, height);+ let wide = geometry.fitted.0 as f32 * geometry.scale.0;+ (geometry.centre.0 - wide / 2.0, wide.max(1e-3))+ };+ let (own_left, own_wide) = span(self, source);+ let (left, wide) = span(backdrop, picture);+ let across = |u: f32| ((own_left + u * own_wide - left) / wide).clamp(0.0, 1.0);+ self.transitions+ .iter()+ .map(|transition| match *transition {+ Transition::Wipe {+ uncovered,+ from_right: false,+ } => Transition::Wipe {+ uncovered: across(uncovered),+ from_right: false,+ },+ Transition::Wipe {+ uncovered,+ from_right: true,+ } => Transition::Wipe {+ uncovered: 1.0 - across(1.0 - uncovered),+ from_right: true,+ },+ fade => fade,+ })+ .collect()+ }+}++/// How much larger than its fitted size `picture` must be drawn, through+/// `layer`'s crop, to cover a `width` by `height` frame.+fn cover_of(layer: &PlannedLayer, picture: &Frame, width: u32, height: u32) -> f32 {+ let fitted = Geometry::of(layer, picture.width(), picture.height(), width, height).fitted;+ (width as f32 / fitted.0 as f32).max(height as f32 / fitted.1 as f32)+}+/// Where a layer's picture comes from and where it lands: the arithmetic/// both compositors share, so a picture cannot be placed two ways.///@@ -717,6 +910,28 @@ mod tests {assert_eq!(cropped.uv_of(1.0, 1.0, false, false), (0.75, 1.0));}+ /// A picture covers the frame only drawn whole and solid over every+ /// pixel of it: a wide picture in a wide frame does, in a tall one only+ /// once it is scaled to fill, and never at half strength, turned, with+ /// effects, or when its pixels may have holes.+ #[test]+ fn a_picture_covers_the_frame_only_solid_and_whole() {+ let mut layer = PlannedLayer::picture(a_clip(), Arc::new(Frame::black(64, 36)));+ assert!(layer.covers(64, 36, true));+ assert!(!layer.covers(64, 36, false), "a still may have holes");+ assert!(!layer.covers(36, 64, true));+ layer.transform.scale = 3.2;+ assert!(layer.covers(36, 64, true), "filled");+ layer.transform.offset_y = 0.1;+ assert!(!layer.covers(36, 64, true), "moved off an edge");+ layer.transform.offset_y = 0.0;+ layer.transform.rotation = 2.0;+ assert!(!layer.covers(36, 64, true));+ layer.transform.rotation = 0.0;+ layer.opacity = 0.5;+ assert!(!layer.covers(36, 64, true));+ }+/// Fades compose into one map of the colour and wipes into one edge a/// side, so two fades to black at half are three quarters black and a/// wipe keeps only what its edge has uncovered.