The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
feat(ui): waveforms at the resolution of the zoom, and a sound band under every video
Jareer10256c752a6a
diff --git a/src/crates/concat-media/src/peaks.rs b/src/crates/concat-media/src/peaks.rsindex ed80a038..6a8053b9 100644--- a/src/crates/concat-media/src/peaks.rs+++ b/src/crates/concat-media/src/peaks.rs@@ -168,7 +168,7 @@ impl Pyramid {/// or a few and never a fraction of one. The finest when even it is/// coarser than the column.pub fn level_for(&self, seconds_per_column: f32) -> &Peaks {- if !(seconds_per_column > 0.0) {+ if seconds_per_column.is_nan() || seconds_per_column <= 0.0 {return self.finest();}self.levels@@ -335,8 +335,20 @@ mod tests {assert_eq!(half.min, vec![-0.9, -0.3, -0.5]);assert_eq!(half.max, vec![0.4, 0.8, 0.6]);assert_eq!(half.buckets_per_second, 500.0);- assert_eq!(fine.coarser(0).len(), 5, "a factor of nothing is the same shape");- assert!(Peaks { min: vec![], max: vec![], buckets_per_second: 1000.0 }.coarser(4).is_empty());+ assert_eq!(+ fine.coarser(0).len(),+ 5,+ "a factor of nothing is the same shape"+ );+ assert!(+ Peaks {+ min: vec![],+ max: vec![],+ buckets_per_second: 1000.0+ }+ .coarser(4)+ .is_empty()+ );}#[test]@@ -352,7 +364,11 @@ mod tests {// coarsest that still puts at least one bucket in it.assert_eq!(pyramid.level_for(0.001).buckets_per_second, 1000.0);let coarse = pyramid.level_for(1.0);- assert!(coarse.buckets_per_second <= 1000.0 / 64.0 + 1e-6, "{}", coarse.buckets_per_second);+ assert!(+ coarse.buckets_per_second <= 1000.0 / 64.0 + 1e-6,+ "{}",+ coarse.buckets_per_second+ );assert!(coarse.buckets_per_second * 1.0 >= 1.0);// A column finer than the finest bucket still gets the finest.assert_eq!(pyramid.level_for(1e-9).buckets_per_second, 1000.0);diff --git a/src/crates/concat-perf/src/main.rs b/src/crates/concat-perf/src/main.rsindex 8de789cc..02040d85 100644--- a/src/crates/concat-perf/src/main.rs+++ b/src/crates/concat-perf/src/main.rs@@ -68,6 +68,7 @@ fn main() {undo_200_edits(),document_round_trip(),publish_window_of_200(),+ waveform_of_an_hour(),];if !quick {let media = Media::synthesise();@@ -302,6 +303,54 @@ fn publish_window_of_200() -> Measure {}}+/// Drawing an hour-long file's waveform across a screen: the pyramid hands+/// the level that fits the column, so the path reads thousands of buckets+/// and not millions, at every zoom.+fn waveform_of_an_hour() -> Measure {+ let buckets = 3600 * 1000;+ let mut min = vec![-0.2f32; buckets];+ let mut max = vec![0.2f32; buckets];+ for index in (0..buckets).step_by(997) {+ min[index] = -0.9;+ max[index] = 0.9;+ }+ let started = Instant::now();+ let pyramid = concat_media::Pyramid::of(concat_media::Peaks {+ min,+ max,+ buckets_per_second: 1000.0,+ });+ let built = started.elapsed();+ let rounds = 50;+ let started = Instant::now();+ let mut columns_read = 0;+ for round in 0..rounds {+ // Every zoom from the whole hour across the screen down to a+ // second across it, each a column a pixel.+ let seconds = 3600.0 / f64::from(1 << (round % 12)) as f32;+ let level = pyramid.level_for(seconds / 2048.0);+ for column in 0..2048 {+ let from = column as f32 * seconds / 2048.0;+ let (low, high) = level.extremes(from, from + seconds / 2048.0);+ assert!(high >= 0.2 && low <= -0.2, "every column has sound in it");+ columns_read += 1;+ }+ }+ let micros = started.elapsed().as_secs_f64() * 1e6 / f64::from(rounds);+ Measure {+ name: "read an hour's waveform across 2048 columns",+ value: micros,+ unit: "us",+ budget: Budget::AtMost(2000.0),+ note: format!(+ "{} levels built in {:.0} ms; {} columns read",+ pyramid.depth(),+ built.as_secs_f64() * 1e3,+ columns_read / rounds+ ),+ }+}+/// The synthetic media the heavier scenarios read: ninety frames of/// 1080p H.264, three seconds at thirty.struct Media {diff --git a/src/crates/concat/src/format.rs b/src/crates/concat/src/format.rsindex b8b73dd2..f9e5156f 100644--- a/src/crates/concat/src/format.rs+++ b/src/crates/concat/src/format.rs@@ -176,7 +176,7 @@ pub fn wave_path(/// rather than a gap in it.const FLOOR: f32 = 0.012;- if !(duration > 0.0) || peaks.finest().is_empty() {+ if duration.is_nan() || duration <= 0.0 || peaks.finest().is_empty() {return String::new();}let columns = columns.clamp(*COLUMNS.start(), *COLUMNS.end());@@ -187,7 +187,10 @@ pub fn wave_path(for column in 0..columns {let left = column as f32 / columns as f32;let right = (column + 1) as f32 / columns as f32;- let (low, high) = level.extremes(source_start + left * duration, source_start + right * duration);+ let (low, high) = level.extremes(+ source_start + left * duration,+ source_start + right * duration,+ );let amplitude = ((high.max(-low) * gain).clamp(0.0, 1.0) * 0.48).max(FLOOR);let (top, bottom) = (0.5 - amplitude, 0.5 + amplitude);path.push_str(&format!(@@ -203,7 +206,8 @@ pub fn wave_path(/// at each step of that and not at every pixel, held to what `wave_path`/// draws.pub fn wave_columns(seconds: f32, seconds_per_pixel: f32) -> usize {- if !(seconds > 0.0) || !(seconds_per_pixel > 0.0) {+ if seconds.is_nan() || seconds <= 0.0 || seconds_per_pixel.is_nan() || seconds_per_pixel <= 0.0+ {return 8;}let pixels = (seconds / seconds_per_pixel).ceil().max(1.0) as usize;@@ -294,7 +298,12 @@ mod tests {assert!(wave_path(&peaks, 0.0, f32::NAN, 1.0, 128).is_empty());// The column count is held to what is worth drawing, either way.assert_eq!(wave_path(&peaks, 0.0, 2.0, 1.0, 0).matches('M').count(), 8);- assert_eq!(wave_path(&peaks, 0.0, 2.0, 1.0, 1_000_000).matches('M').count(), 2048);+ assert_eq!(+ wave_path(&peaks, 0.0, 2.0, 1.0, 1_000_000)+ .matches('M')+ .count(),+ 2048+ );}/// A zoomed-in clip reads the fine buckets: a single loud millisecond@@ -313,19 +322,38 @@ mod tests {buckets_per_second: 1000.0,});let fine = wave_path(&peaks, 0.0, 1.0, 1.0, 1000);- let tall: Vec<&str> = fine.split("Z ").filter(|bar| bar.contains(" 0.0200")).collect();- assert_eq!(tall.len(), 1, "one column carries the spike: {}", tall.len());+ let tall: Vec<&str> = fine+ .split("Z ")+ .filter(|bar| bar.contains(" 0.0200"))+ .collect();+ assert_eq!(+ tall.len(),+ 1,+ "one column carries the spike: {}",+ tall.len()+ );let coarse = wave_path(&peaks, 0.0, 1.0, 1.0, 10);- assert_eq!(coarse.matches(" 0.0200").count(), 2, "the spike survives the fold at full height");+ assert_eq!(+ coarse.matches(" 0.0200").count(),+ 2,+ "the spike survives the fold at full height"+ );let trimmed = wave_path(&peaks, 0.6, 0.4, 1.0, 10);- assert!(!trimmed.contains(" 0.0200"), "a trim past the spike does not show it");+ assert!(+ !trimmed.contains(" 0.0200"),+ "a trim past the spike does not show it"+ );}#[test]fn columns_follow_the_zoom_in_steps_of_sixty_four() {assert_eq!(wave_columns(10.0, 0.05), 256, "200 px rounds up to 256");assert_eq!(wave_columns(10.0, 0.01), 1024);- assert_eq!(wave_columns(600.0, 0.01), 2048, "held to the most worth drawing");+ assert_eq!(+ wave_columns(600.0, 0.01),+ 2048,+ "held to the most worth drawing"+ );assert_eq!(wave_columns(0.1, 0.05), 64, "never under a step");assert_eq!(wave_columns(0.0, 0.05), 8);assert_eq!(wave_columns(f32::NAN, 0.05), 8);diff --git a/src/crates/concat/src/lib.rs b/src/crates/concat/src/lib.rsindex 8f4282d3..d571c834 100644--- a/src/crates/concat/src/lib.rs+++ b/src/crates/concat/src/lib.rs@@ -1289,7 +1289,7 @@ pub fn run() -> Result<(), slint::PlatformError> {.peaks.get(&plan.media).filter(|_| plan.kind == ClipKind::Audio)- .map(|peaks| format::wave_path(peaks, 0.0, plan.duration, 1.0))+ .map(|peaks| format::wave_path(peaks, 0.0, plan.duration, 1.0, 128)).unwrap_or_default();let document = chips::drag_chip_svg(chips::chip_glyph(plan.kind),diff --git a/src/crates/concat/src/panes/media_bin.rs b/src/crates/concat/src/panes/media_bin.rsindex 4f629ac2..8baee306 100644--- a/src/crates/concat/src/panes/media_bin.rs+++ b/src/crates/concat/src/panes/media_bin.rs@@ -309,7 +309,9 @@ impl MediaBin {thumbnail: self.thumbs.get(&item.id).cloned().unwrap_or_default(),wave: match studio.peaks.get(&item.id) {Some(peaks) if item.kind == model::MediaKind::Audio => {- wave_path(peaks, 0.0, item.duration.unwrap_or(0.0) as f32, 1.0).into()+ // A card is a couple of hundred pixels: a column+ // each is all the detail it can show.+ wave_path(peaks, 0.0, item.duration.unwrap_or(0.0) as f32, 1.0, 256).into()}_ => SharedString::new(),},diff --git a/src/crates/concat/src/studio.rs b/src/crates/concat/src/studio.rsindex 5d631c40..cb6f3825 100644--- a/src/crates/concat/src/studio.rs+++ b/src/crates/concat/src/studio.rs@@ -35,7 +35,7 @@ use concat_host::playback::ClipSpec;use concat_host::{AnalyseRequest, Cutouts, ProjectInfo, RegionRequest, Session, media, projects, templates,};-use concat_media::{Peaks, jpeg};+use concat_media::{Pyramid, jpeg};use concat_project::commands::{ClipMove, ClipPatch, TrackFlag, TrimEdge};use concat_project::model::{self, AppliedFilter, Clip, Project, TextAlign, TextStyle, Timeline, Track, Transition,@@ -46,7 +46,9 @@ use slint::{Model, SharedString, VecModel};use crate::dock::{Dock, DockLayout, SEAT_GAP, default_dock, lay_out, nearest_row, row_at, row_top,};-use crate::format::{colour_of, frames_timecode, hex_of, hex_with_alpha, wave_path, when_phrase};+use crate::format::{+ colour_of, frames_timecode, hex_of, hex_with_alpha, wave_columns, wave_path, when_phrase,+};use crate::host::{CachedStrip, Host, MediaArt, WindowArt, cached_media_art, cached_window_art, image_at,image_of, media_art, on_ui, spawn, spawn_art, spawn_strip, strip_window, window_art,@@ -515,7 +517,7 @@ pub struct Studio {// ── the bin ──pub media: crate::panes::media_bin::MediaBin,/// Decoded art by media id, and the ids a worker is decoding for.- pub peaks: HashMap<String, Arc<Peaks>>,+ pub peaks: HashMap<String, Arc<Pyramid>>,/// Filmstrips by media id: the picture, how many frames are in it, one/// frame's width and the strip's height, in the picture's own pixels.pub strips: HashMap<String, Strip>,@@ -2086,8 +2088,14 @@ impl Studio {/// The memoised envelope for one clip, quantised to a thirtieth of a/// second so a trim revisits a handful of entries rather than minting- /// one per pointer event.+ /// one per pointer event, and to a step of columns so a zoom rebuilds+ /// it at each step of that and not at every pixel. A sound clip's+ /// whole body, a picture clip's band under its frames; a picture that+ /// is muted, or whose file has no sound, shows the empty band.fn wave(&self, clip: &Clip) -> SharedString {+ if clip.muted == Some(true) {+ return SharedString::new();+ }// The stream this clip plays; its peaks come when they are decoded,// and until then the lane is bare rather than showing another// track's shape.@@ -2098,11 +2106,12 @@ impl Studio {let step = |seconds: f32| (seconds * WAVE_STEPS).round() / WAVE_STEPS;let (source_start, duration) = (step(clip.source_start as f32), step(clip.duration as f32));let gain = clip.volume as f32;- let key = format!("{art}|{source_start:.3}|{duration:.3}|{gain:.3}");+ let columns = wave_columns(clip.duration as f32, self.lanes.seconds_per_pixel);+ let key = format!("{art}|{source_start:.3}|{duration:.3}|{gain:.3}|{columns}");if let Some(cached) = self.waves.borrow().get(&key) {return cached.clone();}- let built = SharedString::from(wave_path(peaks, source_start, duration, gain));+ let built = SharedString::from(wave_path(peaks, source_start, duration, gain, columns));self.waves.borrow_mut().insert(key, built.clone());built}@@ -4889,7 +4898,7 @@ impl Studio {.map(|text| text.content.as_str()).unwrap_or_default().into(),- wave: if clip.kind == model::ClipKind::Audio {+ wave: if matches!(clip.kind, model::ClipKind::Audio | model::ClipKind::Video) {self.wave(clip)} else {SharedString::new()diff --git a/src/crates/concat/ui/timeline/lanes.slint b/src/crates/concat/ui/timeline/lanes.slintindex d0001073..a36a3e5d 100644--- a/src/crates/concat/ui/timeline/lanes.slint+++ b/src/crates/concat/ui/timeline/lanes.slint@@ -153,6 +153,17 @@ component Clip inherits Rectangle {property <bool> marked: root.item.kind == ClipKind.filter;property <length> text-x: root.marked ? 22px : 7px;property <length> body-h: Math.max(0px, root.height - root.header-h);+ /// The sound band at the foot of a video clip: its file's waveform+ /// under its frames, a third of the body and never a sliver, and there+ /// whether or not the file has sound - a silent file shows the empty+ /// band, so a video that has gone quiet is tellable from one that+ /// never spoke. https://github.com/jub0t/Concat/issues/137+ property <bool> banded: root.item.kind == ClipKind.video;+ property <length> band-h: root.banded+ ? Math.clamp(root.body-h * 0.34, 8px, 24px)+ : 0px;+ /// What the frames get: the body less the band.+ property <length> frames-h: Math.max(0px, root.body-h - root.band-h);/// A fade on a picture clip is shaded in a band at the foot instead of/// over the whole body: the fade is an *audio* property, and shading the/// picture would say it fades to black. Sound shades its whole body; a@@ -191,12 +202,11 @@ component Clip inherits Rectangle {clip: true;// The audio envelope. The commands arrive normalised into a 1x1 box- // and `fit: fill` stretches that onto whatever width the clip has, so- // the shape survives a zoom without Rust rebuilding the string. The- // trade-off is per-pixel extremes when zoomed right in — there the- // envelope is a stretched shape rather than a fresh scan of the- // buckets.- if root.item.wave != "": Path {+ // and `fit: fill` stretches that onto whatever width the clip has;+ // Rust rebuilds the string with a column a pixel at each step of+ // the zoom, from the peaks at the level that fits the column, so+ // zooming in reveals the buckets rather than stretching a shape.+ if root.item.wave != "" && !root.banded: Path {x: 0;y: 0;width: parent.width;@@ -208,7 +218,7 @@ component Clip inherits Rectangle {fill: Theme.wave;}- if root.item.wave != "": Rectangle {+ if root.item.wave != "" && !root.banded: Rectangle {x: 0;y: (parent.height - 1px) / 2;width: parent.width;@@ -239,11 +249,11 @@ component Clip inherits Rectangle {x: 0;y: 0;width: parent.width;- height: parent.height;+ height: root.banded ? root.frames-h : parent.height;clip: true;property <length> tile-w: Math.max(4px,- parent.height * root.item.strip.frame-width / Math.max(1, root.item.strip.height));+ self.height * root.item.strip.frame-width / Math.max(1, root.item.strip.height));property <int> tiles: Math.min(400, Math.ceil(parent.width / self.tile-w));/// The fraction of the clip under the middle of tile `i`'s@@ -276,12 +286,52 @@ component Clip inherits Rectangle {if (root.item.kind == ClipKind.video || root.item.kind == ClipKind.image)&& root.item.strip.frames == 0 && root.width > 40px: Icon {x: (parent.width - self.width) / 2;- y: (parent.height - self.height) / 2;+ y: ((root.banded ? root.frames-h : parent.height) - self.height) / 2;glyph: root.badge;- size: Math.min(22px, parent.height - 6px);+ size: Math.min(22px, (root.banded ? root.frames-h : parent.height) - 6px);tint: root.mark.with-alpha(0.3);}+ // The sound band under a video's frames: a hairline off the frames,+ // the waveform when the file has sound and its peaks have come, the+ // centre line either way.+ if root.banded && root.band-h > 0px: Rectangle {+ x: 0;+ y: root.frames-h;+ width: parent.width;+ height: root.band-h;+ background: Theme.kind-audio-well.with-alpha(0.55);+ clip: true;++ Rectangle {+ x: 0;+ y: 0;+ width: parent.width;+ height: 1px;+ background: Theme.line;+ }++ if root.item.wave != "": Path {+ x: 0;+ y: 1px;+ width: parent.width;+ height: parent.height - 1px;+ commands: root.item.wave;+ viewbox-width: 1;+ viewbox-height: 1;+ fit: fill;+ fill: Theme.wave;+ }++ Rectangle {+ x: 0;+ y: (parent.height - 1px) / 2 + 0.5px;+ width: parent.width;+ height: 1px;+ background: Theme.wave-centre;+ }+ }+// The fade ramps, at the foot of the body.Ramps {x: 0;