Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

feat(ui): waveforms at the resolution of the zoom, and a sound band under every video

Jareer10256c752a6a
diff --git a/src/crates/concat-media/src/peaks.rs b/src/crates/concat-media/src/peaks.rs
index ed80a038..6a8053b9 100644
--- a/src/crates/concat-media/src/peaks.rs
+++ b/src/crates/concat-media/src/peaks.rs
@@ -168,7 +168,7 @@ impl Pyramid {
/// or a few and never a fraction of one. The finest when even it is
/// coarser than the column.
pub fn level_for(&self, seconds_per_column: f32) -> &Peaks {
- if !(seconds_per_column > 0.0) {
+ if seconds_per_column.is_nan() || seconds_per_column <= 0.0 {
return self.finest();
}
self.levels
@@ -335,8 +335,20 @@ mod tests {
assert_eq!(half.min, vec![-0.9, -0.3, -0.5]);
assert_eq!(half.max, vec![0.4, 0.8, 0.6]);
assert_eq!(half.buckets_per_second, 500.0);
- assert_eq!(fine.coarser(0).len(), 5, "a factor of nothing is the same shape");
- assert!(Peaks { min: vec![], max: vec![], buckets_per_second: 1000.0 }.coarser(4).is_empty());
+ assert_eq!(
+ fine.coarser(0).len(),
+ 5,
+ "a factor of nothing is the same shape"
+ );
+ assert!(
+ Peaks {
+ min: vec![],
+ max: vec![],
+ buckets_per_second: 1000.0
+ }
+ .coarser(4)
+ .is_empty()
+ );
}
#[test]
@@ -352,7 +364,11 @@ mod tests {
// coarsest that still puts at least one bucket in it.
assert_eq!(pyramid.level_for(0.001).buckets_per_second, 1000.0);
let coarse = pyramid.level_for(1.0);
- assert!(coarse.buckets_per_second <= 1000.0 / 64.0 + 1e-6, "{}", coarse.buckets_per_second);
+ assert!(
+ coarse.buckets_per_second <= 1000.0 / 64.0 + 1e-6,
+ "{}",
+ coarse.buckets_per_second
+ );
assert!(coarse.buckets_per_second * 1.0 >= 1.0);
// A column finer than the finest bucket still gets the finest.
assert_eq!(pyramid.level_for(1e-9).buckets_per_second, 1000.0);
diff --git a/src/crates/concat-perf/src/main.rs b/src/crates/concat-perf/src/main.rs
index 8de789cc..02040d85 100644
--- a/src/crates/concat-perf/src/main.rs
+++ b/src/crates/concat-perf/src/main.rs
@@ -68,6 +68,7 @@ fn main() {
undo_200_edits(),
document_round_trip(),
publish_window_of_200(),
+ waveform_of_an_hour(),
];
if !quick {
let media = Media::synthesise();
@@ -302,6 +303,54 @@ fn publish_window_of_200() -> Measure {
}
}
+/// Drawing an hour-long file's waveform across a screen: the pyramid hands
+/// the level that fits the column, so the path reads thousands of buckets
+/// and not millions, at every zoom.
+fn waveform_of_an_hour() -> Measure {
+ let buckets = 3600 * 1000;
+ let mut min = vec![-0.2f32; buckets];
+ let mut max = vec![0.2f32; buckets];
+ for index in (0..buckets).step_by(997) {
+ min[index] = -0.9;
+ max[index] = 0.9;
+ }
+ let started = Instant::now();
+ let pyramid = concat_media::Pyramid::of(concat_media::Peaks {
+ min,
+ max,
+ buckets_per_second: 1000.0,
+ });
+ let built = started.elapsed();
+ let rounds = 50;
+ let started = Instant::now();
+ let mut columns_read = 0;
+ for round in 0..rounds {
+ // Every zoom from the whole hour across the screen down to a
+ // second across it, each a column a pixel.
+ let seconds = 3600.0 / f64::from(1 << (round % 12)) as f32;
+ let level = pyramid.level_for(seconds / 2048.0);
+ for column in 0..2048 {
+ let from = column as f32 * seconds / 2048.0;
+ let (low, high) = level.extremes(from, from + seconds / 2048.0);
+ assert!(high >= 0.2 && low <= -0.2, "every column has sound in it");
+ columns_read += 1;
+ }
+ }
+ let micros = started.elapsed().as_secs_f64() * 1e6 / f64::from(rounds);
+ Measure {
+ name: "read an hour's waveform across 2048 columns",
+ value: micros,
+ unit: "us",
+ budget: Budget::AtMost(2000.0),
+ note: format!(
+ "{} levels built in {:.0} ms; {} columns read",
+ pyramid.depth(),
+ built.as_secs_f64() * 1e3,
+ columns_read / rounds
+ ),
+ }
+}
+
/// The synthetic media the heavier scenarios read: ninety frames of
/// 1080p H.264, three seconds at thirty.
struct Media {
diff --git a/src/crates/concat/src/format.rs b/src/crates/concat/src/format.rs
index b8b73dd2..f9e5156f 100644
--- a/src/crates/concat/src/format.rs
+++ b/src/crates/concat/src/format.rs
@@ -176,7 +176,7 @@ pub fn wave_path(
/// rather than a gap in it.
const FLOOR: f32 = 0.012;
- if !(duration > 0.0) || peaks.finest().is_empty() {
+ if duration.is_nan() || duration <= 0.0 || peaks.finest().is_empty() {
return String::new();
}
let columns = columns.clamp(*COLUMNS.start(), *COLUMNS.end());
@@ -187,7 +187,10 @@ pub fn wave_path(
for column in 0..columns {
let left = column as f32 / columns as f32;
let right = (column + 1) as f32 / columns as f32;
- let (low, high) = level.extremes(source_start + left * duration, source_start + right * duration);
+ let (low, high) = level.extremes(
+ source_start + left * duration,
+ source_start + right * duration,
+ );
let amplitude = ((high.max(-low) * gain).clamp(0.0, 1.0) * 0.48).max(FLOOR);
let (top, bottom) = (0.5 - amplitude, 0.5 + amplitude);
path.push_str(&format!(
@@ -203,7 +206,8 @@ pub fn wave_path(
/// at each step of that and not at every pixel, held to what `wave_path`
/// draws.
pub fn wave_columns(seconds: f32, seconds_per_pixel: f32) -> usize {
- if !(seconds > 0.0) || !(seconds_per_pixel > 0.0) {
+ if seconds.is_nan() || seconds <= 0.0 || seconds_per_pixel.is_nan() || seconds_per_pixel <= 0.0
+ {
return 8;
}
let pixels = (seconds / seconds_per_pixel).ceil().max(1.0) as usize;
@@ -294,7 +298,12 @@ mod tests {
assert!(wave_path(&peaks, 0.0, f32::NAN, 1.0, 128).is_empty());
// The column count is held to what is worth drawing, either way.
assert_eq!(wave_path(&peaks, 0.0, 2.0, 1.0, 0).matches('M').count(), 8);
- assert_eq!(wave_path(&peaks, 0.0, 2.0, 1.0, 1_000_000).matches('M').count(), 2048);
+ assert_eq!(
+ wave_path(&peaks, 0.0, 2.0, 1.0, 1_000_000)
+ .matches('M')
+ .count(),
+ 2048
+ );
}
/// A zoomed-in clip reads the fine buckets: a single loud millisecond
@@ -313,19 +322,38 @@ mod tests {
buckets_per_second: 1000.0,
});
let fine = wave_path(&peaks, 0.0, 1.0, 1.0, 1000);
- let tall: Vec<&str> = fine.split("Z ").filter(|bar| bar.contains(" 0.0200")).collect();
- assert_eq!(tall.len(), 1, "one column carries the spike: {}", tall.len());
+ let tall: Vec<&str> = fine
+ .split("Z ")
+ .filter(|bar| bar.contains(" 0.0200"))
+ .collect();
+ assert_eq!(
+ tall.len(),
+ 1,
+ "one column carries the spike: {}",
+ tall.len()
+ );
let coarse = wave_path(&peaks, 0.0, 1.0, 1.0, 10);
- assert_eq!(coarse.matches(" 0.0200").count(), 2, "the spike survives the fold at full height");
+ assert_eq!(
+ coarse.matches(" 0.0200").count(),
+ 2,
+ "the spike survives the fold at full height"
+ );
let trimmed = wave_path(&peaks, 0.6, 0.4, 1.0, 10);
- assert!(!trimmed.contains(" 0.0200"), "a trim past the spike does not show it");
+ assert!(
+ !trimmed.contains(" 0.0200"),
+ "a trim past the spike does not show it"
+ );
}
#[test]
fn columns_follow_the_zoom_in_steps_of_sixty_four() {
assert_eq!(wave_columns(10.0, 0.05), 256, "200 px rounds up to 256");
assert_eq!(wave_columns(10.0, 0.01), 1024);
- assert_eq!(wave_columns(600.0, 0.01), 2048, "held to the most worth drawing");
+ assert_eq!(
+ wave_columns(600.0, 0.01),
+ 2048,
+ "held to the most worth drawing"
+ );
assert_eq!(wave_columns(0.1, 0.05), 64, "never under a step");
assert_eq!(wave_columns(0.0, 0.05), 8);
assert_eq!(wave_columns(f32::NAN, 0.05), 8);
diff --git a/src/crates/concat/src/lib.rs b/src/crates/concat/src/lib.rs
index 8f4282d3..d571c834 100644
--- a/src/crates/concat/src/lib.rs
+++ b/src/crates/concat/src/lib.rs
@@ -1289,7 +1289,7 @@ pub fn run() -> Result<(), slint::PlatformError> {
.peaks
.get(&plan.media)
.filter(|_| plan.kind == ClipKind::Audio)
- .map(|peaks| format::wave_path(peaks, 0.0, plan.duration, 1.0))
+ .map(|peaks| format::wave_path(peaks, 0.0, plan.duration, 1.0, 128))
.unwrap_or_default();
let document = chips::drag_chip_svg(
chips::chip_glyph(plan.kind),
diff --git a/src/crates/concat/src/panes/media_bin.rs b/src/crates/concat/src/panes/media_bin.rs
index 4f629ac2..8baee306 100644
--- a/src/crates/concat/src/panes/media_bin.rs
+++ b/src/crates/concat/src/panes/media_bin.rs
@@ -309,7 +309,9 @@ impl MediaBin {
thumbnail: self.thumbs.get(&item.id).cloned().unwrap_or_default(),
wave: match studio.peaks.get(&item.id) {
Some(peaks) if item.kind == model::MediaKind::Audio => {
- wave_path(peaks, 0.0, item.duration.unwrap_or(0.0) as f32, 1.0).into()
+ // A card is a couple of hundred pixels: a column
+ // each is all the detail it can show.
+ wave_path(peaks, 0.0, item.duration.unwrap_or(0.0) as f32, 1.0, 256).into()
}
_ => SharedString::new(),
},
diff --git a/src/crates/concat/src/studio.rs b/src/crates/concat/src/studio.rs
index 5d631c40..cb6f3825 100644
--- a/src/crates/concat/src/studio.rs
+++ b/src/crates/concat/src/studio.rs
@@ -35,7 +35,7 @@ use concat_host::playback::ClipSpec;
use concat_host::{
AnalyseRequest, Cutouts, ProjectInfo, RegionRequest, Session, media, projects, templates,
};
-use concat_media::{Peaks, jpeg};
+use concat_media::{Pyramid, jpeg};
use concat_project::commands::{ClipMove, ClipPatch, TrackFlag, TrimEdge};
use concat_project::model::{
self, AppliedFilter, Clip, Project, TextAlign, TextStyle, Timeline, Track, Transition,
@@ -46,7 +46,9 @@ use slint::{Model, SharedString, VecModel};
use crate::dock::{
Dock, DockLayout, SEAT_GAP, default_dock, lay_out, nearest_row, row_at, row_top,
};
-use crate::format::{colour_of, frames_timecode, hex_of, hex_with_alpha, wave_path, when_phrase};
+use crate::format::{
+ colour_of, frames_timecode, hex_of, hex_with_alpha, wave_columns, wave_path, when_phrase,
+};
use crate::host::{
CachedStrip, Host, MediaArt, WindowArt, cached_media_art, cached_window_art, image_at,
image_of, media_art, on_ui, spawn, spawn_art, spawn_strip, strip_window, window_art,
@@ -515,7 +517,7 @@ pub struct Studio {
// ── the bin ──
pub media: crate::panes::media_bin::MediaBin,
/// Decoded art by media id, and the ids a worker is decoding for.
- pub peaks: HashMap<String, Arc<Peaks>>,
+ pub peaks: HashMap<String, Arc<Pyramid>>,
/// Filmstrips by media id: the picture, how many frames are in it, one
/// frame's width and the strip's height, in the picture's own pixels.
pub strips: HashMap<String, Strip>,
@@ -2086,8 +2088,14 @@ impl Studio {
/// The memoised envelope for one clip, quantised to a thirtieth of a
/// second so a trim revisits a handful of entries rather than minting
- /// one per pointer event.
+ /// one per pointer event, and to a step of columns so a zoom rebuilds
+ /// it at each step of that and not at every pixel. A sound clip's
+ /// whole body, a picture clip's band under its frames; a picture that
+ /// is muted, or whose file has no sound, shows the empty band.
fn wave(&self, clip: &Clip) -> SharedString {
+ if clip.muted == Some(true) {
+ return SharedString::new();
+ }
// The stream this clip plays; its peaks come when they are decoded,
// and until then the lane is bare rather than showing another
// track's shape.
@@ -2098,11 +2106,12 @@ impl Studio {
let step = |seconds: f32| (seconds * WAVE_STEPS).round() / WAVE_STEPS;
let (source_start, duration) = (step(clip.source_start as f32), step(clip.duration as f32));
let gain = clip.volume as f32;
- let key = format!("{art}|{source_start:.3}|{duration:.3}|{gain:.3}");
+ let columns = wave_columns(clip.duration as f32, self.lanes.seconds_per_pixel);
+ let key = format!("{art}|{source_start:.3}|{duration:.3}|{gain:.3}|{columns}");
if let Some(cached) = self.waves.borrow().get(&key) {
return cached.clone();
}
- let built = SharedString::from(wave_path(peaks, source_start, duration, gain));
+ let built = SharedString::from(wave_path(peaks, source_start, duration, gain, columns));
self.waves.borrow_mut().insert(key, built.clone());
built
}
@@ -4889,7 +4898,7 @@ impl Studio {
.map(|text| text.content.as_str())
.unwrap_or_default()
.into(),
- wave: if clip.kind == model::ClipKind::Audio {
+ wave: if matches!(clip.kind, model::ClipKind::Audio | model::ClipKind::Video) {
self.wave(clip)
} else {
SharedString::new()
diff --git a/src/crates/concat/ui/timeline/lanes.slint b/src/crates/concat/ui/timeline/lanes.slint
index d0001073..a36a3e5d 100644
--- a/src/crates/concat/ui/timeline/lanes.slint
+++ b/src/crates/concat/ui/timeline/lanes.slint
@@ -153,6 +153,17 @@ component Clip inherits Rectangle {
property <bool> marked: root.item.kind == ClipKind.filter;
property <length> text-x: root.marked ? 22px : 7px;
property <length> body-h: Math.max(0px, root.height - root.header-h);
+ /// The sound band at the foot of a video clip: its file's waveform
+ /// under its frames, a third of the body and never a sliver, and there
+ /// whether or not the file has sound - a silent file shows the empty
+ /// band, so a video that has gone quiet is tellable from one that
+ /// never spoke. https://github.com/jub0t/Concat/issues/137
+ property <bool> banded: root.item.kind == ClipKind.video;
+ property <length> band-h: root.banded
+ ? Math.clamp(root.body-h * 0.34, 8px, 24px)
+ : 0px;
+ /// What the frames get: the body less the band.
+ property <length> frames-h: Math.max(0px, root.body-h - root.band-h);
/// A fade on a picture clip is shaded in a band at the foot instead of
/// over the whole body: the fade is an *audio* property, and shading the
/// picture would say it fades to black. Sound shades its whole body; a
@@ -191,12 +202,11 @@ component Clip inherits Rectangle {
clip: true;
// The audio envelope. The commands arrive normalised into a 1x1 box
- // and `fit: fill` stretches that onto whatever width the clip has, so
- // the shape survives a zoom without Rust rebuilding the string. The
- // trade-off is per-pixel extremes when zoomed right in — there the
- // envelope is a stretched shape rather than a fresh scan of the
- // buckets.
- if root.item.wave != "": Path {
+ // and `fit: fill` stretches that onto whatever width the clip has;
+ // Rust rebuilds the string with a column a pixel at each step of
+ // the zoom, from the peaks at the level that fits the column, so
+ // zooming in reveals the buckets rather than stretching a shape.
+ if root.item.wave != "" && !root.banded: Path {
x: 0;
y: 0;
width: parent.width;
@@ -208,7 +218,7 @@ component Clip inherits Rectangle {
fill: Theme.wave;
}
- if root.item.wave != "": Rectangle {
+ if root.item.wave != "" && !root.banded: Rectangle {
x: 0;
y: (parent.height - 1px) / 2;
width: parent.width;
@@ -239,11 +249,11 @@ component Clip inherits Rectangle {
x: 0;
y: 0;
width: parent.width;
- height: parent.height;
+ height: root.banded ? root.frames-h : parent.height;
clip: true;
property <length> tile-w: Math.max(4px,
- parent.height * root.item.strip.frame-width / Math.max(1, root.item.strip.height));
+ self.height * root.item.strip.frame-width / Math.max(1, root.item.strip.height));
property <int> tiles: Math.min(400, Math.ceil(parent.width / self.tile-w));
/// The fraction of the clip under the middle of tile `i`'s
@@ -276,12 +286,52 @@ component Clip inherits Rectangle {
if (root.item.kind == ClipKind.video || root.item.kind == ClipKind.image)
&& root.item.strip.frames == 0 && root.width > 40px: Icon {
x: (parent.width - self.width) / 2;
- y: (parent.height - self.height) / 2;
+ y: ((root.banded ? root.frames-h : parent.height) - self.height) / 2;
glyph: root.badge;
- size: Math.min(22px, parent.height - 6px);
+ size: Math.min(22px, (root.banded ? root.frames-h : parent.height) - 6px);
tint: root.mark.with-alpha(0.3);
}
+ // The sound band under a video's frames: a hairline off the frames,
+ // the waveform when the file has sound and its peaks have come, the
+ // centre line either way.
+ if root.banded && root.band-h > 0px: Rectangle {
+ x: 0;
+ y: root.frames-h;
+ width: parent.width;
+ height: root.band-h;
+ background: Theme.kind-audio-well.with-alpha(0.55);
+ clip: true;
+
+ Rectangle {
+ x: 0;
+ y: 0;
+ width: parent.width;
+ height: 1px;
+ background: Theme.line;
+ }
+
+ if root.item.wave != "": Path {
+ x: 0;
+ y: 1px;
+ width: parent.width;
+ height: parent.height - 1px;
+ commands: root.item.wave;
+ viewbox-width: 1;
+ viewbox-height: 1;
+ fit: fill;
+ fill: Theme.wave;
+ }
+
+ Rectangle {
+ x: 0;
+ y: (parent.height - 1px) / 2 + 0.5px;
+ width: parent.width;
+ height: 1px;
+ background: Theme.wave-centre;
+ }
+ }
+
// The fade ramps, at the foot of the body.
Ramps {
x: 0;