Skip to content

The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.

updates

Jareera48800506ef0
diff --git a/AUDIT-2026-09-19.md b/audits/AUDIT-2026-09-19.md
similarity index 100%
rename from AUDIT-2026-09-19.md
rename to audits/AUDIT-2026-09-19.md
diff --git a/src/crates/concat-host/src/media.rs b/src/crates/concat-host/src/media.rs
index 6523b3e2..1043a15a 100644
--- a/src/crates/concat-host/src/media.rs
+++ b/src/crates/concat-host/src/media.rs
@@ -232,7 +232,10 @@ pub fn read_bytes(path: &str) -> Result<Vec<u8>, String> {
/// timeline zoom, which is enough that the drawn shape does not visibly
/// change as you zoom in a step or two, without storing the whole decoded
/// file.
-pub const PEAKS_BUCKETS_PER_SECOND: u32 = 200;
+/// A thousand a second: a bucket is a millisecond, forty-eight samples,
+/// which is what a clip a screen wide at the closest zoom needs, and the
+/// pyramid folds it down for every wider view. See `concat_media::Pyramid`.
+pub const PEAKS_BUCKETS_PER_SECOND: u32 = 1000;
/// Waveform peaks for one media file: engine-decoded, project-cached.
///
diff --git a/src/crates/concat-media/src/lib.rs b/src/crates/concat-media/src/lib.rs
index 9085d8aa..054fc5d8 100644
--- a/src/crates/concat-media/src/lib.rs
+++ b/src/crates/concat-media/src/lib.rs
@@ -31,7 +31,7 @@ pub use encode::{EncodeOptions, Encoder, FrameSink, VideoCodec, jpeg};
pub use error::{Error, Result};
pub use ffi::{init, linked_version};
pub use hardware::{HwDevice, HwPolicy, hardware_decode, set_hardware_decode};
-pub use peaks::Peaks;
+pub use peaks::{Peaks, Pyramid};
pub use pool::{CacheStats, FrameCache, FrameRequest, ReaderPool};
pub use prefetch::{Cursor, Direction, Moment, Prefetcher, Priority};
pub use probe::{AudioStream, MediaInfo, VideoStream, probe};
diff --git a/src/crates/concat-media/src/peaks.rs b/src/crates/concat-media/src/peaks.rs
index 401172fc..ed80a038 100644
--- a/src/crates/concat-media/src/peaks.rs
+++ b/src/crates/concat-media/src/peaks.rs
@@ -28,6 +28,7 @@ const PEAK_RATE: u32 = 48_000;
/// flat 0/0 pair, and a bucket's minimum can never sit above the axis.
/// That is the shape the timeline has always drawn, and the on-disk caches
/// already hold it.
+#[derive(Clone, Debug, PartialEq)]
pub struct Peaks {
/// The lowest sample in each bucket, in [-1, 0].
pub min: Vec<f32>,
@@ -80,6 +81,109 @@ impl Peaks {
}
}
+impl Peaks {
+ /// How many buckets there are.
+ pub fn len(&self) -> usize {
+ self.min.len().min(self.max.len())
+ }
+
+ /// Whether there are none.
+ pub fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+
+ /// The same waveform at `factor` buckets to one: each new bucket takes
+ /// the extremes of the buckets it folds, so nothing quieter or louder
+ /// than the fine shape appears at the coarse one. A trailing partial
+ /// group still counts.
+ pub fn coarser(&self, factor: usize) -> Peaks {
+ let factor = factor.max(1);
+ let count = self.len();
+ let groups = count.div_ceil(factor);
+ let mut min = Vec::with_capacity(groups);
+ let mut max = Vec::with_capacity(groups);
+ for group in 0..groups {
+ let from = group * factor;
+ let to = (from + factor).min(count);
+ min.push(self.min[from..to].iter().copied().fold(0.0, f32::min));
+ max.push(self.max[from..to].iter().copied().fold(0.0, f32::max));
+ }
+ Peaks {
+ min,
+ max,
+ buckets_per_second: self.buckets_per_second / factor as f32,
+ }
+ }
+
+ /// The extremes over the buckets covering `from` to `to` seconds of the
+ /// file, `(low, high)` with low at or below zero and high at or above
+ /// it. Past the end, or an empty span, is silence.
+ pub fn extremes(&self, from: f32, to: f32) -> (f32, f32) {
+ let count = self.len();
+ if count == 0 || to <= from || self.buckets_per_second <= 0.0 {
+ return (0.0, 0.0);
+ }
+ let first = ((from * self.buckets_per_second).floor().max(0.0) as usize).min(count);
+ let last = ((to * self.buckets_per_second).ceil().max(0.0) as usize).min(count);
+ let (mut low, mut high) = (0.0f32, 0.0f32);
+ for index in first..last.max(first) {
+ low = low.min(self.min[index]);
+ high = high.max(self.max[index]);
+ }
+ (low, high)
+ }
+}
+
+/// A waveform at every resolution a drawing could want: the file's peaks
+/// as extracted, then halved again and again until a level is a handful of
+/// buckets. A drawing asks for the level whose buckets are about the size
+/// of its columns, so a clip drawn across two thousand pixels reads two
+/// thousand buckets and not two million, and a clip drawn across twenty
+/// reads twenty - the same extremes either way, since every level is the
+/// fold of the one beneath.
+#[derive(Clone, Debug)]
+pub struct Pyramid {
+ /// Finest first.
+ levels: Vec<Peaks>,
+}
+
+impl Pyramid {
+ /// Every level, from `finest` down to a few buckets.
+ pub fn of(finest: Peaks) -> Pyramid {
+ let mut levels = vec![finest];
+ while levels.last().is_some_and(|level| level.len() > 64) {
+ let coarser = levels.last().expect("just checked").coarser(2);
+ levels.push(coarser);
+ }
+ Pyramid { levels }
+ }
+
+ /// The peaks as extracted.
+ pub fn finest(&self) -> &Peaks {
+ &self.levels[0]
+ }
+
+ /// The level to draw a column of `seconds` from: the coarsest whose
+ /// buckets are no larger than the column, so a column reads one bucket
+ /// or a few and never a fraction of one. The finest when even it is
+ /// coarser than the column.
+ pub fn level_for(&self, seconds_per_column: f32) -> &Peaks {
+ if !(seconds_per_column > 0.0) {
+ return self.finest();
+ }
+ self.levels
+ .iter()
+ .rev()
+ .find(|level| level.buckets_per_second * seconds_per_column >= 1.0)
+ .unwrap_or_else(|| self.finest())
+ }
+
+ /// How many levels there are.
+ pub fn depth(&self) -> usize {
+ self.levels.len()
+ }
+}
+
/// Decodes one file's audio and reduces it to peaks.
///
/// The decode is mono 16-bit at [`PEAK_RATE`], of the audio stream `stream`
@@ -219,4 +323,47 @@ mod tests {
assert_eq!(back.max, peaks.max);
assert!(Peaks::decode(&bytes[..10]).is_none());
}
+
+ #[test]
+ fn a_coarser_level_keeps_the_extremes_and_the_tail() {
+ let fine = Peaks {
+ min: vec![-0.1, -0.9, -0.2, -0.3, -0.5],
+ max: vec![0.4, 0.2, 0.8, 0.1, 0.6],
+ buckets_per_second: 1000.0,
+ };
+ let half = fine.coarser(2);
+ assert_eq!(half.min, vec![-0.9, -0.3, -0.5]);
+ assert_eq!(half.max, vec![0.4, 0.8, 0.6]);
+ assert_eq!(half.buckets_per_second, 500.0);
+ assert_eq!(fine.coarser(0).len(), 5, "a factor of nothing is the same shape");
+ assert!(Peaks { min: vec![], max: vec![], buckets_per_second: 1000.0 }.coarser(4).is_empty());
+ }
+
+ #[test]
+ fn the_pyramid_hands_out_the_level_that_fits_the_column() {
+ let fine = Peaks {
+ min: vec![-0.5; 4096],
+ max: vec![0.5; 4096],
+ buckets_per_second: 1000.0,
+ };
+ let pyramid = Pyramid::of(fine);
+ assert_eq!(pyramid.depth(), 7, "4096 halves to 64 in six steps");
+ // A column of one millisecond reads the finest; of a second, the
+ // coarsest that still puts at least one bucket in it.
+ assert_eq!(pyramid.level_for(0.001).buckets_per_second, 1000.0);
+ let coarse = pyramid.level_for(1.0);
+ assert!(coarse.buckets_per_second <= 1000.0 / 64.0 + 1e-6, "{}", coarse.buckets_per_second);
+ assert!(coarse.buckets_per_second * 1.0 >= 1.0);
+ // A column finer than the finest bucket still gets the finest.
+ assert_eq!(pyramid.level_for(1e-9).buckets_per_second, 1000.0);
+ assert_eq!(pyramid.level_for(f32::NAN).buckets_per_second, 1000.0);
+ assert_eq!(pyramid.level_for(-1.0).buckets_per_second, 1000.0);
+ // Every level says the same about the whole file.
+ let (low, high) = pyramid.finest().extremes(0.0, 4.096);
+ let (clow, chigh) = coarse.extremes(0.0, 4.096);
+ assert_eq!((low, high), (clow, chigh));
+ // Past the end is silence, not a panic.
+ assert_eq!(coarse.extremes(100.0, 200.0), (0.0, 0.0));
+ assert_eq!(coarse.extremes(2.0, 1.0), (0.0, 0.0));
+ }
}
diff --git a/src/crates/concat/src/format.rs b/src/crates/concat/src/format.rs
index 17e5d11a..b8b73dd2 100644
--- a/src/crates/concat/src/format.rs
+++ b/src/crates/concat/src/format.rs
@@ -149,54 +149,45 @@ pub fn hex_of(colour: slint::Color) -> String {
)
}
-/// The audio envelope, as SVG path commands in a 1x1 box.
+/// A waveform as SVG path commands in a 1x1 box: one column per slot, each
+/// a bar from the slot's lowest sample to its highest, mirrored about the
+/// middle. Mirrored because that is what a waveform looks like; a bar
+/// from the axis up is the rounded shape that reads as a graph of
+/// something rather than as audio.
///
-/// Columns, not a curve. Each column is one rectangle, from the lowest to the
-/// highest peak under it, so the waveform is a stepped silhouette with a flat
-/// top on every column. Drawing the same peaks as a polyline gives a smooth,
-/// rounded shape that reads as a graph of something rather than as audio.
-///
-/// Built from the engine's real peaks: each column takes the extremes of
-/// the buckets that fall under it, so a trim shows the material it kept.
-/// Normalised rather than drawn in pixels so a zoom costs nothing: the Path
-/// that renders it stretches the box onto the clip's current width.
+/// Built from the engine's real peaks at the level that fits the column:
+/// each column takes the extremes of the buckets under it, so a trim shows
+/// the material it kept and a zoom shows the buckets it reveals. `columns`
+/// is how many the drawing has room for - the clip's width in pixels, held
+/// to a few thousand - and the path is normalised so the Path that renders
+/// it stretches the box onto the clip's current width.
pub fn wave_path(
- peaks: &concat_media::Peaks,
+ peaks: &concat_media::Pyramid,
source_start: f32,
duration: f32,
gain: f32,
+ columns: usize,
) -> String {
- /// Columns across the clip. Enough that the steps read as columns and
- /// not as a bar chart, few enough that the string stays a few kilobytes.
- const COLUMNS: usize = 128;
+ /// Fewest columns worth drawing, and the most: enough that a clip a
+ /// screen wide reads a column a pixel, few enough that the string stays
+ /// under a few hundred kilobytes.
+ const COLUMNS: std::ops::RangeInclusive<usize> = 8..=2048;
/// Silence still draws a sliver: a hairline through the middle of a clip
/// rather than a gap in it.
const FLOOR: f32 = 0.012;
- if duration <= 0.0 || peaks.min.is_empty() || peaks.buckets_per_second <= 0.0 {
+ if !(duration > 0.0) || peaks.finest().is_empty() {
return String::new();
}
+ let columns = columns.clamp(*COLUMNS.start(), *COLUMNS.end());
let gain = gain.max(0.0);
- let count = peaks.min.len().min(peaks.max.len());
- let per_second = peaks.buckets_per_second;
- let mut path = String::with_capacity(COLUMNS * 56);
+ let level = peaks.level_for(duration / columns as f32);
+ let mut path = String::with_capacity(columns * 56);
- for column in 0..COLUMNS {
- let left = column as f32 / COLUMNS as f32;
- let right = (column + 1) as f32 / COLUMNS as f32;
- let from = ((source_start + left * duration) * per_second)
- .floor()
- .max(0.0) as usize;
- let to = ((source_start + right * duration) * per_second)
- .ceil()
- .max(0.0) as usize;
- let (mut low, mut high) = (0.0f32, 0.0f32);
- for index in from..to.min(count).max(from) {
- if index < count {
- low = low.min(peaks.min[index]);
- high = high.max(peaks.max[index]);
- }
- }
+ for column in 0..columns {
+ let left = column as f32 / columns as f32;
+ let right = (column + 1) as f32 / columns as f32;
+ let (low, high) = level.extremes(source_start + left * duration, source_start + right * duration);
let amplitude = ((high.max(-low) * gain).clamp(0.0, 1.0) * 0.48).max(FLOOR);
let (top, bottom) = (0.5 - amplitude, 0.5 + amplitude);
path.push_str(&format!(
@@ -207,6 +198,18 @@ pub fn wave_path(
path
}
+/// How many columns a clip `seconds` long gets at `seconds_per_pixel`: one
+/// a pixel, rounded up to the next sixty-four so a zoom rebuilds the path
+/// at each step of that and not at every pixel, held to what `wave_path`
+/// draws.
+pub fn wave_columns(seconds: f32, seconds_per_pixel: f32) -> usize {
+ if !(seconds > 0.0) || !(seconds_per_pixel > 0.0) {
+ return 8;
+ }
+ let pixels = (seconds / seconds_per_pixel).ceil().max(1.0) as usize;
+ pixels.div_ceil(64).max(1).saturating_mul(64).clamp(8, 2048)
+}
+
/// A moment in the past, in the words a recents row wants: "just now",
/// "yesterday", "5 days ago".
pub fn when_phrase(opened_at_millis: u64) -> String {
@@ -271,13 +274,13 @@ mod tests {
#[test]
fn a_waveform_has_one_column_per_slot_and_follows_the_gain() {
- let peaks = concat_media::Peaks {
- min: vec![-0.5; 400],
- max: vec![0.5; 400],
- buckets_per_second: 200.0,
- };
- let loud = wave_path(&peaks, 0.0, 2.0, 1.0);
- let quiet = wave_path(&peaks, 0.0, 2.0, 0.25);
+ let peaks = concat_media::Pyramid::of(concat_media::Peaks {
+ min: vec![-0.5; 2000],
+ max: vec![0.5; 2000],
+ buckets_per_second: 1000.0,
+ });
+ let loud = wave_path(&peaks, 0.0, 2.0, 1.0, 128);
+ let quiet = wave_path(&peaks, 0.0, 2.0, 0.25, 128);
assert_eq!(loud.matches('M').count(), 128);
assert!(
loud.contains("0.2600"),
@@ -287,7 +290,46 @@ mod tests {
quiet.contains("0.4400"),
"an eighth at a quarter gain: {quiet}"
);
- assert!(wave_path(&peaks, 0.0, 0.0, 1.0).is_empty());
+ assert!(wave_path(&peaks, 0.0, 0.0, 1.0, 128).is_empty());
+ assert!(wave_path(&peaks, 0.0, f32::NAN, 1.0, 128).is_empty());
+ // The column count is held to what is worth drawing, either way.
+ assert_eq!(wave_path(&peaks, 0.0, 2.0, 1.0, 0).matches('M').count(), 8);
+ assert_eq!(wave_path(&peaks, 0.0, 2.0, 1.0, 1_000_000).matches('M').count(), 2048);
+ }
+
+ /// A zoomed-in clip reads the fine buckets: a single loud millisecond
+ /// shows in one column at a column a millisecond, and is folded into
+ /// its neighbours' column, still at full height, when a column is a
+ /// tenth of a second.
+ #[test]
+ fn zooming_in_reveals_the_fine_buckets_and_never_loses_a_peak() {
+ let mut min = vec![0.0; 1000];
+ let mut max = vec![0.0; 1000];
+ min[500] = -1.0;
+ max[500] = 1.0;
+ let peaks = concat_media::Pyramid::of(concat_media::Peaks {
+ min,
+ max,
+ buckets_per_second: 1000.0,
+ });
+ let fine = wave_path(&peaks, 0.0, 1.0, 1.0, 1000);
+ let tall: Vec<&str> = fine.split("Z ").filter(|bar| bar.contains(" 0.0200")).collect();
+ assert_eq!(tall.len(), 1, "one column carries the spike: {}", tall.len());
+ let coarse = wave_path(&peaks, 0.0, 1.0, 1.0, 10);
+ assert_eq!(coarse.matches(" 0.0200").count(), 2, "the spike survives the fold at full height");
+ let trimmed = wave_path(&peaks, 0.6, 0.4, 1.0, 10);
+ assert!(!trimmed.contains(" 0.0200"), "a trim past the spike does not show it");
+ }
+
+ #[test]
+ fn columns_follow_the_zoom_in_steps_of_sixty_four() {
+ assert_eq!(wave_columns(10.0, 0.05), 256, "200 px rounds up to 256");
+ assert_eq!(wave_columns(10.0, 0.01), 1024);
+ assert_eq!(wave_columns(600.0, 0.01), 2048, "held to the most worth drawing");
+ assert_eq!(wave_columns(0.1, 0.05), 64, "never under a step");
+ assert_eq!(wave_columns(0.0, 0.05), 8);
+ assert_eq!(wave_columns(f32::NAN, 0.05), 8);
+ assert_eq!(wave_columns(10.0, 0.0), 8);
}
#[test]
diff --git a/src/crates/concat/src/host.rs b/src/crates/concat/src/host.rs
index 79c83316..da01996f 100644
--- a/src/crates/concat/src/host.rs
+++ b/src/crates/concat/src/host.rs
@@ -523,7 +523,7 @@ pub struct MediaArt {
/// image: a Slint image cannot cross a thread, and this is made on one.
pub thumbnail: Option<concat_core::frame::Frame>,
/// The waveform, for anything with sound.
- pub peaks: Option<Arc<concat_media::Peaks>>,
+ pub peaks: Option<Arc<concat_media::Pyramid>>,
/// Frames sampled evenly across the footage, side by side in one
/// picture, and how many there are. A still is a strip of one.
pub strip: Option<(concat_core::frame::Frame, u32)>,
@@ -558,7 +558,7 @@ pub fn media_art(
thumbnail: None,
peaks: media::peaks(&path, stream, Some(&project))
.ok()
- .map(Arc::new),
+ .map(|peaks| Arc::new(concat_media::Pyramid::of(peaks))),
strip: None,
};
}
@@ -576,7 +576,11 @@ pub fn media_art(
None
};
let peaks = (kind == MediaKind::Audio || has_audio)
- .then(|| media::peaks(&path, None, Some(&project)).ok().map(Arc::new))
+ .then(|| {
+ media::peaks(&path, None, Some(&project))
+ .ok()
+ .map(|peaks| Arc::new(concat_media::Pyramid::of(peaks)))
+ })
.flatten();
// The filmstrip reads the proxy where the file has one: the tiles are
// small, and a 4K file's frames cost more than they show.