The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
refactor: decode waveform peaks in the engine, cap read_media_bytes
Jareer8c1a72c6cce4
diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rsindex e37b9b4e..65df2185 100644--- a/desktop/src-tauri/src/lib.rs+++ b/desktop/src-tauri/src/lib.rs@@ -163,18 +163,36 @@ fn engine_version() -> &'static str {env!("CARGO_PKG_VERSION")}+/// The most `read_media_bytes` will hand the webview at once.+///+/// Its remaining callers decode still images and register fonts - assets+/// that are megabytes, not gigabytes. The cap is what keeps the command+/// from quietly growing back into a whole-disk read primitive now that+/// waveform peaks (the one legitimate whole-file consumer) decode in the+/// engine instead.+const MEDIA_READ_CAP: u64 = 64 * 1024 * 1024;+/// Reads a whole file and returns it as raw bytes.///-/// Two callers: the waveform peak decode in `lib/assets.ts`, and custom font+/// Two callers: still-image decode in `lib/assets.ts`, and custom font/// registration. `tauri::ipc::Response` puts the bytes on the wire as an/// ArrayBuffer rather than a JSON array of numbers, which for a few/// megabytes is the difference between instant and unusable.////// It loads the entire file into memory, so it is not a general-purpose-/// reader and must not become one.+/// reader and must not become one: anything past [`MEDIA_READ_CAP`] is+/// refused.#[tauri::command]async fn read_media_bytes(path: String) -> Result<tauri::ipc::Response, String> {tauri::async_runtime::spawn_blocking(move || {+ let size = std::fs::metadata(&path)+ .map_err(|error| format!("could not read {path}: {error}"))?+ .len();+ if size > MEDIA_READ_CAP {+ return Err(format!(+ "refusing to read {path}: {size} bytes is over the {MEDIA_READ_CAP} byte limit"+ ));+ }std::fs::read(&path).map_err(|error| format!("could not read {path}: {error}"))}).await@@ -182,6 +200,83 @@ async fn read_media_bytes(path: String) -> Result<tauri::ipc::Response, String>.map(tauri::ipc::Response::new)}+/// Resolution of the cached waveform.+///+/// 200 buckets per second is roughly two buckets per pixel at the default+/// timeline zoom, which is enough that the drawn shape does not visibly+/// change as you zoom in a step or two, without storing the whole decoded+/// file.+const PEAKS_BUCKETS_PER_SECOND: u32 = 200;++/// Waveform peaks for one media file: engine-decoded, project-cached.+///+/// The engine streams FFmpeg's decode into min/max buckets, so neither the+/// file nor its samples are ever resident - and nothing crosses the IPC+/// boundary but the buckets themselves. The result is cached in the+/// project's `cache/` folder under a key derived from the path, and served+/// from there on every later call; `project: None` (an unsaved session)+/// just skips the cache.+#[tauri::command]+async fn extract_peaks(path: String, project: Option<String>) -> Result<tauri::ipc::Response, String> {+ tauri::async_runtime::spawn_blocking(move || peaks_bytes(&path, project.as_deref()))+ .await+ .map_err(|error| format!("peaks task failed: {error}"))?+ .map(tauri::ipc::Response::new)+}++fn peaks_bytes(path: &str, project: Option<&str>) -> Result<Vec<u8>, String> {+ let cached = project.and_then(|project| artwork_file(project, &peaks_key(path)).ok());+ if let Some(file) = &cached {+ if let Ok(bytes) = std::fs::read(file) {+ // A corrupt entry falls through to regeneration rather than+ // being served - the frontend has no second request to make.+ if plausible_peaks(&bytes) {+ return Ok(bytes);+ }+ }+ }++ let peaks = wolfcut_media::peaks::extract(std::path::Path::new(path), PEAKS_BUCKETS_PER_SECOND)+ .map_err(describe)?;+ let bytes = peaks.encode();++ // Best-effort, like every artwork write: a failed cache write only+ // means decoding again next launch.+ if let Some(file) = &cached {+ if let Some(parent) = file.parent() {+ let _ = std::fs::create_dir_all(parent);+ }+ let _ = std::fs::write(file, &bytes);+ }+ Ok(bytes)+}++/// The cache filename for one file's peaks.+///+/// FNV-1a 64 over the path, like the audio cache's `decode_key` and for the+/// same reason: these keys name files that outlive the process, and+/// `DefaultHasher` is free to change between Rust releases. The bucket rate+/// rides in the name so a resolution change regenerates instead of serving+/// yesterday's shape.+fn peaks_key(path: &str) -> String {+ let mut hash: u64 = 0xcbf2_9ce4_8422_2325;+ for &byte in path.as_bytes() {+ hash ^= u64::from(byte);+ hash = hash.wrapping_mul(0x0000_0100_0000_01b3);+ }+ format!("{hash:016x}-b{PEAKS_BUCKETS_PER_SECOND}.peaks")+}++/// Whether bytes have the shape `Peaks::encode` writes:+/// `[rate f32][count u32][min f32 x count][max f32 x count]`.+fn plausible_peaks(bytes: &[u8]) -> bool {+ if bytes.len() < 8 {+ return false;+ }+ let count = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]) as usize;+ bytes.len() == 8 + count * 8+}+/// Where one artwork file lives inside a project's cache.////// The cache sits in the project folder so it travels with the project and@@ -749,6 +844,7 @@ pub fn run() {probe_media,engine_version,read_media_bytes,+ extract_peaks,audio_set_clips,transport_play,transport_pause,diff --git a/desktop/src/lib/assets.ts b/desktop/src/lib/assets.tsindex 1d199b4d..dff42629 100644--- a/desktop/src/lib/assets.ts+++ b/desktop/src/lib/assets.ts@@ -14,6 +14,7 @@import {extractFilmstrip,+ extractPeaks,readArtwork as readArtworkFile,readMediaBytes,writeArtwork as writeArtworkFile,@@ -64,15 +65,6 @@ function announce(assets: MediaAssets): void {for (const listener of assets.listeners) listener();}-/**- * Resolution of the cached waveform.- *- * 200 buckets per second is roughly two buckets per pixel at the default zoom,- * which is enough that the drawn shape does not visibly change as you zoom in- * a step or two, without storing the whole decoded file.- */-const BUCKETS_PER_SECOND = 200;-/** Frames per filmstrip. Enough to read the shot, few enough to stay cheap. */const STRIP_FRAMES = 24;/**@@ -103,10 +95,9 @@ export function requestAssets(): void {if (assets.pending.has(media.id)) return;- // Video peaks are deliberately NOT loaded here. Decoding a video's audio- // means reading the whole file and crunching every sample, which stalls the- // main thread hard enough to make playback stutter - so it happens only on- // demand, via requestVideoPeaks, when a detached audio clip needs drawing.+ // Video peaks are deliberately NOT loaded here. Even engine-side, decoding+ // a video's whole audio track is real work - so it happens only on demand,+ // via requestVideoPeaks, when a detached audio clip needs drawing.const wantsPeaks = media.kind === "audio" && !assets.peaks.has(media.id);const wantsStrip = media.kind !== "audio" && !assets.strips.has(media.id);if (!wantsPeaks && !wantsStrip) return;@@ -153,8 +144,8 @@ export function requestAssets(** Called when a detached audio clip exists (or is being created) for a video,* which is the only time a video needs peaks. Expensive on first run - the- * whole file is read and decoded - but the result lands in the disk cache, so- * a reopened project pays only a file read.+ * engine decodes the whole audio track - but the result lands in the disk+ * cache, so a reopened project pays only a file read.*/export function requestVideoPeaks(assets: MediaAssets,@@ -242,48 +233,17 @@ export function decodePeaks(bytes: ArrayBuffer): Peaks | null {};}-/** Decodes a file and reduces it to min/max pairs. */+/**+ * Asks the engine for a file's peaks.+ *+ * The decode, the bucketing and the disk cache all live host-side now - the+ * engine streams FFmpeg's output straight into buckets, so neither the file+ * nor its samples ever cross the IPC boundary. What arrives here is the+ * encoded buckets, ready for `decodePeaks`.+ */async function loadPeaks(path: string, project: string | null): Promise<Peaks | null> {- const key = `${artworkKey(path)}-b${BUCKETS_PER_SECOND}.peaks`;- const cached = await readArtwork(project, key);- if (cached) {- const peaks = decodePeaks(cached);- if (peaks) return peaks;- }-try {- const bytes = await readMediaBytes(path);-- // An OfflineAudioContext decodes without opening an output device, which- // matters because this runs at import time with no user gesture behind it.- const context = new OfflineAudioContext(1, 1, 44100);- const buffer = await context.decodeAudioData(bytes);-- const samples = buffer.getChannelData(0);- const bucketSize = Math.max(1, Math.floor(buffer.sampleRate / BUCKETS_PER_SECOND));- const bucketCount = Math.ceil(samples.length / bucketSize);-- const min = new Float32Array(bucketCount);- const max = new Float32Array(bucketCount);-- for (let bucket = 0; bucket < bucketCount; bucket += 1) {- const from = bucket * bucketSize;- const to = Math.min(from + bucketSize, samples.length);-- let low = 0;- let high = 0;- for (let index = from; index < to; index += 1) {- const sample = samples[index];- if (sample < low) low = sample;- if (sample > high) high = sample;- }- min[bucket] = low;- max[bucket] = high;- }-- const peaks = { min, max, bucketsPerSecond: buffer.sampleRate / bucketSize };- writeArtwork(project, key, encodePeaks(peaks));- return peaks;+ return decodePeaks(await extractPeaks(path, project));} catch (cause) {console.warn(`WolfCut: no waveform for ${path}`, cause);return null;diff --git a/desktop/src/lib/engine.ts b/desktop/src/lib/engine.tsindex 292f46ac..1f7832a0 100644--- a/desktop/src/lib/engine.ts+++ b/desktop/src/lib/engine.ts@@ -76,14 +76,25 @@ export async function engineVersion(): Promise<string> {/*** Reads a whole file into memory as bytes.*- * Two callers: the waveform peak decode in `lib/assets.ts`, and custom font- * registration. It loads the entire file, so do not reach for it as a- * general-purpose reader.+ * Two callers: still-image decode in `lib/assets.ts`, and custom font+ * registration. It loads the entire file and the host refuses anything past+ * its size cap, so do not reach for it as a general-purpose reader.*/export async function readMediaBytes(path: string): Promise<ArrayBuffer> {return invoke<ArrayBuffer>("read_media_bytes", { path });}+/**+ * Waveform peaks for one media file, decoded and bucketed by the engine.+ *+ * Returns the encoded form `decodePeaks` in `lib/assets.ts` reads. The host+ * caches the result in the project folder, so only the first call per file+ * pays for a decode; `project: null` skips that cache.+ */+export async function extractPeaks(path: string, project: string | null): Promise<ArrayBuffer> {+ return invoke<ArrayBuffer>("extract_peaks", { path, project });+}+/** A project on disk, as the launch screen sees it. */export interface ProjectInfo {diff --git a/engine/crates/wolfcut-media/src/lib.rs b/engine/crates/wolfcut-media/src/lib.rsindex 76d638c9..abd479d5 100644--- a/engine/crates/wolfcut-media/src/lib.rs+++ b/engine/crates/wolfcut-media/src/lib.rs@@ -16,6 +16,7 @@ pub mod encode;#[cfg(feature = "ffi")]pub mod ffi;pub mod error;+pub mod peaks;pub mod pool;pub mod probe;@@ -25,5 +26,6 @@ pub use binaries::{ffmpeg, ffprobe, set_binaries};pub use decode::{DecodeOptions, FfmpegDecoder, FrameSource, SeekableSource};pub use encode::{EncodeOptions, FfmpegEncoder, FrameSink};pub use error::{Error, Result};+pub use peaks::Peaks;pub use pool::{FrameCache, ReaderPool};pub use probe::{AudioStream, MediaInfo, VideoStream, probe};diff --git a/engine/crates/wolfcut-media/src/peaks.rs b/engine/crates/wolfcut-media/src/peaks.rsnew file mode 100644index 00000000..5e471310--- /dev/null+++ b/engine/crates/wolfcut-media/src/peaks.rs@@ -0,0 +1,301 @@+//! Waveform peaks: min/max sample pairs, bucketed at a fixed rate.+//!+//! The timeline draws a clip's waveform from a few hundred buckets per+//! second, not from the samples themselves. This module produces those+//! buckets by streaming FFmpeg's decode of the file's audio - the samples+//! are folded into buckets as they arrive and never accumulate, so an+//! hour-long recording costs the same memory as a jingle.+//!+//! This used to live in the UI, which read the entire file over IPC and+//! decoded it with WebAudio on the main thread. Moving it here removed the+//! app's last whole-file read across the IPC boundary.++use std::io::Read;+use std::path::Path;+use std::process::Stdio;++use crate::binaries::ffmpeg;+use crate::error::{Error, Result};+use crate::process::{StderrTail, base_command};++/// The rate audio is resampled to before bucketing.+///+/// A fixed rate makes the bucket size exact - at 200 buckets per second a+/// 48 kHz stream folds precisely 240 samples per bucket - so the encoded+/// `buckets_per_second` is the requested number, not a near miss that+/// depends on the source file's native rate.+const PEAK_RATE: u32 = 48_000;++/// A file's waveform, reduced to per-bucket extremes.+///+/// Buckets are seeded at zero rather than ±infinity: silence reads as a+/// flat 0/0 pair, and a bucket's minimum can never sit above the axis.+/// That is the shape the timeline has always drawn, and the on-disk caches+/// already hold it.+pub struct Peaks {+ /// The lowest sample in each bucket, in [-1, 0].+ pub min: Vec<f32>,+ /// The highest sample in each bucket, in [0, 1].+ pub max: Vec<f32>,+ /// How many buckets cover one second of audio.+ pub buckets_per_second: f32,+}++impl Peaks {+ /// The wire and cache format the UI reads:+ /// `[buckets_per_second f32][count u32][min f32 x count][max f32 x count]`,+ /// little-endian. Kept byte-identical to what the UI used to write, so+ /// existing project caches stay valid.+ pub fn encode(&self) -> Vec<u8> {+ let count = self.min.len().min(self.max.len());+ let mut bytes = Vec::with_capacity(8 + count * 8);+ bytes.extend_from_slice(&self.buckets_per_second.to_le_bytes());+ bytes.extend_from_slice(&(count as u32).to_le_bytes());+ for value in self.min.iter().take(count) {+ bytes.extend_from_slice(&value.to_le_bytes());+ }+ for value in self.max.iter().take(count) {+ bytes.extend_from_slice(&value.to_le_bytes());+ }+ bytes+ }+}++/// Decodes one file's audio and reduces it to peaks.+///+/// The decode is mono 16-bit at [`PEAK_RATE`], piped back as WAV - `wav`+/// rather than raw `s16le` because the bundled FFmpeg is a trimmed build+/// and the raw muxer is not in it. A file with no audio stream fails the+/// FFmpeg run, and the error carries what FFmpeg said.+pub fn extract(path: &Path, buckets_per_second: u32) -> Result<Peaks> {+ let mut child = base_command(ffmpeg())+ .arg("-i")+ .arg(path)+ .args(["-vn", "-ac", "1"])+ .args(["-ar", &PEAK_RATE.to_string()])+ .args(["-c:a", "pcm_s16le", "-f", "wav", "pipe:1"])+ .stdin(Stdio::null())+ .stdout(Stdio::piped())+ .stderr(Stdio::piped())+ .spawn()+ .map_err(|source| Error::Spawn { program: "ffmpeg", source })?;++ let mut tail = StderrTail::drain(&mut child);+ let mut stdout = child.stdout.take().expect("stdout was piped");+ let parsed = peaks_from_wav(&mut stdout, path, buckets_per_second);++ if parsed.is_err() {+ // A parse failure leaves FFmpeg still writing into a pipe nobody+ // reads; waiting on it now would deadlock on the full pipe.+ let _ = child.kill();+ }+ let status = child.wait().map_err(|source| Error::Io { program: "ffmpeg", source })?;++ // FFmpeg's own account of the failure - "no audio stream", "no such+ // file" - beats whatever the parser tripped over downstream of it.+ if !status.success() {+ return Err(Error::Exited {+ program: "ffmpeg",+ path: path.to_path_buf(),+ status,+ stderr: tail.summary(),+ });+ }+ parsed+}++/// Folds a WAV stream into peaks without holding the samples.+///+/// A plain chunk walk, not a format library: the stream comes from our own+/// FFmpeg invocation above, so the only variability is which bookkeeping+/// chunks precede `data`. Sizes written to a pipe are `u32::MAX`+/// placeholders - the muxer could not seek back to patch them - in which+/// case the payload is simply the rest of the stream.+fn peaks_from_wav(+ reader: &mut impl Read,+ path: &Path,+ buckets_per_second: u32,+) -> Result<Peaks> {+ let malformed = |detail: &str| Error::Probe {+ path: path.to_path_buf(),+ detail: format!("ffmpeg piped back something other than the wav asked for: {detail}"),+ };+ let io = |source| Error::Io { program: "ffmpeg", source };++ let buckets_per_second = buckets_per_second.clamp(1, PEAK_RATE);+ let bucket_size = (PEAK_RATE / buckets_per_second) as usize;++ let mut riff = [0u8; 12];+ reader.read_exact(&mut riff).map_err(io)?;+ if &riff[0..4] != b"RIFF" || &riff[8..12] != b"WAVE" {+ return Err(malformed("missing RIFF/WAVE header"));+ }++ // Walk the bookkeeping chunks until `data`. Its payload runs to EOF+ // when the size is a placeholder.+ let mut remaining: Option<usize> = loop {+ let mut header = [0u8; 8];+ reader.read_exact(&mut header).map_err(io)?;+ let size = u32::from_le_bytes([header[4], header[5], header[6], header[7]]);+ if &header[0..4] == b"data" {+ break if size == u32::MAX { None } else { Some(size as usize) };+ }+ // A placeholder size on any other chunk means the walk cannot+ // continue - refuse rather than misread samples as headers.+ if size == u32::MAX {+ return Err(malformed("placeholder size before the data chunk"));+ }+ // Chunks are word-aligned; an odd size is followed by a pad byte.+ let skip = size as usize + (size as usize & 1);+ std::io::copy(&mut reader.take(skip as u64), &mut std::io::sink()).map_err(io)?;+ };++ let mut min = Vec::new();+ let mut max = Vec::new();+ let mut low = 0.0f32;+ let mut high = 0.0f32;+ let mut filled = 0usize;++ let mut chunk = [0u8; 8192];+ // A sample split across two reads: its first byte waits here.+ let mut half: Option<u8> = None;+ loop {+ let want = remaining.map_or(chunk.len(), |left| left.min(chunk.len()));+ if want == 0 {+ break;+ }+ let read = reader.read(&mut chunk[..want]).map_err(io)?;+ if read == 0 {+ break;+ }+ if let Some(left) = remaining.as_mut() {+ *left -= read;+ }++ let mut bytes = &chunk[..read];+ if let Some(first) = half.take() {+ let sample = f32::from(i16::from_le_bytes([first, bytes[0]])) / 32768.0;+ fold(sample, &mut low, &mut high);+ bump(&mut filled, bucket_size, &mut min, &mut max, &mut low, &mut high);+ bytes = &bytes[1..];+ }+ for pair in bytes.chunks_exact(2) {+ let sample = f32::from(i16::from_le_bytes([pair[0], pair[1]])) / 32768.0;+ fold(sample, &mut low, &mut high);+ bump(&mut filled, bucket_size, &mut min, &mut max, &mut low, &mut high);+ }+ if bytes.len() & 1 == 1 {+ half = Some(bytes[bytes.len() - 1]);+ }+ }++ // The trailing partial bucket still counts - dropping it would shave+ // the last fraction of a second off every waveform.+ if filled > 0 {+ min.push(low);+ max.push(high);+ }++ Ok(Peaks {+ min,+ max,+ buckets_per_second: PEAK_RATE as f32 / bucket_size as f32,+ })+}++fn fold(sample: f32, low: &mut f32, high: &mut f32) {+ if sample < *low {+ *low = sample;+ }+ if sample > *high {+ *high = sample;+ }+}++fn bump(+ filled: &mut usize,+ bucket_size: usize,+ min: &mut Vec<f32>,+ max: &mut Vec<f32>,+ low: &mut f32,+ high: &mut f32,+) {+ *filled += 1;+ if *filled == bucket_size {+ min.push(*low);+ max.push(*high);+ *low = 0.0;+ *high = 0.0;+ *filled = 0;+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ /// A WAV as FFmpeg writes one to a pipe: placeholder sizes, an `fmt `+ /// chunk, then `data` running to EOF.+ fn piped_wav(samples: &[i16]) -> Vec<u8> {+ let mut bytes = Vec::new();+ bytes.extend_from_slice(b"RIFF");+ bytes.extend_from_slice(&u32::MAX.to_le_bytes());+ bytes.extend_from_slice(b"WAVE");+ bytes.extend_from_slice(b"fmt ");+ bytes.extend_from_slice(&16u32.to_le_bytes());+ bytes.extend_from_slice(&[0u8; 16]);+ bytes.extend_from_slice(b"data");+ bytes.extend_from_slice(&u32::MAX.to_le_bytes());+ for sample in samples {+ bytes.extend_from_slice(&sample.to_le_bytes());+ }+ bytes+ }++ #[test]+ fn buckets_carry_the_extremes_and_the_tail_partial_counts() {+ // 240 samples per bucket at 200 buckets/second: one full bucket+ // holding the extremes, then a 10-sample partial.+ let mut samples = vec![0i16; 240];+ samples[7] = i16::MIN;+ samples[100] = 16384;+ samples.extend(std::iter::repeat_n(-8192i16, 10));++ let wav = piped_wav(&samples);+ let peaks = peaks_from_wav(&mut wav.as_slice(), Path::new("test.wav"), 200).unwrap();++ assert_eq!(peaks.buckets_per_second, 200.0);+ assert_eq!(peaks.min.len(), 2);+ assert_eq!(peaks.min[0], -1.0);+ assert_eq!(peaks.max[0], 0.5);+ assert_eq!(peaks.min[1], -0.25);+ // Seeded at zero: an all-negative bucket still reports max 0.+ assert_eq!(peaks.max[1], 0.0);+ }++ #[test]+ fn silence_is_flat_zeroes() {+ let wav = piped_wav(&[0i16; 480]);+ let peaks = peaks_from_wav(&mut wav.as_slice(), Path::new("test.wav"), 200).unwrap();+ assert_eq!(peaks.min, vec![0.0, 0.0]);+ assert_eq!(peaks.max, vec![0.0, 0.0]);+ }++ #[test]+ fn a_stream_that_is_not_wav_is_refused() {+ let junk = b"MPEG something else entirely, long enough to read".to_vec();+ let error = peaks_from_wav(&mut junk.as_slice(), Path::new("test.mp3"), 200);+ assert!(error.is_err());+ }++ #[test]+ fn encode_lays_out_rate_count_min_max() {+ let peaks = Peaks { min: vec![-0.5, 0.0], max: vec![0.25, 0.0], buckets_per_second: 200.0 };+ let bytes = peaks.encode();+ assert_eq!(bytes.len(), 8 + 2 * 8);+ assert_eq!(f32::from_le_bytes(bytes[0..4].try_into().unwrap()), 200.0);+ assert_eq!(u32::from_le_bytes(bytes[4..8].try_into().unwrap()), 2);+ assert_eq!(f32::from_le_bytes(bytes[8..12].try_into().unwrap()), -0.5);+ assert_eq!(f32::from_le_bytes(bytes[16..20].try_into().unwrap()), 0.25);+ }+}