Open source CapCut alternative.
TypeScript
perf(site): the sound band fills in as the peaks decode walks the file [rebuild]
David Lec61d88c53eb3
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex 4eddc141..4ed2b59b 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -1536,6 +1536,30 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {useEditor.getState().updateAsset(asset.id, { thumbs: [src], thumbStep: IMAGE_CLIP_SECONDS });}} else if (asset.type === "video") {+ // The clip box draws the sound under the picture, so a video enriches+ // waveform peaks alongside its filmstrip. The two are separate reads of+ // the file, so the peaks decode starts here and runs beside the strip's+ // sweep: the sound band lands the moment its own read is done, however+ // long a source's frame grabs take. `[]` is an answer too — a silent+ // file — and stops the strip from re-decoding every open.+ const peaksRun =+ asset.peaks === undefined+ ? loadPeaks(asset, held ?? src, (partial) =>+ useEditor.getState().updateAsset(asset.id, { peaks: partial })+ )+ .then((peaks) => {+ if (peaks) useEditor.getState().updateAsset(asset.id, { peaks });+ })+ .catch((err: unknown) => {+ // The strip paints on its own; a peaks failure (an expired+ // signed URL, an undecodable track) never marks it failed.+ // What the walk had published comes off the asset, so the+ // next open reads the file again in place of keeping a+ // strip that stops partway.+ useEditor.getState().updateAsset(asset.id, { peaks: undefined });+ reportSwallowed(`[cut] peaks failed for ${asset.fileName}`, err);+ })+ : undefined;if (!stripComplete(asset)) {setStripFailed(asset.id, false);const key = stripCacheKey(useEditor.getState().projectId, asset.id);@@ -1568,23 +1592,19 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {reportSwallowed(`[cut] scene probe failed for ${asset.fileName}`, err);});}- // The clip box draws the sound under the picture, so a video enriches- // waveform peaks alongside its filmstrip. `[]` is an answer too — a- // silent file — and stops the strip from re-decoding every open.- if (asset.peaks === undefined) {- try {- const peaks = await loadPeaks(asset, held ?? src);- if (peaks) useEditor.getState().updateAsset(asset.id, { peaks });- } catch (err) {- // The strip above already painted; a peaks failure (an expired- // signed URL, an undecodable track) never marks it failed.- reportSwallowed(`[cut] peaks failed for ${asset.fileName}`, err);- }- }+ await peaksRun;enrichRetries.delete(asset.id);} else if (asset.type === "audio" && !asset.peaks?.length) {- const peaks = await loadPeaks(asset, held ?? src);- if (peaks) useEditor.getState().updateAsset(asset.id, { peaks });+ const publish = (peaks: number[]) => useEditor.getState().updateAsset(asset.id, { peaks });+ try {+ const peaks = await loadPeaks(asset, held ?? src, publish);+ if (peaks) publish(peaks);+ } catch (err) {+ // A walk that published partway comes off the asset before the retry+ // below, so the retry reads the file again.+ useEditor.getState().updateAsset(asset.id, { peaks: undefined });+ throw err;+ }enrichRetries.delete(asset.id);}} catch (err) {@@ -1659,13 +1679,15 @@ export async function detectAssetBeats(asset: MediaAsset): Promise<AssetBeats> {/** Waveform peaks on demand — e.g. when a video clip's audio is detached* onto the soundtrack track (video assets don't get peaks at import). */export async function ensurePeaks(asset: MediaAsset) {+ if (asset.peaks?.length) return;+ const publish = (peaks: number[]) => useEditor.getState().updateAsset(asset.id, { peaks });try {- if (!asset.peaks?.length) {- const peaks = await loadPeaks(asset);- if (peaks?.length) useEditor.getState().updateAsset(asset.id, { peaks });- }+ const peaks = await loadPeaks(asset, asset.url, publish);+ if (peaks?.length) publish(peaks);} catch (err) {- // Waveforms are decorative; editing works without them.+ // Waveforms are decorative; editing works without them. A walk that+ // published partway comes off the asset, so the next read starts clean.+ useEditor.getState().updateAsset(asset.id, { peaks: undefined });reportSwallowed(`[cut] peaks failed for ${asset.fileName}`, err);}}@@ -1755,12 +1777,16 @@ async function readCachedPeaks(key: string, duration: number): Promise<number[]* decoded — a real audio decode otherwise. Empty peaks are an answer (a file* with no audible track) and cache like any other. A headless runtime returns* undefined so the browser decodes on its next open. */-async function loadPeaks(asset: MediaAsset, src: string | Blob = asset.url): Promise<number[] | undefined> {+async function loadPeaks(+ asset: MediaAsset,+ src: string | Blob = asset.url,+ onPartial?: (peaks: number[]) => void+): Promise<number[] | undefined> {if (typeof AudioDecoder === "undefined") return undefined;const key = peaksCacheKey(useEditor.getState().projectId, asset.id);const cached = await readCachedPeaks(key, asset.duration);if (cached) return cached;- const peaks = await makePeaks(src, asset.duration);+ const peaks = await makePeaks(src, asset.duration, onPartial);writeCache(key, { peaks, duration: asset.duration, at: Date.now() });return peaks;}@@ -2202,10 +2228,14 @@ const PEAKS_PER_SECOND = 200;* source into each bar the further in it is zoomed. */const PEAKS_MAX = 60_000;-function makePeaks(src: string | Blob, duration: number): Promise<number[]> {+function makePeaks(+ src: string | Blob,+ duration: number,+ onPartial?: (peaks: number[]) => void+): Promise<number[]> {const buckets =duration > 0? Math.min(PEAKS_MAX, Math.max(400, Math.round(duration * PEAKS_PER_SECOND))): 1600;- return audioPeaks(src, buckets);+ return audioPeaks(src, buckets, onPartial);}diff --git a/site/src/cut/lib/mediaRead.ts b/site/src/cut/lib/mediaRead.tsindex 72e1b0e1..47a6661c 100644--- a/site/src/cut/lib/mediaRead.ts+++ b/site/src/cut/lib/mediaRead.ts@@ -578,14 +578,27 @@ export function openAudioWalk(src: string | Blob, from = 0, to?: number): AudioW};}+/** How often a running peaks decode publishes what it has folded so far. */+const PEAKS_PUBLISH_MS = 250;+/*** Normalized 0..1 waveform peaks across the whole file, `buckets` wide.** Folded chunk by chunk, so the cost is the decode rather than the decode plus* a copy of the file in memory — an hour of audio used to mean holding an hour* of float samples to draw a strip a few hundred pixels wide.+ *+ * The decode walks the file front to back, and a file that has to come over+ * the network walks at the link's pace, so `onPartial` hands out the buckets+ * folded so far as the walk runs: the strip is the full width from the first+ * chunk — the moments the walk has not reached yet sit flat — and fills in+ * left to right until the returned array is the whole read.*/-export async function audioPeaks(src: string | Blob, buckets: number): Promise<number[]> {+export async function audioPeaks(+ src: string | Blob,+ buckets: number,+ onPartial?: (peaks: number[]) => void+): Promise<number[]> {// Peaks are a real decode; a headless runtime skips them and the waveform// enriches the next time a browser holds the asset.if (typeof AudioDecoder === "undefined") return [];@@ -600,6 +613,7 @@ export async function audioPeaks(src: string | Blob, buckets: number): Promise<nif (!(duration > 0)) return [];const peaks = new Array<number>(buckets).fill(0);+ let published = 0;for await (const { buffer, timestamp } of new AudioBufferSink(track).buffers()) {const data = buffer.getChannelData(0);const rate = buffer.sampleRate;@@ -610,6 +624,11 @@ export async function audioPeaks(src: string | Blob, buckets: number): Promise<nconst bucket = Math.min(buckets - 1, Math.max(0, Math.floor(at * buckets)));if (v > peaks[bucket]) peaks[bucket] = v;}+ if (!onPartial) continue;+ const now = Date.now();+ if (now - published < PEAKS_PUBLISH_MS) continue;+ published = now;+ onPartial(peaks.slice());}return peaks;} finally {