Skip to content

Open source CapCut alternative.

TypeScript

perf(site): the sound band fills in as the peaks decode walks the file [rebuild]

David Lec61d88c53eb3
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index 4eddc141..4ed2b59b 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -1536,6 +1536,30 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {
useEditor.getState().updateAsset(asset.id, { thumbs: [src], thumbStep: IMAGE_CLIP_SECONDS });
}
} else if (asset.type === "video") {
+ // The clip box draws the sound under the picture, so a video enriches
+ // waveform peaks alongside its filmstrip. The two are separate reads of
+ // the file, so the peaks decode starts here and runs beside the strip's
+ // sweep: the sound band lands the moment its own read is done, however
+ // long a source's frame grabs take. `[]` is an answer too — a silent
+ // file — and stops the strip from re-decoding every open.
+ const peaksRun =
+ asset.peaks === undefined
+ ? loadPeaks(asset, held ?? src, (partial) =>
+ useEditor.getState().updateAsset(asset.id, { peaks: partial })
+ )
+ .then((peaks) => {
+ if (peaks) useEditor.getState().updateAsset(asset.id, { peaks });
+ })
+ .catch((err: unknown) => {
+ // The strip paints on its own; a peaks failure (an expired
+ // signed URL, an undecodable track) never marks it failed.
+ // What the walk had published comes off the asset, so the
+ // next open reads the file again in place of keeping a
+ // strip that stops partway.
+ useEditor.getState().updateAsset(asset.id, { peaks: undefined });
+ reportSwallowed(`[cut] peaks failed for ${asset.fileName}`, err);
+ })
+ : undefined;
if (!stripComplete(asset)) {
setStripFailed(asset.id, false);
const key = stripCacheKey(useEditor.getState().projectId, asset.id);
@@ -1568,23 +1592,19 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {
reportSwallowed(`[cut] scene probe failed for ${asset.fileName}`, err);
});
}
- // The clip box draws the sound under the picture, so a video enriches
- // waveform peaks alongside its filmstrip. `[]` is an answer too — a
- // silent file — and stops the strip from re-decoding every open.
- if (asset.peaks === undefined) {
- try {
- const peaks = await loadPeaks(asset, held ?? src);
- if (peaks) useEditor.getState().updateAsset(asset.id, { peaks });
- } catch (err) {
- // The strip above already painted; a peaks failure (an expired
- // signed URL, an undecodable track) never marks it failed.
- reportSwallowed(`[cut] peaks failed for ${asset.fileName}`, err);
- }
- }
+ await peaksRun;
enrichRetries.delete(asset.id);
} else if (asset.type === "audio" && !asset.peaks?.length) {
- const peaks = await loadPeaks(asset, held ?? src);
- if (peaks) useEditor.getState().updateAsset(asset.id, { peaks });
+ const publish = (peaks: number[]) => useEditor.getState().updateAsset(asset.id, { peaks });
+ try {
+ const peaks = await loadPeaks(asset, held ?? src, publish);
+ if (peaks) publish(peaks);
+ } catch (err) {
+ // A walk that published partway comes off the asset before the retry
+ // below, so the retry reads the file again.
+ useEditor.getState().updateAsset(asset.id, { peaks: undefined });
+ throw err;
+ }
enrichRetries.delete(asset.id);
}
} catch (err) {
@@ -1659,13 +1679,15 @@ export async function detectAssetBeats(asset: MediaAsset): Promise<AssetBeats> {
/** Waveform peaks on demand — e.g. when a video clip's audio is detached
* onto the soundtrack track (video assets don't get peaks at import). */
export async function ensurePeaks(asset: MediaAsset) {
+ if (asset.peaks?.length) return;
+ const publish = (peaks: number[]) => useEditor.getState().updateAsset(asset.id, { peaks });
try {
- if (!asset.peaks?.length) {
- const peaks = await loadPeaks(asset);
- if (peaks?.length) useEditor.getState().updateAsset(asset.id, { peaks });
- }
+ const peaks = await loadPeaks(asset, asset.url, publish);
+ if (peaks?.length) publish(peaks);
} catch (err) {
- // Waveforms are decorative; editing works without them.
+ // Waveforms are decorative; editing works without them. A walk that
+ // published partway comes off the asset, so the next read starts clean.
+ useEditor.getState().updateAsset(asset.id, { peaks: undefined });
reportSwallowed(`[cut] peaks failed for ${asset.fileName}`, err);
}
}
@@ -1755,12 +1777,16 @@ async function readCachedPeaks(key: string, duration: number): Promise<number[]
* decoded — a real audio decode otherwise. Empty peaks are an answer (a file
* with no audible track) and cache like any other. A headless runtime returns
* undefined so the browser decodes on its next open. */
-async function loadPeaks(asset: MediaAsset, src: string | Blob = asset.url): Promise<number[] | undefined> {
+async function loadPeaks(
+ asset: MediaAsset,
+ src: string | Blob = asset.url,
+ onPartial?: (peaks: number[]) => void
+): Promise<number[] | undefined> {
if (typeof AudioDecoder === "undefined") return undefined;
const key = peaksCacheKey(useEditor.getState().projectId, asset.id);
const cached = await readCachedPeaks(key, asset.duration);
if (cached) return cached;
- const peaks = await makePeaks(src, asset.duration);
+ const peaks = await makePeaks(src, asset.duration, onPartial);
writeCache(key, { peaks, duration: asset.duration, at: Date.now() });
return peaks;
}
@@ -2202,10 +2228,14 @@ const PEAKS_PER_SECOND = 200;
* source into each bar the further in it is zoomed. */
const PEAKS_MAX = 60_000;
-function makePeaks(src: string | Blob, duration: number): Promise<number[]> {
+function makePeaks(
+ src: string | Blob,
+ duration: number,
+ onPartial?: (peaks: number[]) => void
+): Promise<number[]> {
const buckets =
duration > 0
? Math.min(PEAKS_MAX, Math.max(400, Math.round(duration * PEAKS_PER_SECOND)))
: 1600;
- return audioPeaks(src, buckets);
+ return audioPeaks(src, buckets, onPartial);
}
diff --git a/site/src/cut/lib/mediaRead.ts b/site/src/cut/lib/mediaRead.ts
index 72e1b0e1..47a6661c 100644
--- a/site/src/cut/lib/mediaRead.ts
+++ b/site/src/cut/lib/mediaRead.ts
@@ -578,14 +578,27 @@ export function openAudioWalk(src: string | Blob, from = 0, to?: number): AudioW
};
}
+/** How often a running peaks decode publishes what it has folded so far. */
+const PEAKS_PUBLISH_MS = 250;
+
/**
* Normalized 0..1 waveform peaks across the whole file, `buckets` wide.
*
* Folded chunk by chunk, so the cost is the decode rather than the decode plus
* a copy of the file in memory — an hour of audio used to mean holding an hour
* of float samples to draw a strip a few hundred pixels wide.
+ *
+ * The decode walks the file front to back, and a file that has to come over
+ * the network walks at the link's pace, so `onPartial` hands out the buckets
+ * folded so far as the walk runs: the strip is the full width from the first
+ * chunk — the moments the walk has not reached yet sit flat — and fills in
+ * left to right until the returned array is the whole read.
*/
-export async function audioPeaks(src: string | Blob, buckets: number): Promise<number[]> {
+export async function audioPeaks(
+ src: string | Blob,
+ buckets: number,
+ onPartial?: (peaks: number[]) => void
+): Promise<number[]> {
// Peaks are a real decode; a headless runtime skips them and the waveform
// enriches the next time a browser holds the asset.
if (typeof AudioDecoder === "undefined") return [];
@@ -600,6 +613,7 @@ export async function audioPeaks(src: string | Blob, buckets: number): Promise<n
if (!(duration > 0)) return [];
const peaks = new Array<number>(buckets).fill(0);
+ let published = 0;
for await (const { buffer, timestamp } of new AudioBufferSink(track).buffers()) {
const data = buffer.getChannelData(0);
const rate = buffer.sampleRate;
@@ -610,6 +624,11 @@ export async function audioPeaks(src: string | Blob, buckets: number): Promise<n
const bucket = Math.min(buckets - 1, Math.max(0, Math.floor(at * buckets)));
if (v > peaks[bucket]) peaks[bucket] = v;
}
+ if (!onPartial) continue;
+ const now = Date.now();
+ if (now - published < PEAKS_PUBLISH_MS) continue;
+ published = now;
+ onPartial(peaks.slice());
}
return peaks;
} finally {