Skip to content

🎥 Make videos programmatically with React

`@remotion/studio`: Fix waveform peak sampling when zooming or trimming

Jonny Burger2ed96e3f1ea3
diff --git a/packages/studio/src/components/AudioWaveform.tsx b/packages/studio/src/components/AudioWaveform.tsx
index 3bed8237f6..1e7d336166 100644
--- a/packages/studio/src/components/AudioWaveform.tsx
+++ b/packages/studio/src/components/AudioWaveform.tsx
@@ -1,8 +1,8 @@
import {
drawBars,
getVisibleWaveformVolume,
- sliceVisibleWaveformPeaks,
subscribeToWaveformPeaks,
+ type WaveformDrawRange,
type WaveformVolume,
} from '@remotion/timeline-utils';
import React, {useLayoutEffect, useMemo, useRef, useState} from 'react';
@@ -115,29 +115,25 @@ const AudioWaveformInner: React.FC<{
});
}, [src, waveformSampleRate]);
- const portionPeaks = useMemo(() => {
- if (!peaks) {
- return null;
- }
-
- return sliceVisibleWaveformPeaks({
- displayDurationInFrames,
- displayOffsetInFrames: displayOffsetInFrames + loopDisplayOffsetInFrames,
- durationInFrames,
- fps: vidConf.fps,
- loopDisplay,
- peaks,
- playbackRate,
- startFrom,
- waveformSampleRate,
- });
+ const peakRange = useMemo((): WaveformDrawRange => {
+ const peaksPerFrame = (waveformSampleRate * playbackRate) / vidConf.fps;
+ const loop = loopDisplay !== undefined && loopDisplay.durationInFrames > 0;
+ return {
+ sourceStart: (startFrom / vidConf.fps) * waveformSampleRate,
+ sourceDuration:
+ (loop ? loopDisplay.durationInFrames : durationInFrames) *
+ peaksPerFrame,
+ displayStart:
+ (displayOffsetInFrames + loopDisplayOffsetInFrames) * peaksPerFrame,
+ displayDuration: displayDurationInFrames * peaksPerFrame,
+ loop,
+ };
}, [
displayDurationInFrames,
displayOffsetInFrames,
durationInFrames,
loopDisplay,
loopDisplayOffsetInFrames,
- peaks,
playbackRate,
startFrom,
vidConf.fps,
@@ -164,7 +160,8 @@ const AudioWaveformInner: React.FC<{
drawBars({
canvas: canvasElement,
- peaks: portionPeaks ?? EMPTY_PEAKS,
+ peaks: peaks ?? EMPTY_PEAKS,
+ range: peakRange,
color: resolveStudioColor(
WHITE_ALPHA_60,
getComputedStyle(canvasElement),
@@ -173,7 +170,7 @@ const AudioWaveformInner: React.FC<{
width: drawingWidth,
horizontalOffset,
});
- }, [height, portionPeaks, visibleVolume, visualizationWidth]);
+ }, [height, peakRange, peaks, visibleVolume, visualizationWidth]);
useLayoutEffect(() => {
if (!shouldRenderVolumeOverlay || !peaks) {
diff --git a/packages/timeline-utils/src/audio-waveform-worker.ts b/packages/timeline-utils/src/audio-waveform-worker.ts
index f890bc5f1a..026fc563d8 100644
--- a/packages/timeline-utils/src/audio-waveform-worker.ts
+++ b/packages/timeline-utils/src/audio-waveform-worker.ts
@@ -3,25 +3,18 @@
import type {
AudioWaveformWorkerIncomingMessage,
AudioWaveformWorkerOutgoingMessage,
+ AudioWaveformWorkerPeaksMessage,
} from './audio-waveform/audio-waveform-worker-types';
import {loadWaveformPeaks} from './audio-waveform/load-waveform-peaks';
declare const self: DedicatedWorkerGlobalScope;
-const postPeaks = (
- requestId: number,
- peaks: Float32Array,
- final: boolean,
- averageVolume: number | null,
-) => {
+const postPeaks = (data: Omit<AudioWaveformWorkerPeaksMessage, 'type'>) => {
// Structured cloning copies the array, so the decoder can keep
// mutating its buffer while the main thread reads the snapshot.
const payload: AudioWaveformWorkerOutgoingMessage = {
type: 'peaks',
- averageVolume,
- requestId,
- peaks,
- final,
+ ...data,
};
self.postMessage(payload);
};
@@ -46,14 +39,20 @@ self.addEventListener(
loadWaveformPeaks(message.src, controller.signal, {
waveformSampleRate: message.waveformSampleRate,
- onProgress: ({peaks, final}) => {
+ onProgress: ({peaks, maxima, final}) => {
if (!final) {
- postPeaks(message.requestId, peaks, false, null);
+ postPeaks({
+ requestId: message.requestId,
+ peaks,
+ maxima,
+ final: false,
+ averageVolume: null,
+ });
}
},
})
- .then(({peaks, averageVolume}) => {
- postPeaks(message.requestId, peaks, true, averageVolume);
+ .then((result) => {
+ postPeaks({requestId: message.requestId, ...result, final: true});
})
.catch((error) => {
postError(message.requestId, error);
diff --git a/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts b/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts
index f0462ce699..dddced67d7 100644
--- a/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts
+++ b/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts
@@ -1,3 +1,5 @@
+import type {WaveformMaxima} from './waveform-maxima';
+
export type AudioWaveformWorkerLoadMessage = {
readonly type: 'load';
readonly requestId: number;
@@ -11,6 +13,7 @@ export type AudioWaveformWorkerPeaksMessage = {
readonly type: 'peaks';
readonly requestId: number;
readonly peaks: Float32Array;
+ readonly maxima: WaveformMaxima;
readonly final: boolean;
readonly averageVolume: number | null;
};
diff --git a/packages/timeline-utils/src/audio-waveform/draw-peaks.ts b/packages/timeline-utils/src/audio-waveform/draw-peaks.ts
index 15ad3705f7..f153f3e58b 100644
--- a/packages/timeline-utils/src/audio-waveform/draw-peaks.ts
+++ b/packages/timeline-utils/src/audio-waveform/draw-peaks.ts
@@ -1,4 +1,5 @@
import {parseColor} from './parse-color';
+import {getWaveformMaxima, getWaveformMaximum} from './waveform-maxima';
const CLIPPING_COLOR = '#FF7F50';
const WAVEFORM_HEIGHT_SCALE = 0.8;
@@ -6,6 +7,15 @@ const MIN_VISIBLE_DECIBELS = -37;
export type WaveformVolume = number | readonly number[];
+export type WaveformDrawRange = {
+ // All positions and durations use the original source's peak index units.
+ readonly sourceStart: number;
+ readonly sourceDuration: number;
+ readonly displayStart: number;
+ readonly displayDuration: number;
+ readonly loop: boolean;
+};
+
const getVolumeAtBar = ({
barIndex,
numBars,
@@ -37,12 +47,14 @@ export const drawBars = ({
canvas,
color,
peaks,
+ range,
volume,
width,
horizontalOffset,
}: {
readonly canvas: HTMLCanvasElement | OffscreenCanvas;
readonly peaks: Float32Array;
+ readonly range: WaveformDrawRange;
readonly color: string;
readonly volume: WaveformVolume;
readonly width: number;
@@ -74,6 +86,17 @@ export const drawBars = ({
const {data} = imageData;
const numBars = width;
const fullScaleHalfBar = (height * WAVEFORM_HEIGHT_SCALE) / 2;
+ const maxima = getWaveformMaxima(peaks, null);
+ const peaksPerPixel = range.displayDuration / width;
+ const loopMaximum =
+ range.loop && range.sourceDuration > 0
+ ? getWaveformMaximum({
+ peaks,
+ maxima,
+ from: range.sourceStart,
+ to: range.sourceStart + range.sourceDuration,
+ })
+ : 0;
for (let x = 0; x < w; x++) {
// The canvas may start before its fractionally positioned container so its
@@ -86,8 +109,53 @@ export const drawBars = ({
),
);
- const peakIndex = Math.floor((barIndex / numBars) * peaks.length);
- const peak = peaks[peakIndex] || 0;
+ // Sample the exact source-time interval covered by this bitmap column.
+ // Rounding the visible source slice and stretching it across the canvas
+ // would change this mapping when a clip is trimmed or virtualized.
+ const from =
+ range.displayStart +
+ Math.max(0, Math.min(width, x - horizontalOffset)) * peaksPerPixel;
+ const to =
+ range.displayStart +
+ Math.max(0, Math.min(width, x + 1 - horizontalOffset)) * peaksPerPixel;
+ let peak = 0;
+ if (to > from && range.loop && range.sourceDuration > 0) {
+ const duration = to - from;
+ if (duration >= range.sourceDuration) {
+ // A zoomed-out column can cover many repeats. Query the loop once,
+ // without allocating or visiting every repeated source segment.
+ peak = loopMaximum;
+ } else {
+ const loopFrom =
+ ((from % range.sourceDuration) + range.sourceDuration) %
+ range.sourceDuration;
+ const loopTo = loopFrom + duration;
+ peak = getWaveformMaximum({
+ peaks,
+ maxima,
+ from: range.sourceStart + loopFrom,
+ to: range.sourceStart + Math.min(range.sourceDuration, loopTo),
+ });
+ if (loopTo > range.sourceDuration) {
+ peak = Math.max(
+ peak,
+ getWaveformMaximum({
+ peaks,
+ maxima,
+ from: range.sourceStart,
+ to: range.sourceStart + loopTo - range.sourceDuration,
+ }),
+ );
+ }
+ }
+ } else if (!range.loop) {
+ peak = getWaveformMaximum({
+ peaks,
+ maxima,
+ from: range.sourceStart + Math.max(0, from),
+ to: range.sourceStart + Math.min(range.sourceDuration, to),
+ });
+ }
const barVolume = getVolumeAtBar({barIndex, numBars, volume});
const scaledPeak = peak * barVolume;
diff --git a/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts b/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts
index 773494fb17..7de6719016 100644
--- a/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts
+++ b/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts
@@ -3,14 +3,13 @@ import {ALL_FORMATS, AudioSampleSink, Input, UrlSource} from 'mediabunny';
import {TARGET_SAMPLE_RATE} from './constants';
import {getWaveformCacheKey} from './get-waveform-cache-key';
import {getAudioSampleStartFrameAtTimelineZero} from './trim-audio-sample-before-zero';
-import {
- createWaveformPeakProcessor,
- emitWaveformProgress,
-} from './waveform-peak-processor';
+import {getWaveformMaxima, type WaveformMaxima} from './waveform-maxima';
+import {createWaveformPeakProcessor} from './waveform-peak-processor';
const DEFAULT_PROGRESS_INTERVAL_IN_MS = 50;
export type WaveformResult = {
readonly peaks: Float32Array;
+ readonly maxima: WaveformMaxima;
readonly averageVolume: number | null;
};
@@ -21,6 +20,7 @@ export {TARGET_SAMPLE_RATE};
type Progress = {
readonly averageVolume: number | null;
readonly peaks: Float32Array;
+ readonly maxima: WaveformMaxima;
readonly completedPeaks: number;
readonly totalPeaks: number;
readonly final: boolean;
@@ -45,13 +45,13 @@ export async function loadWaveformPeaks(
const cacheKey = getWaveformCacheKey(src, waveformSampleRate);
const cached = peaksCache.get(cacheKey);
if (cached) {
- emitWaveformProgress({
+ options?.onProgress?.({
peaks: cached.peaks,
+ maxima: cached.maxima,
averageVolume: cached.averageVolume,
completedPeaks: cached.peaks.length,
totalPeaks: cached.peaks.length,
final: true,
- onProgress: options?.onProgress,
});
return cached;
}
@@ -65,7 +65,7 @@ export async function loadWaveformPeaks(
const audioTrack =
typeof src === 'string' ? await input!.getPrimaryAudioTrack() : src;
if (!audioTrack) {
- return {peaks: new Float32Array(0), averageVolume: null};
+ return {peaks: new Float32Array(0), maxima: [], averageVolume: null};
}
if (await audioTrack.isLive()) {
@@ -96,7 +96,13 @@ export async function loadWaveformPeaks(
const processor = createWaveformPeakProcessor({
totalPeaks,
samplesPerPeak,
- onProgress: options?.onProgress,
+ onProgress: (progress) => {
+ const maxima = getWaveformMaxima(
+ progress.peaks,
+ progress.completedPeaks,
+ );
+ options?.onProgress?.({...progress, maxima});
+ },
progressIntervalInMs:
options?.progressIntervalInMs ??
DEFAULT_PROGRESS_INTERVAL_IN_MS *
@@ -107,7 +113,7 @@ export async function loadWaveformPeaks(
for await (const sample of sink.samples()) {
if (signal.aborted) {
sample.close();
- return {peaks: new Float32Array(0), averageVolume: null};
+ return {peaks: new Float32Array(0), maxima: [], averageVolume: null};
}
const startFrame = getAudioSampleStartFrameAtTimelineZero(sample);
@@ -145,6 +151,7 @@ export async function loadWaveformPeaks(
processor.finalize();
const result = {
peaks: processor.peaks,
+ maxima: getWaveformMaxima(processor.peaks, null),
averageVolume: processor.averageVolume,
};
peaksCache.set(cacheKey, result);
diff --git a/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts b/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts
index 0d79b81f15..6d3f1b549c 100644
--- a/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts
+++ b/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts
@@ -7,6 +7,7 @@ import {TARGET_SAMPLE_RATE} from './constants';
import {getWaveformCacheKey} from './get-waveform-cache-key';
import {type WaveformResult, loadWaveformPeaks} from './load-waveform-peaks';
import {makeAudioWaveformWorker} from './make-audio-waveform-worker';
+import {registerWaveformMaxima} from './waveform-maxima';
type WaveformPeaksListener = {
readonly onPeaks: (
@@ -39,12 +40,13 @@ let nextRequestId = 0;
const emitPeaks = (
load: InFlightLoad,
- peaks: Float32Array,
+ result: WaveformResult,
final: boolean,
- averageVolume: number | null,
) => {
+ const {peaks, maxima, averageVolume} = result;
+ registerWaveformMaxima(peaks, maxima);
if (final) {
- peaksCache.set(load.cacheKey, {peaks, averageVolume});
+ peaksCache.set(load.cacheKey, {peaks, maxima, averageVolume});
load.latestPeaks = null;
inFlightByCacheKey.delete(load.cacheKey);
if (load.requestId !== null) {
@@ -79,18 +81,26 @@ const startMainThreadLoad = (load: InFlightLoad) => {
loadWaveformPeaks(load.src, controller.signal, {
waveformSampleRate: load.waveformSampleRate,
- onProgress: ({peaks, final}) => {
+ onProgress: ({peaks, maxima, final}) => {
if (final) {
return;
}
// The processor mutates the same array in place; snapshot it so
// React state updates see a new reference.
- emitPeaks(load, peaks.slice(), false, null);
+ emitPeaks(
+ load,
+ {
+ peaks: peaks.slice(),
+ maxima: maxima.map((level) => level.slice()),
+ averageVolume: null,
+ },
+ false,
+ );
},
})
- .then(({peaks, averageVolume}) => {
- emitPeaks(load, peaks, true, averageVolume);
+ .then((result) => {
+ emitPeaks(load, result, true);
})
.catch((error) => {
emitError(
@@ -144,7 +154,7 @@ const getOrCreateWorker = (): Worker | null => {
return;
}
- emitPeaks(load, message.peaks, message.final, message.averageVolume);
+ emitPeaks(load, message, message.final);
},
);
worker.addEventListener('error', (event) => {
diff --git a/packages/timeline-utils/src/audio-waveform/waveform-maxima.ts b/packages/timeline-utils/src/audio-waveform/waveform-maxima.ts
new file mode 100644
index 0000000000..4807dc5794
--- /dev/null
+++ b/packages/timeline-utils/src/audio-waveform/waveform-maxima.ts
@@ -0,0 +1,120 @@
+export type WaveformMaxima = readonly Float32Array[];
+
+const maximaCache = new WeakMap<
+ Float32Array,
+ {maxima: WaveformMaxima; completedPeaks: number}
+>();
+
+export const registerWaveformMaxima = (
+ peaks: Float32Array,
+ maxima: WaveformMaxima,
+) => {
+ maximaCache.set(peaks, {maxima, completedPeaks: peaks.length});
+};
+
+export const getWaveformMaxima = (
+ peaks: Float32Array,
+ completedPeaks: number | null,
+): WaveformMaxima => {
+ let cached = maximaCache.get(peaks);
+ if (!cached) {
+ const maxima: Float32Array[] = [];
+ let {length} = peaks;
+ while (length > 1) {
+ length = Math.ceil(length / 2);
+ maxima.push(new Float32Array(length));
+ }
+
+ cached = {maxima, completedPeaks: 0};
+ maximaCache.set(peaks, cached);
+ }
+
+ const completed = Math.min(
+ peaks.length,
+ Math.max(0, completedPeaks ?? peaks.length),
+ );
+ if (completed <= cached.completedPeaks) {
+ return cached.maxima;
+ }
+
+ // Only update parents of newly decoded peaks. Include the last partial pair
+ // so progress snapshots can query the same tree as the completed waveform.
+ let start = cached.completedPeaks;
+ let end = completed;
+ let previous = peaks;
+ for (const level of cached.maxima) {
+ start = Math.floor(start / 2);
+ end = Math.ceil(end / 2);
+ for (let i = start; i < end; i++) {
+ level[i] = Math.max(previous[i * 2] ?? 0, previous[i * 2 + 1] ?? 0);
+ }
+
+ previous = level;
+ }
+
+ cached.completedPeaks = completed;
+ return cached.maxima;
+};
+
+export const getWaveformMaximum = ({
+ peaks,
+ maxima,
+ from,
+ to,
+}: {
+ readonly peaks: Float32Array;
+ readonly maxima: WaveformMaxima;
+ readonly from: number;
+ readonly to: number;
+}) => {
+ if (to <= from) {
+ return 0;
+ }
+
+ // Ignore floating-point noise at exact source-peak boundaries, including
+ // fractional loop periods, so a column never picks up the neighboring peak.
+ let start = Math.max(
+ 0,
+ Math.min(
+ peaks.length,
+ Math.floor(
+ from +
+ (Number.isFinite(from)
+ ? Number.EPSILON * Math.max(1, Math.abs(from)) * 4
+ : 0),
+ ),
+ ),
+ );
+ let end = Math.max(
+ 0,
+ Math.min(
+ peaks.length,
+ Math.ceil(
+ to -
+ (Number.isFinite(to)
+ ? Number.EPSILON * Math.max(1, Math.abs(to)) * 4
+ : 0),
+ ),
+ ),
+ );
+ let maximum = 0;
+ let data = peaks;
+ let level = 0;
+ while (start < end) {
+ if (start % 2 === 1) {
+ maximum = Math.max(maximum, data[start]);
+ start++;
+ }
+
+ if (end % 2 === 1) {
+ end--;
+ maximum = Math.max(maximum, data[end]);
+ }
+
+ start = Math.floor(start / 2);
+ end = Math.floor(end / 2);
+ data = maxima[level++];
+ }
+
+ return maximum;
+};
diff --git a/packages/timeline-utils/src/index.ts b/packages/timeline-utils/src/index.ts
index 3f2451ac7d..e6fb4f6474 100644
--- a/packages/timeline-utils/src/index.ts
+++ b/packages/timeline-utils/src/index.ts
@@ -3,7 +3,11 @@ export type {
AudioWaveformWorkerOutgoingMessage,
} from './audio-waveform/audio-waveform-worker-types';
export {TARGET_SAMPLE_RATE} from './audio-waveform/constants';
-export {drawBars, type WaveformVolume} from './audio-waveform/draw-peaks';
+export {
+ drawBars,
+ type WaveformDrawRange,
+ type WaveformVolume,
+} from './audio-waveform/draw-peaks';
export {getVisibleWaveformVolume} from './audio-waveform/get-visible-waveform-volume';
export {loadWaveformPeaks} from './audio-waveform/load-waveform-peaks';
export {sliceVisibleWaveformPeaks} from './audio-waveform/slice-visible-waveform-peaks';