🎥 Make videos programmatically with React
`@remotion/studio`: Fix waveform peak sampling when zooming or trimming
Jonny Burger2ed96e3f1ea3
diff --git a/packages/studio/src/components/AudioWaveform.tsx b/packages/studio/src/components/AudioWaveform.tsxindex 3bed8237f6..1e7d336166 100644--- a/packages/studio/src/components/AudioWaveform.tsx+++ b/packages/studio/src/components/AudioWaveform.tsx@@ -1,8 +1,8 @@import {drawBars,getVisibleWaveformVolume,- sliceVisibleWaveformPeaks,subscribeToWaveformPeaks,+ type WaveformDrawRange,type WaveformVolume,} from '@remotion/timeline-utils';import React, {useLayoutEffect, useMemo, useRef, useState} from 'react';@@ -115,29 +115,25 @@ const AudioWaveformInner: React.FC<{});}, [src, waveformSampleRate]);- const portionPeaks = useMemo(() => {- if (!peaks) {- return null;- }-- return sliceVisibleWaveformPeaks({- displayDurationInFrames,- displayOffsetInFrames: displayOffsetInFrames + loopDisplayOffsetInFrames,- durationInFrames,- fps: vidConf.fps,- loopDisplay,- peaks,- playbackRate,- startFrom,- waveformSampleRate,- });+ const peakRange = useMemo((): WaveformDrawRange => {+ const peaksPerFrame = (waveformSampleRate * playbackRate) / vidConf.fps;+ const loop = loopDisplay !== undefined && loopDisplay.durationInFrames > 0;+ return {+ sourceStart: (startFrom / vidConf.fps) * waveformSampleRate,+ sourceDuration:+ (loop ? loopDisplay.durationInFrames : durationInFrames) *+ peaksPerFrame,+ displayStart:+ (displayOffsetInFrames + loopDisplayOffsetInFrames) * peaksPerFrame,+ displayDuration: displayDurationInFrames * peaksPerFrame,+ loop,+ };}, [displayDurationInFrames,displayOffsetInFrames,durationInFrames,loopDisplay,loopDisplayOffsetInFrames,- peaks,playbackRate,startFrom,vidConf.fps,@@ -164,7 +160,8 @@ const AudioWaveformInner: React.FC<{drawBars({canvas: canvasElement,- peaks: portionPeaks ?? EMPTY_PEAKS,+ peaks: peaks ?? EMPTY_PEAKS,+ range: peakRange,color: resolveStudioColor(WHITE_ALPHA_60,getComputedStyle(canvasElement),@@ -173,7 +170,7 @@ const AudioWaveformInner: React.FC<{width: drawingWidth,horizontalOffset,});- }, [height, portionPeaks, visibleVolume, visualizationWidth]);+ }, [height, peakRange, peaks, visibleVolume, visualizationWidth]);useLayoutEffect(() => {if (!shouldRenderVolumeOverlay || !peaks) {diff --git a/packages/timeline-utils/src/audio-waveform-worker.ts b/packages/timeline-utils/src/audio-waveform-worker.tsindex f890bc5f1a..026fc563d8 100644--- a/packages/timeline-utils/src/audio-waveform-worker.ts+++ b/packages/timeline-utils/src/audio-waveform-worker.ts@@ -3,25 +3,18 @@import type {AudioWaveformWorkerIncomingMessage,AudioWaveformWorkerOutgoingMessage,+ AudioWaveformWorkerPeaksMessage,} from './audio-waveform/audio-waveform-worker-types';import {loadWaveformPeaks} from './audio-waveform/load-waveform-peaks';declare const self: DedicatedWorkerGlobalScope;-const postPeaks = (- requestId: number,- peaks: Float32Array,- final: boolean,- averageVolume: number | null,-) => {+const postPeaks = (data: Omit<AudioWaveformWorkerPeaksMessage, 'type'>) => {// Structured cloning copies the array, so the decoder can keep// mutating its buffer while the main thread reads the snapshot.const payload: AudioWaveformWorkerOutgoingMessage = {type: 'peaks',- averageVolume,- requestId,- peaks,- final,+ ...data,};self.postMessage(payload);};@@ -46,14 +39,20 @@ self.addEventListener(loadWaveformPeaks(message.src, controller.signal, {waveformSampleRate: message.waveformSampleRate,- onProgress: ({peaks, final}) => {+ onProgress: ({peaks, maxima, final}) => {if (!final) {- postPeaks(message.requestId, peaks, false, null);+ postPeaks({+ requestId: message.requestId,+ peaks,+ maxima,+ final: false,+ averageVolume: null,+ });}},})- .then(({peaks, averageVolume}) => {- postPeaks(message.requestId, peaks, true, averageVolume);+ .then((result) => {+ postPeaks({requestId: message.requestId, ...result, final: true});}).catch((error) => {postError(message.requestId, error);diff --git a/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts b/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.tsindex f0462ce699..dddced67d7 100644--- a/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts+++ b/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts@@ -1,3 +1,5 @@+import type {WaveformMaxima} from './waveform-maxima';+export type AudioWaveformWorkerLoadMessage = {readonly type: 'load';readonly requestId: number;@@ -11,6 +13,7 @@ export type AudioWaveformWorkerPeaksMessage = {readonly type: 'peaks';readonly requestId: number;readonly peaks: Float32Array;+ readonly maxima: WaveformMaxima;readonly final: boolean;readonly averageVolume: number | null;};diff --git a/packages/timeline-utils/src/audio-waveform/draw-peaks.ts b/packages/timeline-utils/src/audio-waveform/draw-peaks.tsindex 15ad3705f7..f153f3e58b 100644--- a/packages/timeline-utils/src/audio-waveform/draw-peaks.ts+++ b/packages/timeline-utils/src/audio-waveform/draw-peaks.ts@@ -1,4 +1,5 @@import {parseColor} from './parse-color';+import {getWaveformMaxima, getWaveformMaximum} from './waveform-maxima';const CLIPPING_COLOR = '#FF7F50';const WAVEFORM_HEIGHT_SCALE = 0.8;@@ -6,6 +7,15 @@ const MIN_VISIBLE_DECIBELS = -37;export type WaveformVolume = number | readonly number[];+export type WaveformDrawRange = {+ // All positions and durations use the original source's peak index units.+ readonly sourceStart: number;+ readonly sourceDuration: number;+ readonly displayStart: number;+ readonly displayDuration: number;+ readonly loop: boolean;+};+const getVolumeAtBar = ({barIndex,numBars,@@ -37,12 +47,14 @@ export const drawBars = ({canvas,color,peaks,+ range,volume,width,horizontalOffset,}: {readonly canvas: HTMLCanvasElement | OffscreenCanvas;readonly peaks: Float32Array;+ readonly range: WaveformDrawRange;readonly color: string;readonly volume: WaveformVolume;readonly width: number;@@ -74,6 +86,17 @@ export const drawBars = ({const {data} = imageData;const numBars = width;const fullScaleHalfBar = (height * WAVEFORM_HEIGHT_SCALE) / 2;+ const maxima = getWaveformMaxima(peaks, null);+ const peaksPerPixel = range.displayDuration / width;+ const loopMaximum =+ range.loop && range.sourceDuration > 0+ ? getWaveformMaximum({+ peaks,+ maxima,+ from: range.sourceStart,+ to: range.sourceStart + range.sourceDuration,+ })+ : 0;for (let x = 0; x < w; x++) {// The canvas may start before its fractionally positioned container so its@@ -86,8 +109,53 @@ export const drawBars = ({),);- const peakIndex = Math.floor((barIndex / numBars) * peaks.length);- const peak = peaks[peakIndex] || 0;+ // Sample the exact source-time interval covered by this bitmap column.+ // Rounding the visible source slice and stretching it across the canvas+ // would change this mapping when a clip is trimmed or virtualized.+ const from =+ range.displayStart ++ Math.max(0, Math.min(width, x - horizontalOffset)) * peaksPerPixel;+ const to =+ range.displayStart ++ Math.max(0, Math.min(width, x + 1 - horizontalOffset)) * peaksPerPixel;+ let peak = 0;+ if (to > from && range.loop && range.sourceDuration > 0) {+ const duration = to - from;+ if (duration >= range.sourceDuration) {+ // A zoomed-out column can cover many repeats. Query the loop once,+ // without allocating or visiting every repeated source segment.+ peak = loopMaximum;+ } else {+ const loopFrom =+ ((from % range.sourceDuration) + range.sourceDuration) %+ range.sourceDuration;+ const loopTo = loopFrom + duration;+ peak = getWaveformMaximum({+ peaks,+ maxima,+ from: range.sourceStart + loopFrom,+ to: range.sourceStart + Math.min(range.sourceDuration, loopTo),+ });+ if (loopTo > range.sourceDuration) {+ peak = Math.max(+ peak,+ getWaveformMaximum({+ peaks,+ maxima,+ from: range.sourceStart,+ to: range.sourceStart + loopTo - range.sourceDuration,+ }),+ );+ }+ }+ } else if (!range.loop) {+ peak = getWaveformMaximum({+ peaks,+ maxima,+ from: range.sourceStart + Math.max(0, from),+ to: range.sourceStart + Math.min(range.sourceDuration, to),+ });+ }const barVolume = getVolumeAtBar({barIndex, numBars, volume});const scaledPeak = peak * barVolume;diff --git a/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts b/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.tsindex 773494fb17..7de6719016 100644--- a/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts+++ b/packages/timeline-utils/src/audio-waveform/load-waveform-peaks.ts@@ -3,14 +3,13 @@ import {ALL_FORMATS, AudioSampleSink, Input, UrlSource} from 'mediabunny';import {TARGET_SAMPLE_RATE} from './constants';import {getWaveformCacheKey} from './get-waveform-cache-key';import {getAudioSampleStartFrameAtTimelineZero} from './trim-audio-sample-before-zero';-import {- createWaveformPeakProcessor,- emitWaveformProgress,-} from './waveform-peak-processor';+import {getWaveformMaxima, type WaveformMaxima} from './waveform-maxima';+import {createWaveformPeakProcessor} from './waveform-peak-processor';const DEFAULT_PROGRESS_INTERVAL_IN_MS = 50;export type WaveformResult = {readonly peaks: Float32Array;+ readonly maxima: WaveformMaxima;readonly averageVolume: number | null;};@@ -21,6 +20,7 @@ export {TARGET_SAMPLE_RATE};type Progress = {readonly averageVolume: number | null;readonly peaks: Float32Array;+ readonly maxima: WaveformMaxima;readonly completedPeaks: number;readonly totalPeaks: number;readonly final: boolean;@@ -45,13 +45,13 @@ export async function loadWaveformPeaks(const cacheKey = getWaveformCacheKey(src, waveformSampleRate);const cached = peaksCache.get(cacheKey);if (cached) {- emitWaveformProgress({+ options?.onProgress?.({peaks: cached.peaks,+ maxima: cached.maxima,averageVolume: cached.averageVolume,completedPeaks: cached.peaks.length,totalPeaks: cached.peaks.length,final: true,- onProgress: options?.onProgress,});return cached;}@@ -65,7 +65,7 @@ export async function loadWaveformPeaks(const audioTrack =typeof src === 'string' ? await input!.getPrimaryAudioTrack() : src;if (!audioTrack) {- return {peaks: new Float32Array(0), averageVolume: null};+ return {peaks: new Float32Array(0), maxima: [], averageVolume: null};}if (await audioTrack.isLive()) {@@ -96,7 +96,13 @@ export async function loadWaveformPeaks(const processor = createWaveformPeakProcessor({totalPeaks,samplesPerPeak,- onProgress: options?.onProgress,+ onProgress: (progress) => {+ const maxima = getWaveformMaxima(+ progress.peaks,+ progress.completedPeaks,+ );+ options?.onProgress?.({...progress, maxima});+ },progressIntervalInMs:options?.progressIntervalInMs ??DEFAULT_PROGRESS_INTERVAL_IN_MS *@@ -107,7 +113,7 @@ export async function loadWaveformPeaks(for await (const sample of sink.samples()) {if (signal.aborted) {sample.close();- return {peaks: new Float32Array(0), averageVolume: null};+ return {peaks: new Float32Array(0), maxima: [], averageVolume: null};}const startFrame = getAudioSampleStartFrameAtTimelineZero(sample);@@ -145,6 +151,7 @@ export async function loadWaveformPeaks(processor.finalize();const result = {peaks: processor.peaks,+ maxima: getWaveformMaxima(processor.peaks, null),averageVolume: processor.averageVolume,};peaksCache.set(cacheKey, result);diff --git a/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts b/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.tsindex 0d79b81f15..6d3f1b549c 100644--- a/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts+++ b/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts@@ -7,6 +7,7 @@ import {TARGET_SAMPLE_RATE} from './constants';import {getWaveformCacheKey} from './get-waveform-cache-key';import {type WaveformResult, loadWaveformPeaks} from './load-waveform-peaks';import {makeAudioWaveformWorker} from './make-audio-waveform-worker';+import {registerWaveformMaxima} from './waveform-maxima';type WaveformPeaksListener = {readonly onPeaks: (@@ -39,12 +40,13 @@ let nextRequestId = 0;const emitPeaks = (load: InFlightLoad,- peaks: Float32Array,+ result: WaveformResult,final: boolean,- averageVolume: number | null,) => {+ const {peaks, maxima, averageVolume} = result;+ registerWaveformMaxima(peaks, maxima);if (final) {- peaksCache.set(load.cacheKey, {peaks, averageVolume});+ peaksCache.set(load.cacheKey, {peaks, maxima, averageVolume});load.latestPeaks = null;inFlightByCacheKey.delete(load.cacheKey);if (load.requestId !== null) {@@ -79,18 +81,26 @@ const startMainThreadLoad = (load: InFlightLoad) => {loadWaveformPeaks(load.src, controller.signal, {waveformSampleRate: load.waveformSampleRate,- onProgress: ({peaks, final}) => {+ onProgress: ({peaks, maxima, final}) => {if (final) {return;}// The processor mutates the same array in place; snapshot it so// React state updates see a new reference.- emitPeaks(load, peaks.slice(), false, null);+ emitPeaks(+ load,+ {+ peaks: peaks.slice(),+ maxima: maxima.map((level) => level.slice()),+ averageVolume: null,+ },+ false,+ );},})- .then(({peaks, averageVolume}) => {- emitPeaks(load, peaks, true, averageVolume);+ .then((result) => {+ emitPeaks(load, result, true);}).catch((error) => {emitError(@@ -144,7 +154,7 @@ const getOrCreateWorker = (): Worker | null => {return;}- emitPeaks(load, message.peaks, message.final, message.averageVolume);+ emitPeaks(load, message, message.final);},);worker.addEventListener('error', (event) => {diff --git a/packages/timeline-utils/src/audio-waveform/waveform-maxima.ts b/packages/timeline-utils/src/audio-waveform/waveform-maxima.tsnew file mode 100644index 0000000000..4807dc5794--- /dev/null+++ b/packages/timeline-utils/src/audio-waveform/waveform-maxima.ts@@ -0,0 +1,120 @@+export type WaveformMaxima = readonly Float32Array[];++const maximaCache = new WeakMap<+ Float32Array,+ {maxima: WaveformMaxima; completedPeaks: number}+>();++export const registerWaveformMaxima = (+ peaks: Float32Array,+ maxima: WaveformMaxima,+) => {+ maximaCache.set(peaks, {maxima, completedPeaks: peaks.length});+};++export const getWaveformMaxima = (+ peaks: Float32Array,+ completedPeaks: number | null,+): WaveformMaxima => {+ let cached = maximaCache.get(peaks);+ if (!cached) {+ const maxima: Float32Array[] = [];+ let {length} = peaks;+ while (length > 1) {+ length = Math.ceil(length / 2);+ maxima.push(new Float32Array(length));+ }++ cached = {maxima, completedPeaks: 0};+ maximaCache.set(peaks, cached);+ }++ const completed = Math.min(+ peaks.length,+ Math.max(0, completedPeaks ?? peaks.length),+ );+ if (completed <= cached.completedPeaks) {+ return cached.maxima;+ }++ // Only update parents of newly decoded peaks. Include the last partial pair+ // so progress snapshots can query the same tree as the completed waveform.+ let start = cached.completedPeaks;+ let end = completed;+ let previous = peaks;+ for (const level of cached.maxima) {+ start = Math.floor(start / 2);+ end = Math.ceil(end / 2);+ for (let i = start; i < end; i++) {+ level[i] = Math.max(previous[i * 2] ?? 0, previous[i * 2 + 1] ?? 0);+ }++ previous = level;+ }++ cached.completedPeaks = completed;+ return cached.maxima;+};++export const getWaveformMaximum = ({+ peaks,+ maxima,+ from,+ to,+}: {+ readonly peaks: Float32Array;+ readonly maxima: WaveformMaxima;+ readonly from: number;+ readonly to: number;+}) => {+ if (to <= from) {+ return 0;+ }++ // Ignore floating-point noise at exact source-peak boundaries, including+ // fractional loop periods, so a column never picks up the neighboring peak.+ let start = Math.max(+ 0,+ Math.min(+ peaks.length,+ Math.floor(+ from ++ (Number.isFinite(from)+ ? Number.EPSILON * Math.max(1, Math.abs(from)) * 4+ : 0),+ ),+ ),+ );+ let end = Math.max(+ 0,+ Math.min(+ peaks.length,+ Math.ceil(+ to -+ (Number.isFinite(to)+ ? Number.EPSILON * Math.max(1, Math.abs(to)) * 4+ : 0),+ ),+ ),+ );+ let maximum = 0;+ let data = peaks;+ let level = 0;+ while (start < end) {+ if (start % 2 === 1) {+ maximum = Math.max(maximum, data[start]);+ start++;+ }++ if (end % 2 === 1) {+ end--;+ maximum = Math.max(maximum, data[end]);+ }++ start = Math.floor(start / 2);+ end = Math.floor(end / 2);+ data = maxima[level++];+ }++ return maximum;+};diff --git a/packages/timeline-utils/src/index.ts b/packages/timeline-utils/src/index.tsindex 3f2451ac7d..e6fb4f6474 100644--- a/packages/timeline-utils/src/index.ts+++ b/packages/timeline-utils/src/index.ts@@ -3,7 +3,11 @@ export type {AudioWaveformWorkerOutgoingMessage,} from './audio-waveform/audio-waveform-worker-types';export {TARGET_SAMPLE_RATE} from './audio-waveform/constants';-export {drawBars, type WaveformVolume} from './audio-waveform/draw-peaks';+export {+ drawBars,+ type WaveformDrawRange,+ type WaveformVolume,+} from './audio-waveform/draw-peaks';export {getVisibleWaveformVolume} from './audio-waveform/get-visible-waveform-volume';export {loadWaveformPeaks} from './audio-waveform/load-waveform-peaks';export {sliceVisibleWaveformPeaks} from './audio-waveform/slice-visible-waveform-peaks';