Skip to content

🎥 Make videos programmatically with React

`@remotion/studio`: Fix timeline waveform flicker when zooming (#10630)

Jonny Burger411a0de8a351
diff --git a/packages/studio/src/components/AudioWaveform.tsx b/packages/studio/src/components/AudioWaveform.tsx
index 5ef6d2ccb1..44ed89c632 100644
--- a/packages/studio/src/components/AudioWaveform.tsx
+++ b/packages/studio/src/components/AudioWaveform.tsx
@@ -1,14 +1,11 @@
import {
drawBars,
getVisibleWaveformVolume,
- loadWaveformPeaks,
- makeAudioWaveformWorker,
sliceVisibleWaveformPeaks,
- type AudioWaveformWorkerOutgoingMessage,
- type AudioWaveformWorkerRenderMessage,
+ subscribeToWaveformPeaks,
type WaveformVolume,
} from '@remotion/timeline-utils';
-import React, {useEffect, useMemo, useRef, useState} from 'react';
+import React, {useLayoutEffect, useMemo, useRef, useState} from 'react';
import type {LoopDisplay} from 'remotion';
import {Internals} from 'remotion';
import {WHITE_ALPHA_70, WHITE_ALPHA_60} from '../helpers/colors';
@@ -16,24 +13,6 @@ import {TIMELINE_BORDER} from '../helpers/timeline-layout';
const EMPTY_PEAKS = new Float32Array(0);
-// Recreate the canvas after Fast Refresh because an already transferred canvas
-// cannot be handed to OffscreenCanvas again.
-const canRetryCanvasTransfer = (err: unknown) => {
- return err instanceof DOMException && err.name === 'InvalidStateError';
-};
-
-const canUseAudioWaveformWorker = () => {
- if (
- typeof Worker === 'undefined' ||
- typeof OffscreenCanvas === 'undefined' ||
- typeof HTMLCanvasElement === 'undefined'
- ) {
- return false;
- }
-
- return 'transferControlToOffscreen' in HTMLCanvasElement.prototype;
-};
-
const getContainerStyle = (height: number): React.CSSProperties => {
return {
display: 'flex',
@@ -45,12 +24,6 @@ const getContainerStyle = (height: number): React.CSSProperties => {
};
};
-const getWaveformErrorMessage = () => {
- return new Error(
- 'No waveform available. The audio could not be decoded or may not support CORS.',
- );
-};
-
const waveformCanvasStyle: React.CSSProperties = {
pointerEvents: 'none',
width: '100%',
@@ -100,8 +73,6 @@ const AudioWaveformInner: React.FC<{
}) => {
const [peaks, setPeaks] = useState<Float32Array | null>(null);
const [error, setError] = useState<Error | null>(null);
- const [waveformCanvasKey, setWaveformCanvasKey] = useState(0);
- const canUseWorkerPath = useMemo(() => canUseAudioWaveformWorker(), []);
const vidConf = Internals.useUnsafeVideoConfig();
if (vidConf === null) {
throw new Error('Expected video config');
@@ -109,9 +80,6 @@ const AudioWaveformInner: React.FC<{
const waveformCanvas = useRef<HTMLCanvasElement>(null);
const volumeCanvas = useRef<HTMLCanvasElement>(null);
- const waveformWorker = useRef<Worker | null>(null);
- const hasTransferredCanvas = useRef(false);
- const latestRequestId = useRef(0);
const shouldRenderVolumeOverlay =
doesVolumeChange && typeof volume === 'string';
const parsedVolume = useMemo(() => parseVolume(volume), [volume]);
@@ -134,99 +102,21 @@ const AudioWaveformInner: React.FC<{
parsedVolume,
]);
- useEffect(() => {
- if (canUseWorkerPath) {
- return;
- }
-
- const controller = new AbortController();
-
+ // Layout effect so that a cache hit sets the peaks synchronously and the
+ // waveform is painted on the very first frame after mounting.
+ useLayoutEffect(() => {
setPeaks(null);
setError(null);
- loadWaveformPeaks(src, controller.signal)
- .then((p) => {
- if (!controller.signal.aborted) {
- setPeaks(p);
- }
- })
- .catch((err) => {
- if (!controller.signal.aborted) {
- setError(err);
- }
- });
-
- return () => controller.abort();
- }, [canUseWorkerPath, src]);
-
- useEffect(() => {
- if (!canUseWorkerPath) {
- return;
- }
-
- const canvasElement = waveformCanvas.current;
- if (!canvasElement || hasTransferredCanvas.current) {
- return;
- }
-
- const worker = makeAudioWaveformWorker();
- let workerFailed = false;
- waveformWorker.current = worker;
- worker.addEventListener(
- 'message',
- (event: MessageEvent<AudioWaveformWorkerOutgoingMessage>) => {
- if (event.data.type === 'error') {
- if (event.data.requestId !== latestRequestId.current) {
- return;
- }
-
- setError(new Error(event.data.message));
- }
- },
- );
- worker.addEventListener('error', (event) => {
- event.preventDefault();
- workerFailed = true;
-
- if (worker !== waveformWorker.current) {
- return;
- }
- worker.terminate();
- waveformWorker.current = null;
- hasTransferredCanvas.current = false;
- setError(getWaveformErrorMessage());
+ return subscribeToWaveformPeaks({
+ src,
+ onPeaks: (p) => setPeaks(p),
+ onError: (err) => setError(err),
});
-
- let offscreen: OffscreenCanvas;
- try {
- offscreen = canvasElement.transferControlToOffscreen();
- } catch (err) {
- worker.terminate();
- waveformWorker.current = null;
- if (canRetryCanvasTransfer(err)) {
- setWaveformCanvasKey((key) => key + 1);
- return;
- }
-
- throw err;
- }
-
- hasTransferredCanvas.current = true;
- worker.postMessage({type: 'init', canvas: offscreen}, [offscreen]);
-
- return () => {
- if (!workerFailed) {
- worker.postMessage({type: 'dispose'});
- }
-
- worker.terminate();
- waveformWorker.current = null;
- hasTransferredCanvas.current = false;
- };
- }, [canUseWorkerPath, waveformCanvasKey]);
+ }, [src]);
const portionPeaks = useMemo(() => {
- if (canUseWorkerPath || !peaks) {
+ if (!peaks) {
return null;
}
@@ -241,7 +131,6 @@ const AudioWaveformInner: React.FC<{
startFrom,
});
}, [
- canUseWorkerPath,
displayDurationInFrames,
displayOffsetInFrames,
durationInFrames,
@@ -252,7 +141,10 @@ const AudioWaveformInner: React.FC<{
vidConf.fps,
]);
- useEffect(() => {
+ // Drawing must happen in a layout effect: when the timeline zooms, the
+ // canvas CSS box resizes in the same commit, and painting the bitmap
+ // before the browser paints avoids showing a stretched stale waveform.
+ useLayoutEffect(() => {
const {current: canvasElement} = waveformCanvas;
if (!canvasElement) {
return;
@@ -262,33 +154,6 @@ const AudioWaveformInner: React.FC<{
const h = Math.ceil(height * pixelRatio);
const w = Math.ceil(visualizationWidth * pixelRatio);
- if (canUseWorkerPath) {
- const worker = waveformWorker.current;
- if (!worker || !hasTransferredCanvas.current) {
- return;
- }
-
- latestRequestId.current += 1;
- setError(null);
- const message: AudioWaveformWorkerRenderMessage = {
- type: 'render',
- requestId: latestRequestId.current,
- src,
- width: w,
- height: h,
- volume: visibleVolume,
- startFrom,
- durationInFrames,
- displayOffsetInFrames,
- displayDurationInFrames,
- fps: vidConf.fps,
- playbackRate,
- loopDisplay,
- };
- worker.postMessage(message);
- return;
- }
-
canvasElement.width = w;
canvasElement.height = h;
@@ -299,24 +164,9 @@ const AudioWaveformInner: React.FC<{
volume: visibleVolume,
width: w,
});
- }, [
- canUseWorkerPath,
- displayDurationInFrames,
- displayOffsetInFrames,
- durationInFrames,
- height,
- loopDisplay,
- playbackRate,
- portionPeaks,
- src,
- startFrom,
- vidConf.fps,
- visualizationWidth,
- visibleVolume,
- waveformCanvasKey,
- ]);
+ }, [height, portionPeaks, visibleVolume, visualizationWidth]);
- useEffect(() => {
+ useLayoutEffect(() => {
if (!shouldRenderVolumeOverlay) {
return;
}
@@ -365,17 +215,13 @@ const AudioWaveformInner: React.FC<{
return null;
}
- if (!canUseWorkerPath && !peaks) {
+ if (!peaks) {
return null;
}
return (
<div style={getContainerStyle(height)}>
- <canvas
- key={waveformCanvasKey}
- ref={waveformCanvas}
- style={waveformCanvasStyle}
- />
+ <canvas ref={waveformCanvas} style={waveformCanvasStyle} />
{shouldRenderVolumeOverlay ? (
<canvas ref={volumeCanvas} style={volumeCanvasStyle} />
) : null}
diff --git a/packages/timeline-utils/src/audio-waveform-worker.ts b/packages/timeline-utils/src/audio-waveform-worker.ts
index 263a7c140f..8a4fd48989 100644
--- a/packages/timeline-utils/src/audio-waveform-worker.ts
+++ b/packages/timeline-utils/src/audio-waveform-worker.ts
@@ -3,21 +3,26 @@
import type {
AudioWaveformWorkerIncomingMessage,
AudioWaveformWorkerOutgoingMessage,
- AudioWaveformWorkerRenderMessage,
} from './audio-waveform/audio-waveform-worker-types';
-import {drawBars} from './audio-waveform/draw-peaks';
import {loadWaveformPeaks} from './audio-waveform/load-waveform-peaks';
-import {sliceVisibleWaveformPeaks} from './audio-waveform/slice-visible-waveform-peaks';
declare const self: DedicatedWorkerGlobalScope;
-let canvas: OffscreenCanvas | null = null;
-let currentController: AbortController | null = null;
-let latestRequestId = 0;
+const postPeaks = (requestId: number, peaks: Float32Array, final: boolean) => {
+ // Structured cloning copies the array, so the decoder can keep
+ // mutating its buffer while the main thread reads the snapshot.
+ const payload: AudioWaveformWorkerOutgoingMessage = {
+ type: 'peaks',
+ requestId,
+ peaks,
+ final,
+ };
+ self.postMessage(payload);
+};
const postError = (requestId: number, error: unknown) => {
const message =
- error instanceof Error ? error.message : 'Failed to render waveform';
+ error instanceof Error ? error.message : 'Failed to load waveform';
const payload: AudioWaveformWorkerOutgoingMessage = {
type: 'error',
@@ -27,93 +32,24 @@ const postError = (requestId: number, error: unknown) => {
self.postMessage(payload);
};
-const drawPartialWaveform = (
- message: AudioWaveformWorkerRenderMessage,
- peaks: Float32Array,
-) => {
- if (!canvas) {
- return;
- }
-
- const portionPeaks = sliceVisibleWaveformPeaks({
- displayDurationInFrames: message.displayDurationInFrames,
- displayOffsetInFrames: message.displayOffsetInFrames,
- durationInFrames: message.durationInFrames,
- fps: message.fps,
- loopDisplay: message.loopDisplay,
- peaks,
- playbackRate: message.playbackRate,
- startFrom: message.startFrom,
- });
-
- drawBars({
- canvas,
- peaks: portionPeaks,
- color: 'rgba(255, 255, 255, 0.6)',
- volume: message.volume,
- width: message.width,
- });
-};
-
-const renderWaveform = async (message: AudioWaveformWorkerRenderMessage) => {
- if (!canvas) {
- postError(message.requestId, new Error('Waveform canvas not initialized'));
- return;
- }
-
- const controller = new AbortController();
- currentController?.abort();
- currentController = controller;
- latestRequestId = message.requestId;
-
- try {
- canvas.width = message.width;
- canvas.height = message.height;
-
- const peaks = await loadWaveformPeaks(message.src, controller.signal, {
- onProgress: ({peaks: nextPeaks}) => {
- if (
- controller.signal.aborted ||
- latestRequestId !== message.requestId
- ) {
- return;
- }
-
- drawPartialWaveform(message, nextPeaks);
- },
- });
- if (controller.signal.aborted || latestRequestId !== message.requestId) {
- return;
- }
-
- drawPartialWaveform(message, peaks);
- } catch (error) {
- if (controller.signal.aborted || latestRequestId !== message.requestId) {
- return;
- }
-
- postError(message.requestId, error);
- }
-};
-
self.addEventListener(
'message',
(event: MessageEvent<AudioWaveformWorkerIncomingMessage>) => {
const message = event.data;
- if (message.type === 'init') {
- canvas = message.canvas;
- return;
- }
+ const controller = new AbortController();
- if (message.type === 'dispose') {
- currentController?.abort();
- currentController = null;
- canvas = null;
- return;
- }
-
- renderWaveform(message).catch((error) => {
- postError(message.requestId, error);
- });
+ loadWaveformPeaks(message.src, controller.signal, {
+ onProgress: ({peaks, final}) => {
+ if (!final) {
+ postPeaks(message.requestId, peaks, false);
+ }
+ },
+ })
+ .then((peaks) => {
+ postPeaks(message.requestId, peaks, true);
+ })
+ .catch((error) => {
+ postError(message.requestId, error);
+ });
},
);
diff --git a/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts b/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts
index 63e4bdef1f..a7ae3cbe8b 100644
--- a/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts
+++ b/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts
@@ -1,35 +1,17 @@
-import type {TimelineLoopDisplay} from '../loop-display';
-import type {WaveformVolume} from './draw-peaks';
-
-export type AudioWaveformWorkerInitMessage = {
- readonly type: 'init';
- readonly canvas: OffscreenCanvas;
-};
-
-export type AudioWaveformWorkerRenderMessage = {
- readonly type: 'render';
+export type AudioWaveformWorkerLoadMessage = {
+ readonly type: 'load';
readonly requestId: number;
readonly src: string;
- readonly width: number;
- readonly height: number;
- readonly volume: WaveformVolume;
- readonly startFrom: number;
- readonly durationInFrames: number;
- readonly displayOffsetInFrames: number;
- readonly displayDurationInFrames: number;
- readonly fps: number;
- readonly playbackRate: number;
- readonly loopDisplay: TimelineLoopDisplay | undefined;
};
-export type AudioWaveformWorkerDisposeMessage = {
- readonly type: 'dispose';
-};
+export type AudioWaveformWorkerIncomingMessage = AudioWaveformWorkerLoadMessage;
-export type AudioWaveformWorkerIncomingMessage =
- | AudioWaveformWorkerInitMessage
- | AudioWaveformWorkerRenderMessage
- | AudioWaveformWorkerDisposeMessage;
+export type AudioWaveformWorkerPeaksMessage = {
+ readonly type: 'peaks';
+ readonly requestId: number;
+ readonly peaks: Float32Array;
+ readonly final: boolean;
+};
export type AudioWaveformWorkerErrorMessage = {
readonly type: 'error';
@@ -38,4 +20,5 @@ export type AudioWaveformWorkerErrorMessage = {
};
export type AudioWaveformWorkerOutgoingMessage =
- AudioWaveformWorkerErrorMessage;
+ | AudioWaveformWorkerPeaksMessage
+ | AudioWaveformWorkerErrorMessage;
diff --git a/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts b/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts
new file mode 100644
index 0000000000..7ed2000e0f
--- /dev/null
+++ b/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts
@@ -0,0 +1,197 @@
+import type {
+ AudioWaveformWorkerLoadMessage,
+ AudioWaveformWorkerOutgoingMessage,
+} from './audio-waveform-worker-types';
+import {loadWaveformPeaks} from './load-waveform-peaks';
+import {makeAudioWaveformWorker} from './make-audio-waveform-worker';
+
+type WaveformPeaksListener = {
+ readonly onPeaks: (peaks: Float32Array, final: boolean) => void;
+ readonly onError: (error: Error) => void;
+};
+
+type InFlightLoad = {
+ readonly src: string;
+ readonly listeners: Set<WaveformPeaksListener>;
+ latestPeaks: Float32Array | null;
+ requestId: number | null;
+};
+
+// All AudioWaveform instances share one decode worker and one peaks cache,
+// so remounts (timeline virtualization, zoom window cropping) and multiple
+// clips with the same src never re-fetch or re-decode the audio.
+const peaksCache = new Map<string, Float32Array>();
+const inFlightBySrc = new Map<string, InFlightLoad>();
+const inFlightByRequestId = new Map<number, InFlightLoad>();
+
+let worker: Worker | null = null;
+let workerFailed = false;
+let nextRequestId = 0;
+
+const emitPeaks = (load: InFlightLoad, peaks: Float32Array, final: boolean) => {
+ if (final) {
+ peaksCache.set(load.src, peaks);
+ inFlightBySrc.delete(load.src);
+ if (load.requestId !== null) {
+ inFlightByRequestId.delete(load.requestId);
+ }
+ } else {
+ load.latestPeaks = peaks;
+ }
+
+ for (const listener of load.listeners) {
+ listener.onPeaks(peaks, final);
+ }
+};
+
+const emitError = (load: InFlightLoad, error: Error) => {
+ inFlightBySrc.delete(load.src);
+ if (load.requestId !== null) {
+ inFlightByRequestId.delete(load.requestId);
+ }
+
+ for (const listener of load.listeners) {
+ listener.onError(error);
+ }
+};
+
+const startMainThreadLoad = (load: InFlightLoad) => {
+ load.requestId = null;
+
+ // Never aborted: even if all subscribers unsubscribe, finishing the
+ // decode fills the cache for the next mount.
+ const controller = new AbortController();
+
+ loadWaveformPeaks(load.src, controller.signal, {
+ onProgress: ({peaks, final}) => {
+ if (final) {
+ return;
+ }
+
+ // The processor mutates the same array in place; snapshot it so
+ // React state updates see a new reference.
+ emitPeaks(load, peaks.slice(), false);
+ },
+ })
+ .then((peaks) => {
+ emitPeaks(load, peaks, true);
+ })
+ .catch((error) => {
+ emitError(
+ load,
+ error instanceof Error
+ ? error
+ : new Error('Failed to load waveform peaks'),
+ );
+ });
+};
+
+const handleWorkerFailure = () => {
+ workerFailed = true;
+ worker?.terminate();
+ worker = null;
+
+ const pending = [...inFlightByRequestId.values()];
+ inFlightByRequestId.clear();
+ for (const load of pending) {
+ startMainThreadLoad(load);
+ }
+};
+
+const getOrCreateWorker = (): Worker | null => {
+ if (workerFailed || typeof Worker === 'undefined') {
+ return null;
+ }
+
+ if (worker) {
+ return worker;
+ }
+
+ try {
+ worker = makeAudioWaveformWorker();
+ } catch {
+ workerFailed = true;
+ return null;
+ }
+
+ worker.addEventListener(
+ 'message',
+ (event: MessageEvent<AudioWaveformWorkerOutgoingMessage>) => {
+ const message = event.data;
+ const load = inFlightByRequestId.get(message.requestId);
+ if (!load) {
+ return;
+ }
+
+ if (message.type === 'error') {
+ emitError(load, new Error(message.message));
+ return;
+ }
+
+ emitPeaks(load, message.peaks, message.final);
+ },
+ );
+ worker.addEventListener('error', (event) => {
+ event.preventDefault();
+ handleWorkerFailure();
+ });
+
+ return worker;
+};
+
+export const subscribeToWaveformPeaks = ({
+ src,
+ onPeaks,
+ onError,
+}: {
+ readonly src: string;
+ readonly onPeaks: (peaks: Float32Array, final: boolean) => void;
+ readonly onError: (error: Error) => void;
+}): (() => void) => {
+ const cached = peaksCache.get(src);
+ if (cached) {
+ onPeaks(cached, true);
+ return () => undefined;
+ }
+
+ const listener: WaveformPeaksListener = {onPeaks, onError};
+
+ const existing = inFlightBySrc.get(src);
+ if (existing) {
+ existing.listeners.add(listener);
+ if (existing.latestPeaks) {
+ onPeaks(existing.latestPeaks, false);
+ }
+
+ return () => {
+ existing.listeners.delete(listener);
+ };
+ }
+
+ const load: InFlightLoad = {
+ src,
+ listeners: new Set([listener]),
+ latestPeaks: null,
+ requestId: null,
+ };
+ inFlightBySrc.set(src, load);
+
+ const workerInstance = getOrCreateWorker();
+ if (workerInstance) {
+ const requestId = nextRequestId++;
+ load.requestId = requestId;
+ inFlightByRequestId.set(requestId, load);
+ const message: AudioWaveformWorkerLoadMessage = {
+ type: 'load',
+ requestId,
+ src,
+ };
+ workerInstance.postMessage(message);
+ } else {
+ startMainThreadLoad(load);
+ }
+
+ return () => {
+ load.listeners.delete(listener);
+ };
+};
diff --git a/packages/timeline-utils/src/index.ts b/packages/timeline-utils/src/index.ts
index 4726eec0db..2288c443ce 100644
--- a/packages/timeline-utils/src/index.ts
+++ b/packages/timeline-utils/src/index.ts
@@ -1,14 +1,13 @@
export type {
AudioWaveformWorkerIncomingMessage,
AudioWaveformWorkerOutgoingMessage,
- AudioWaveformWorkerRenderMessage,
} from './audio-waveform/audio-waveform-worker-types';
export {TARGET_SAMPLE_RATE} from './audio-waveform/constants';
export {drawBars, type WaveformVolume} from './audio-waveform/draw-peaks';
export {getVisibleWaveformVolume} from './audio-waveform/get-visible-waveform-volume';
export {loadWaveformPeaks} from './audio-waveform/load-waveform-peaks';
-export {makeAudioWaveformWorker} from './audio-waveform/make-audio-waveform-worker';
export {sliceVisibleWaveformPeaks} from './audio-waveform/slice-visible-waveform-peaks';
+export {subscribeToWaveformPeaks} from './audio-waveform/subscribe-to-waveform-peaks';
export {sliceWaveformPeaks} from './audio-waveform/slice-waveform-peaks';
export {
createWaveformPeakProcessor,