🎥 Make videos programmatically with React
`@remotion/studio`: Fix timeline waveform flicker when zooming (#10630)
Jonny Burger411a0de8a351
diff --git a/packages/studio/src/components/AudioWaveform.tsx b/packages/studio/src/components/AudioWaveform.tsxindex 5ef6d2ccb1..44ed89c632 100644--- a/packages/studio/src/components/AudioWaveform.tsx+++ b/packages/studio/src/components/AudioWaveform.tsx@@ -1,14 +1,11 @@import {drawBars,getVisibleWaveformVolume,- loadWaveformPeaks,- makeAudioWaveformWorker,sliceVisibleWaveformPeaks,- type AudioWaveformWorkerOutgoingMessage,- type AudioWaveformWorkerRenderMessage,+ subscribeToWaveformPeaks,type WaveformVolume,} from '@remotion/timeline-utils';-import React, {useEffect, useMemo, useRef, useState} from 'react';+import React, {useLayoutEffect, useMemo, useRef, useState} from 'react';import type {LoopDisplay} from 'remotion';import {Internals} from 'remotion';import {WHITE_ALPHA_70, WHITE_ALPHA_60} from '../helpers/colors';@@ -16,24 +13,6 @@ import {TIMELINE_BORDER} from '../helpers/timeline-layout';const EMPTY_PEAKS = new Float32Array(0);-// Recreate the canvas after Fast Refresh because an already transferred canvas-// cannot be handed to OffscreenCanvas again.-const canRetryCanvasTransfer = (err: unknown) => {- return err instanceof DOMException && err.name === 'InvalidStateError';-};--const canUseAudioWaveformWorker = () => {- if (- typeof Worker === 'undefined' ||- typeof OffscreenCanvas === 'undefined' ||- typeof HTMLCanvasElement === 'undefined'- ) {- return false;- }-- return 'transferControlToOffscreen' in HTMLCanvasElement.prototype;-};-const getContainerStyle = (height: number): React.CSSProperties => {return {display: 'flex',@@ -45,12 +24,6 @@ const getContainerStyle = (height: number): React.CSSProperties => {};};-const getWaveformErrorMessage = () => {- return new Error(- 'No waveform available. The audio could not be decoded or may not support CORS.',- );-};-const waveformCanvasStyle: React.CSSProperties = {pointerEvents: 'none',width: '100%',@@ -100,8 +73,6 @@ const AudioWaveformInner: React.FC<{}) => {const [peaks, setPeaks] = useState<Float32Array | null>(null);const [error, setError] = useState<Error | null>(null);- const [waveformCanvasKey, setWaveformCanvasKey] = useState(0);- const canUseWorkerPath = useMemo(() => canUseAudioWaveformWorker(), []);const vidConf = Internals.useUnsafeVideoConfig();if (vidConf === null) {throw new Error('Expected video config');@@ -109,9 +80,6 @@ const AudioWaveformInner: React.FC<{const waveformCanvas = useRef<HTMLCanvasElement>(null);const volumeCanvas = useRef<HTMLCanvasElement>(null);- const waveformWorker = useRef<Worker | null>(null);- const hasTransferredCanvas = useRef(false);- const latestRequestId = useRef(0);const shouldRenderVolumeOverlay =doesVolumeChange && typeof volume === 'string';const parsedVolume = useMemo(() => parseVolume(volume), [volume]);@@ -134,99 +102,21 @@ const AudioWaveformInner: React.FC<{parsedVolume,]);- useEffect(() => {- if (canUseWorkerPath) {- return;- }-- const controller = new AbortController();-+ // Layout effect so that a cache hit sets the peaks synchronously and the+ // waveform is painted on the very first frame after mounting.+ useLayoutEffect(() => {setPeaks(null);setError(null);- loadWaveformPeaks(src, controller.signal)- .then((p) => {- if (!controller.signal.aborted) {- setPeaks(p);- }- })- .catch((err) => {- if (!controller.signal.aborted) {- setError(err);- }- });-- return () => controller.abort();- }, [canUseWorkerPath, src]);-- useEffect(() => {- if (!canUseWorkerPath) {- return;- }-- const canvasElement = waveformCanvas.current;- if (!canvasElement || hasTransferredCanvas.current) {- return;- }-- const worker = makeAudioWaveformWorker();- let workerFailed = false;- waveformWorker.current = worker;- worker.addEventListener(- 'message',- (event: MessageEvent<AudioWaveformWorkerOutgoingMessage>) => {- if (event.data.type === 'error') {- if (event.data.requestId !== latestRequestId.current) {- return;- }-- setError(new Error(event.data.message));- }- },- );- worker.addEventListener('error', (event) => {- event.preventDefault();- workerFailed = true;-- if (worker !== waveformWorker.current) {- return;- }- worker.terminate();- waveformWorker.current = null;- hasTransferredCanvas.current = false;- setError(getWaveformErrorMessage());+ return subscribeToWaveformPeaks({+ src,+ onPeaks: (p) => setPeaks(p),+ onError: (err) => setError(err),});-- let offscreen: OffscreenCanvas;- try {- offscreen = canvasElement.transferControlToOffscreen();- } catch (err) {- worker.terminate();- waveformWorker.current = null;- if (canRetryCanvasTransfer(err)) {- setWaveformCanvasKey((key) => key + 1);- return;- }-- throw err;- }-- hasTransferredCanvas.current = true;- worker.postMessage({type: 'init', canvas: offscreen}, [offscreen]);-- return () => {- if (!workerFailed) {- worker.postMessage({type: 'dispose'});- }-- worker.terminate();- waveformWorker.current = null;- hasTransferredCanvas.current = false;- };- }, [canUseWorkerPath, waveformCanvasKey]);+ }, [src]);const portionPeaks = useMemo(() => {- if (canUseWorkerPath || !peaks) {+ if (!peaks) {return null;}@@ -241,7 +131,6 @@ const AudioWaveformInner: React.FC<{startFrom,});}, [- canUseWorkerPath,displayDurationInFrames,displayOffsetInFrames,durationInFrames,@@ -252,7 +141,10 @@ const AudioWaveformInner: React.FC<{vidConf.fps,]);- useEffect(() => {+ // Drawing must happen in a layout effect: when the timeline zooms, the+ // canvas CSS box resizes in the same commit, and painting the bitmap+ // before the browser paints avoids showing a stretched stale waveform.+ useLayoutEffect(() => {const {current: canvasElement} = waveformCanvas;if (!canvasElement) {return;@@ -262,33 +154,6 @@ const AudioWaveformInner: React.FC<{const h = Math.ceil(height * pixelRatio);const w = Math.ceil(visualizationWidth * pixelRatio);- if (canUseWorkerPath) {- const worker = waveformWorker.current;- if (!worker || !hasTransferredCanvas.current) {- return;- }-- latestRequestId.current += 1;- setError(null);- const message: AudioWaveformWorkerRenderMessage = {- type: 'render',- requestId: latestRequestId.current,- src,- width: w,- height: h,- volume: visibleVolume,- startFrom,- durationInFrames,- displayOffsetInFrames,- displayDurationInFrames,- fps: vidConf.fps,- playbackRate,- loopDisplay,- };- worker.postMessage(message);- return;- }-canvasElement.width = w;canvasElement.height = h;@@ -299,24 +164,9 @@ const AudioWaveformInner: React.FC<{volume: visibleVolume,width: w,});- }, [- canUseWorkerPath,- displayDurationInFrames,- displayOffsetInFrames,- durationInFrames,- height,- loopDisplay,- playbackRate,- portionPeaks,- src,- startFrom,- vidConf.fps,- visualizationWidth,- visibleVolume,- waveformCanvasKey,- ]);+ }, [height, portionPeaks, visibleVolume, visualizationWidth]);- useEffect(() => {+ useLayoutEffect(() => {if (!shouldRenderVolumeOverlay) {return;}@@ -365,17 +215,13 @@ const AudioWaveformInner: React.FC<{return null;}- if (!canUseWorkerPath && !peaks) {+ if (!peaks) {return null;}return (<div style={getContainerStyle(height)}>- <canvas- key={waveformCanvasKey}- ref={waveformCanvas}- style={waveformCanvasStyle}- />+ <canvas ref={waveformCanvas} style={waveformCanvasStyle} />{shouldRenderVolumeOverlay ? (<canvas ref={volumeCanvas} style={volumeCanvasStyle} />) : null}diff --git a/packages/timeline-utils/src/audio-waveform-worker.ts b/packages/timeline-utils/src/audio-waveform-worker.tsindex 263a7c140f..8a4fd48989 100644--- a/packages/timeline-utils/src/audio-waveform-worker.ts+++ b/packages/timeline-utils/src/audio-waveform-worker.ts@@ -3,21 +3,26 @@import type {AudioWaveformWorkerIncomingMessage,AudioWaveformWorkerOutgoingMessage,- AudioWaveformWorkerRenderMessage,} from './audio-waveform/audio-waveform-worker-types';-import {drawBars} from './audio-waveform/draw-peaks';import {loadWaveformPeaks} from './audio-waveform/load-waveform-peaks';-import {sliceVisibleWaveformPeaks} from './audio-waveform/slice-visible-waveform-peaks';declare const self: DedicatedWorkerGlobalScope;-let canvas: OffscreenCanvas | null = null;-let currentController: AbortController | null = null;-let latestRequestId = 0;+const postPeaks = (requestId: number, peaks: Float32Array, final: boolean) => {+ // Structured cloning copies the array, so the decoder can keep+ // mutating its buffer while the main thread reads the snapshot.+ const payload: AudioWaveformWorkerOutgoingMessage = {+ type: 'peaks',+ requestId,+ peaks,+ final,+ };+ self.postMessage(payload);+};const postError = (requestId: number, error: unknown) => {const message =- error instanceof Error ? error.message : 'Failed to render waveform';+ error instanceof Error ? error.message : 'Failed to load waveform';const payload: AudioWaveformWorkerOutgoingMessage = {type: 'error',@@ -27,93 +32,24 @@ const postError = (requestId: number, error: unknown) => {self.postMessage(payload);};-const drawPartialWaveform = (- message: AudioWaveformWorkerRenderMessage,- peaks: Float32Array,-) => {- if (!canvas) {- return;- }-- const portionPeaks = sliceVisibleWaveformPeaks({- displayDurationInFrames: message.displayDurationInFrames,- displayOffsetInFrames: message.displayOffsetInFrames,- durationInFrames: message.durationInFrames,- fps: message.fps,- loopDisplay: message.loopDisplay,- peaks,- playbackRate: message.playbackRate,- startFrom: message.startFrom,- });-- drawBars({- canvas,- peaks: portionPeaks,- color: 'rgba(255, 255, 255, 0.6)',- volume: message.volume,- width: message.width,- });-};--const renderWaveform = async (message: AudioWaveformWorkerRenderMessage) => {- if (!canvas) {- postError(message.requestId, new Error('Waveform canvas not initialized'));- return;- }-- const controller = new AbortController();- currentController?.abort();- currentController = controller;- latestRequestId = message.requestId;-- try {- canvas.width = message.width;- canvas.height = message.height;-- const peaks = await loadWaveformPeaks(message.src, controller.signal, {- onProgress: ({peaks: nextPeaks}) => {- if (- controller.signal.aborted ||- latestRequestId !== message.requestId- ) {- return;- }-- drawPartialWaveform(message, nextPeaks);- },- });- if (controller.signal.aborted || latestRequestId !== message.requestId) {- return;- }-- drawPartialWaveform(message, peaks);- } catch (error) {- if (controller.signal.aborted || latestRequestId !== message.requestId) {- return;- }-- postError(message.requestId, error);- }-};-self.addEventListener('message',(event: MessageEvent<AudioWaveformWorkerIncomingMessage>) => {const message = event.data;- if (message.type === 'init') {- canvas = message.canvas;- return;- }+ const controller = new AbortController();- if (message.type === 'dispose') {- currentController?.abort();- currentController = null;- canvas = null;- return;- }-- renderWaveform(message).catch((error) => {- postError(message.requestId, error);- });+ loadWaveformPeaks(message.src, controller.signal, {+ onProgress: ({peaks, final}) => {+ if (!final) {+ postPeaks(message.requestId, peaks, false);+ }+ },+ })+ .then((peaks) => {+ postPeaks(message.requestId, peaks, true);+ })+ .catch((error) => {+ postError(message.requestId, error);+ });},);diff --git a/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts b/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.tsindex 63e4bdef1f..a7ae3cbe8b 100644--- a/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts+++ b/packages/timeline-utils/src/audio-waveform/audio-waveform-worker-types.ts@@ -1,35 +1,17 @@-import type {TimelineLoopDisplay} from '../loop-display';-import type {WaveformVolume} from './draw-peaks';--export type AudioWaveformWorkerInitMessage = {- readonly type: 'init';- readonly canvas: OffscreenCanvas;-};--export type AudioWaveformWorkerRenderMessage = {- readonly type: 'render';+export type AudioWaveformWorkerLoadMessage = {+ readonly type: 'load';readonly requestId: number;readonly src: string;- readonly width: number;- readonly height: number;- readonly volume: WaveformVolume;- readonly startFrom: number;- readonly durationInFrames: number;- readonly displayOffsetInFrames: number;- readonly displayDurationInFrames: number;- readonly fps: number;- readonly playbackRate: number;- readonly loopDisplay: TimelineLoopDisplay | undefined;};-export type AudioWaveformWorkerDisposeMessage = {- readonly type: 'dispose';-};+export type AudioWaveformWorkerIncomingMessage = AudioWaveformWorkerLoadMessage;-export type AudioWaveformWorkerIncomingMessage =- | AudioWaveformWorkerInitMessage- | AudioWaveformWorkerRenderMessage- | AudioWaveformWorkerDisposeMessage;+export type AudioWaveformWorkerPeaksMessage = {+ readonly type: 'peaks';+ readonly requestId: number;+ readonly peaks: Float32Array;+ readonly final: boolean;+};export type AudioWaveformWorkerErrorMessage = {readonly type: 'error';@@ -38,4 +20,5 @@ export type AudioWaveformWorkerErrorMessage = {};export type AudioWaveformWorkerOutgoingMessage =- AudioWaveformWorkerErrorMessage;+ | AudioWaveformWorkerPeaksMessage+ | AudioWaveformWorkerErrorMessage;diff --git a/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts b/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.tsnew file mode 100644index 0000000000..7ed2000e0f--- /dev/null+++ b/packages/timeline-utils/src/audio-waveform/subscribe-to-waveform-peaks.ts@@ -0,0 +1,197 @@+import type {+ AudioWaveformWorkerLoadMessage,+ AudioWaveformWorkerOutgoingMessage,+} from './audio-waveform-worker-types';+import {loadWaveformPeaks} from './load-waveform-peaks';+import {makeAudioWaveformWorker} from './make-audio-waveform-worker';++type WaveformPeaksListener = {+ readonly onPeaks: (peaks: Float32Array, final: boolean) => void;+ readonly onError: (error: Error) => void;+};++type InFlightLoad = {+ readonly src: string;+ readonly listeners: Set<WaveformPeaksListener>;+ latestPeaks: Float32Array | null;+ requestId: number | null;+};++// All AudioWaveform instances share one decode worker and one peaks cache,+// so remounts (timeline virtualization, zoom window cropping) and multiple+// clips with the same src never re-fetch or re-decode the audio.+const peaksCache = new Map<string, Float32Array>();+const inFlightBySrc = new Map<string, InFlightLoad>();+const inFlightByRequestId = new Map<number, InFlightLoad>();++let worker: Worker | null = null;+let workerFailed = false;+let nextRequestId = 0;++const emitPeaks = (load: InFlightLoad, peaks: Float32Array, final: boolean) => {+ if (final) {+ peaksCache.set(load.src, peaks);+ inFlightBySrc.delete(load.src);+ if (load.requestId !== null) {+ inFlightByRequestId.delete(load.requestId);+ }+ } else {+ load.latestPeaks = peaks;+ }++ for (const listener of load.listeners) {+ listener.onPeaks(peaks, final);+ }+};++const emitError = (load: InFlightLoad, error: Error) => {+ inFlightBySrc.delete(load.src);+ if (load.requestId !== null) {+ inFlightByRequestId.delete(load.requestId);+ }++ for (const listener of load.listeners) {+ listener.onError(error);+ }+};++const startMainThreadLoad = (load: InFlightLoad) => {+ load.requestId = null;++ // Never aborted: even if all subscribers unsubscribe, finishing the+ // decode fills the cache for the next mount.+ const controller = new AbortController();++ loadWaveformPeaks(load.src, controller.signal, {+ onProgress: ({peaks, final}) => {+ if (final) {+ return;+ }++ // The processor mutates the same array in place; snapshot it so+ // React state updates see a new reference.+ emitPeaks(load, peaks.slice(), false);+ },+ })+ .then((peaks) => {+ emitPeaks(load, peaks, true);+ })+ .catch((error) => {+ emitError(+ load,+ error instanceof Error+ ? error+ : new Error('Failed to load waveform peaks'),+ );+ });+};++const handleWorkerFailure = () => {+ workerFailed = true;+ worker?.terminate();+ worker = null;++ const pending = [...inFlightByRequestId.values()];+ inFlightByRequestId.clear();+ for (const load of pending) {+ startMainThreadLoad(load);+ }+};++const getOrCreateWorker = (): Worker | null => {+ if (workerFailed || typeof Worker === 'undefined') {+ return null;+ }++ if (worker) {+ return worker;+ }++ try {+ worker = makeAudioWaveformWorker();+ } catch {+ workerFailed = true;+ return null;+ }++ worker.addEventListener(+ 'message',+ (event: MessageEvent<AudioWaveformWorkerOutgoingMessage>) => {+ const message = event.data;+ const load = inFlightByRequestId.get(message.requestId);+ if (!load) {+ return;+ }++ if (message.type === 'error') {+ emitError(load, new Error(message.message));+ return;+ }++ emitPeaks(load, message.peaks, message.final);+ },+ );+ worker.addEventListener('error', (event) => {+ event.preventDefault();+ handleWorkerFailure();+ });++ return worker;+};++export const subscribeToWaveformPeaks = ({+ src,+ onPeaks,+ onError,+}: {+ readonly src: string;+ readonly onPeaks: (peaks: Float32Array, final: boolean) => void;+ readonly onError: (error: Error) => void;+}): (() => void) => {+ const cached = peaksCache.get(src);+ if (cached) {+ onPeaks(cached, true);+ return () => undefined;+ }++ const listener: WaveformPeaksListener = {onPeaks, onError};++ const existing = inFlightBySrc.get(src);+ if (existing) {+ existing.listeners.add(listener);+ if (existing.latestPeaks) {+ onPeaks(existing.latestPeaks, false);+ }++ return () => {+ existing.listeners.delete(listener);+ };+ }++ const load: InFlightLoad = {+ src,+ listeners: new Set([listener]),+ latestPeaks: null,+ requestId: null,+ };+ inFlightBySrc.set(src, load);++ const workerInstance = getOrCreateWorker();+ if (workerInstance) {+ const requestId = nextRequestId++;+ load.requestId = requestId;+ inFlightByRequestId.set(requestId, load);+ const message: AudioWaveformWorkerLoadMessage = {+ type: 'load',+ requestId,+ src,+ };+ workerInstance.postMessage(message);+ } else {+ startMainThreadLoad(load);+ }++ return () => {+ load.listeners.delete(listener);+ };+};diff --git a/packages/timeline-utils/src/index.ts b/packages/timeline-utils/src/index.tsindex 4726eec0db..2288c443ce 100644--- a/packages/timeline-utils/src/index.ts+++ b/packages/timeline-utils/src/index.ts@@ -1,14 +1,13 @@export type {AudioWaveformWorkerIncomingMessage,AudioWaveformWorkerOutgoingMessage,- AudioWaveformWorkerRenderMessage,} from './audio-waveform/audio-waveform-worker-types';export {TARGET_SAMPLE_RATE} from './audio-waveform/constants';export {drawBars, type WaveformVolume} from './audio-waveform/draw-peaks';export {getVisibleWaveformVolume} from './audio-waveform/get-visible-waveform-volume';export {loadWaveformPeaks} from './audio-waveform/load-waveform-peaks';-export {makeAudioWaveformWorker} from './audio-waveform/make-audio-waveform-worker';export {sliceVisibleWaveformPeaks} from './audio-waveform/slice-visible-waveform-peaks';+export {subscribeToWaveformPeaks} from './audio-waveform/subscribe-to-waveform-peaks';export {sliceWaveformPeaks} from './audio-waveform/slice-waveform-peaks';export {createWaveformPeakProcessor,