🎥 Make videos programmatically with React
Pass render sample rate through audio context and extraction
JonnyBurgerf6bdfd65462c
diff --git a/packages/core/src/audio/shared-audio-tags.tsx b/packages/core/src/audio/shared-audio-tags.tsxindex 62db135dd4..4e602b3a20 100644--- a/packages/core/src/audio/shared-audio-tags.tsx+++ b/packages/core/src/audio/shared-audio-tags.tsx@@ -79,6 +79,7 @@ export type AudioSyncAnchorEmitter = {};type SharedAudioContextValue = {+ sampleRate: number;audioContext: AudioContext | null;getAudioContextState: () => RemotionAudioContextState | null;gainNode: GainNode | null;@@ -218,14 +219,6 @@ export const SharedAudioContextProvider: React.FC<{const logLevel = useLogLevel();const sampleRate = previewSampleRate ?? 48000;- useEffect(() => {- if (typeof window === 'undefined') {- return;- }-- window.remotion_sampleRate = sampleRate;- }, [sampleRate]);-const ctxAndGain = useSingletonAudioContext({logLevel,latencyHint: audioLatencyHint,@@ -559,6 +552,7 @@ export const SharedAudioContextProvider: React.FC<{const audioContextValue: SharedAudioContextValue = useMemo(() => {return {+ sampleRate,audioContext: ctxAndGain?.audioContext ?? null,getAudioContextState: () => ctxAndGain?.getState() ?? null,gainNode: ctxAndGain?.gainNode ?? null,@@ -573,6 +567,7 @@ export const SharedAudioContextProvider: React.FC<{_experimentalKeepAudioContextAlive,};}, [+ sampleRate,ctxAndGain,audioSyncAnchor,audioSyncAnchorEmitter,diff --git a/packages/media/src/audio-extraction/extract-audio.ts b/packages/media/src/audio-extraction/extract-audio.tsindex b557c3dab1..078f76be32 100644--- a/packages/media/src/audio-extraction/extract-audio.ts+++ b/packages/media/src/audio-extraction/extract-audio.ts@@ -18,6 +18,7 @@ import type {MediaRequestInit} from '../request-init';type ExtractAudioReturnType = Awaited<ReturnType<typeof extractAudioInternal>>;type ExtractAudioParams = {+ sampleRate: number;src: string;timeInSeconds: number;durationInSeconds: number;@@ -35,6 +36,7 @@ type ExtractAudioParams = {};const extractAudioInternal = async ({+ sampleRate,src,timeInSeconds: unloopedTimeInSeconds,durationInSeconds: durationNotYetApplyingPlaybackRate,@@ -209,6 +211,7 @@ const extractAudioInternal = async ({const combined = combineAudioDataAndClosePrevious(audioDataArray);const resampled = resamplePcmS16AudioData({+ targetSampleRate: sampleRate,audioData: combined,playbackRate,isLast: true,diff --git a/packages/media/src/audio/audio-for-rendering.tsx b/packages/media/src/audio/audio-for-rendering.tsxindex 7b1fb75edc..a2b0fb0a7b 100644--- a/packages/media/src/audio/audio-for-rendering.tsx+++ b/packages/media/src/audio/audio-for-rendering.tsx@@ -11,7 +11,6 @@ import {} from 'remotion';import {useMaxMediaCacheSize, useRenderMediaCache} from '../caches';import {applyVolume} from '../convert-audiodata/apply-volume';-import {getTargetSampleRate} from '../convert-audiodata/resample-audiodata';import {frameForVolumeProp} from '../looped-frame';import {callOnErrorAndResolve} from '../on-error';import {extractFrameViaBroadcastChannel} from '../video-extraction/extract-frame-via-broadcast-channel';@@ -40,6 +39,8 @@ export const AudioForRendering: React.FC<AudioProps> = ({credentials,requestInit,}) => {+ const audioContext = useContext(Internals.SharedAudioContext);+ const sampleRate = audioContext?.sampleRate ?? 48000;const defaultLogLevel = Internals.useLogLevel();const logLevel = overriddenLogLevel ?? defaultLogLevel;const frame = useCurrentFrame();@@ -122,6 +123,7 @@ export const AudioForRendering: React.FC<AudioProps> = ({});extractFrameViaBroadcastChannel({+ sampleRate,src,timeInSeconds: timestamp,durationInSeconds,@@ -243,8 +245,7 @@ export const AudioForRendering: React.FC<AudioProps> = ({frame: absoluteFrame,startInVideo,timestamp: audio.timestamp,- duration:- (audio.numberOfFrames / getTargetSampleRate()) * 1_000_000,+ duration: audio.durationInMicroSeconds,toneFrequency: toneFrequency ?? 1,});}@@ -264,6 +265,7 @@ export const AudioForRendering: React.FC<AudioProps> = ({unregisterRenderAsset(id);};}, [+ sampleRate,absoluteFrame,continueRender,delayRender,diff --git a/packages/media/src/convert-audiodata/convert-audiodata.ts b/packages/media/src/convert-audiodata/convert-audiodata.tsindex a24a45453c..8b6800b7ac 100644--- a/packages/media/src/convert-audiodata/convert-audiodata.ts+++ b/packages/media/src/convert-audiodata/convert-audiodata.ts@@ -1,10 +1,10 @@import {resampleAudioData,TARGET_NUMBER_OF_CHANNELS,- getTargetSampleRate,} from './resample-audiodata';export type ConvertAudioDataOptions = {+ targetSampleRate: number;audioData: AudioData;trimStartInSeconds: number;trimEndInSeconds: number;@@ -54,7 +54,7 @@ export const convertAudioDataToS16 = ({isLast,}: Omit<ConvertAudioDataOptions,- 'playbackRate'+ 'playbackRate' | 'targetSampleRate'>): UnresampledPcmS16AudioData => {const {numberOfChannels: srcNumberOfChannels,@@ -151,15 +151,17 @@ export const convertAudioDataToS16 = ({};export const resamplePcmS16AudioData = ({+ targetSampleRate,audioData,playbackRate,isLast,}: {+ targetSampleRate: number;audioData: UnresampledPcmS16AudioData;playbackRate: number;isLast: boolean;}): PcmS16AudioData => {- const ratio = audioData.sampleRate / getTargetSampleRate();+ const ratio = audioData.sampleRate / targetSampleRate;const newNumberOfFrames = isLast? ceilButNotIfFloatingPointIssue(audioData.numberOfFrames / ratio / playbackRate,@@ -182,7 +184,7 @@ export const resamplePcmS16AudioData = ({numberOfFrames: newNumberOfFrames,timestamp: audioData.timestamp,durationInMicroSeconds: fixFloatingPoint(- (newNumberOfFrames / getTargetSampleRate()) * 1_000_000,+ (newNumberOfFrames / targetSampleRate) * 1_000_000,),};}@@ -203,7 +205,7 @@ export const resamplePcmS16AudioData = ({numberOfFrames: newNumberOfFrames,timestamp: audioData.timestamp,durationInMicroSeconds: fixFloatingPoint(- (newNumberOfFrames / getTargetSampleRate()) * 1_000_000,+ (newNumberOfFrames / targetSampleRate) * 1_000_000,),};@@ -216,6 +218,7 @@ export const convertAudioData = (const audioData = convertAudioDataToS16(options);return resamplePcmS16AudioData({+ targetSampleRate: options.targetSampleRate,audioData,playbackRate: options.playbackRate,isLast: options.isLast,diff --git a/packages/media/src/extract-frame-and-audio.ts b/packages/media/src/extract-frame-and-audio.tsindex 76e586649c..413db2282b 100644--- a/packages/media/src/extract-frame-and-audio.ts+++ b/packages/media/src/extract-frame-and-audio.ts@@ -8,6 +8,7 @@ import type {ExtractFrameViaBroadcastChannelResult} from './video-extraction/extimport {rotateFrame} from './video-extraction/rotate-frame';export const extractFrameAndAudio = async ({+ sampleRate,src,timeInSeconds,logLevel,@@ -25,6 +26,7 @@ export const extractFrameAndAudio = async ({requestInit,mediaCache,}: {+ sampleRate: number;src: string;timeInSeconds: number;logLevel: LogLevel;@@ -62,6 +64,7 @@ export const extractFrameAndAudio = async ({: null,includeAudio? extractAudio({+ sampleRate,src,timeInSeconds,durationInSeconds,diff --git a/packages/media/src/test/audio-encoding.test.ts b/packages/media/src/test/audio-encoding.test.tsindex f68783aca9..21c41720ff 100644--- a/packages/media/src/test/audio-encoding.test.ts+++ b/packages/media/src/test/audio-encoding.test.ts@@ -24,6 +24,7 @@ test('Audio samples from MP3 should produce identical s16 output on Chrome and F}const a = await extractAudio({+ sampleRate: 48000,src: '/mp3-f32-audio.mp3',timeInSeconds: 0.03333333333333333,logLevel: 'info',diff --git a/packages/media/src/test/browser.test.ts b/packages/media/src/test/browser.test.tsindex 560eb6f7a8..370a8ed6e1 100644--- a/packages/media/src/test/browser.test.ts+++ b/packages/media/src/test/browser.test.ts@@ -11,6 +11,7 @@ test('Should be able to extract a frame', async () => {keyframeManager.clearAll('info');const result = await extractFrameAndAudio({+ sampleRate: 48000,src: '/bigbuckbunny.mp4',timeInSeconds: 1,durationInSeconds: 1 / 30,@@ -67,6 +68,7 @@ test('Should be able to extract the last frame', async () => {keyframeManager.clearAll('info');const result = await extractFrameAndAudio({+ sampleRate: 48000,src: '/bigbuckbunny.mp4',timeInSeconds: 1_000_000,durationInSeconds: 1 / 30,@@ -123,6 +125,7 @@ test('Should manage the cache', async (t) => {for (let i = 0; i < 50; i++) {await extractFrameAndAudio({+ sampleRate: 48000,src: `/bigbuckbunny.mp4?i=${i}`,timeInSeconds: 1,durationInSeconds: 1 / 30,@@ -154,6 +157,7 @@ test('Should be apply volume correctly', async () => {keyframeManager.clearAll('info');const result = await extractFrameAndAudio({+ sampleRate: 48000,src: '/bigbuckbunny.mp4',timeInSeconds: 1,durationInSeconds: 1 / 30,@@ -214,6 +218,7 @@ test('Should be able to loop', async () => {keyframeManager.clearAll('info');const result = await extractFrameAndAudio({+ sampleRate: 48000,src: `/bigbuckbunny.mp4`,timeInSeconds: 10000001,durationInSeconds: 1 / 30,diff --git a/packages/media/src/test/editlist-offset.test.ts b/packages/media/src/test/editlist-offset.test.tsindex db74c99bea..d0a9bf3a21 100644--- a/packages/media/src/test/editlist-offset.test.ts+++ b/packages/media/src/test/editlist-offset.test.ts@@ -5,6 +5,7 @@ import {getMaxVideoCacheSize, globalMediaCache} from '../caches';test('Audio extraction should be correct if there is edit list offset', async () => {// Time: 0.00sec, should return nullconst audio1 = await extractAudio({+ sampleRate: 48000,src: '/audio-with-64msoffset-editlist.mp4',audioStreamIndex: 0,durationInSeconds: 1 / 30,@@ -27,6 +28,7 @@ test('Audio extraction should be correct if there is edit list offset', async ()// Time: 0.033sec, should return audioconst audio2 = await extractAudio({+ sampleRate: 48000,src: '/audio-with-64msoffset-editlist.mp4',audioStreamIndex: 0,durationInSeconds: 1 / 30,@@ -52,6 +54,7 @@ test('Audio extraction should be correct if there is edit list offset', async ()// Time: 0.066sec, should return audioconst audio3 = await extractAudio({+ sampleRate: 48000,src: '/audio-with-64msoffset-editlist.mp4',audioStreamIndex: 0,durationInSeconds: 1 / 30,diff --git a/packages/media/src/test/extract-accuracy-uneven.test.ts b/packages/media/src/test/extract-accuracy-uneven.test.tsindex fd1e1284bb..c08a22d12e 100644--- a/packages/media/src/test/extract-accuracy-uneven.test.ts+++ b/packages/media/src/test/extract-accuracy-uneven.test.ts@@ -10,6 +10,7 @@ test('Extract accuracy over 100 frames with playback rate 1.75', async () => {for (let i = 0; i < NUM_FRAMES; i++) {const timeInSeconds = i / FPS;const audio = await extractAudio({+ sampleRate: 48000,audioStreamIndex: 0,timeInSeconds,durationInSeconds: 1 / FPS,diff --git a/packages/media/src/test/extract-accuracy.test.ts b/packages/media/src/test/extract-accuracy.test.tsindex a75960ecac..1f29811030 100644--- a/packages/media/src/test/extract-accuracy.test.ts+++ b/packages/media/src/test/extract-accuracy.test.ts@@ -10,6 +10,7 @@ test('Extract accuracy over 100 frames with playback rate 2', async () => {for (let i = 0; i < NUM_FRAMES; i++) {const timeInSeconds = i / FPS;const audio = await extractAudio({+ sampleRate: 48000,audioStreamIndex: 0,timeInSeconds,durationInSeconds: 1 / FPS,diff --git a/packages/media/src/test/extract-audio-resampling.test.ts b/packages/media/src/test/extract-audio-resampling.test.tsindex 4a83fa35da..b2a4abe251 100644--- a/packages/media/src/test/extract-audio-resampling.test.ts+++ b/packages/media/src/test/extract-audio-resampling.test.ts@@ -20,6 +20,7 @@ test('resamples Mediabunny AudioSamples as one continuous frame', async () => {const source = new Int16Array(wav, dataOffset + 8);const extracted = await extractAudio({+ sampleRate: 48000,audioStreamIndex: 0,timeInSeconds: 0,durationInSeconds: 1 / FPS,diff --git a/packages/media/src/test/pitch-shift-preview.test.tsx b/packages/media/src/test/pitch-shift-preview.test.tsxindex 7e5ac9f1ff..89fc579702 100644--- a/packages/media/src/test/pitch-shift-preview.test.tsx+++ b/packages/media/src/test/pitch-shift-preview.test.tsx@@ -100,7 +100,10 @@ test('Audio and Video update toneFrequency during preview', async () => {return node;});- const verify = async (component: React.ComponentType<ToneFrequencyProps>) => {+ const verify = async (+ component: React.ComponentType<ToneFrequencyProps>,+ sampleRate: number,+ ) => {createdNodes.length = 0;const container = document.createElement('div');document.body.appendChild(container);@@ -121,6 +124,7 @@ test('Audio and Video update toneFrequency during preview', async () => {compositionWidth={100}durationInFrames={30}fps={30}+ sampleRate={sampleRate}inputProps={{}}/>,);@@ -130,6 +134,10 @@ test('Audio and Video update toneFrequency during preview', async () => {() => createdNodes.filter((node) => node.buffer !== null).length >= 20,);const getScheduledFrequency = () => {+ for (const node of createdNodes) {+ expect(node.context.sampleRate).toBe(sampleRate);+ }+const buffers = createdNodes.map((node) => node.buffer).filter((buffer): buffer is AudioBuffer => buffer !== null);@@ -184,8 +192,10 @@ test('Audio and Video update toneFrequency during preview', async () => {);try {- await verify(AudioComposition);- await verify(VideoComposition);+ for (const sampleRate of [48000, 44100]) {+ await verify(AudioComposition, sampleRate);+ await verify(VideoComposition, sampleRate);+ }} finally {createBufferSourceSpy.mockRestore();URL.revokeObjectURL(src);diff --git a/packages/media/src/test/resample.test.ts b/packages/media/src/test/resample.test.tsindex fa15540985..8a75d9c621 100644--- a/packages/media/src/test/resample.test.ts+++ b/packages/media/src/test/resample.test.ts@@ -12,6 +12,7 @@ test('Should be able to convert audio that is on the verge', () => {});const spedUp = convertAudioData({+ targetSampleRate: 48000,audioData: sine,trimStartInSeconds: 0.041666666666666664,trimEndInSeconds: 0,@@ -45,6 +46,7 @@ test('convert with playbackrate', () => {});const spedUp = convertAudioData({+ targetSampleRate: 48000,audioData: sine,trimStartInSeconds: 0,trimEndInSeconds: 0,diff --git a/packages/media/src/video-extraction/add-broadcast-channel-listener.ts b/packages/media/src/video-extraction/add-broadcast-channel-listener.tsindex 8a30c362e8..8578809181 100644--- a/packages/media/src/video-extraction/add-broadcast-channel-listener.ts+++ b/packages/media/src/video-extraction/add-broadcast-channel-listener.ts@@ -46,6 +46,7 @@ export type MessageFromMainTab =export type ExtractFrameRequest = {type: 'request';+ sampleRate: number;src: string;timeInSeconds: number;durationInSeconds: number;@@ -98,6 +99,7 @@ export const addBroadcastChannelListener = () => {if (data.type === 'request') {try {const result = await extractFrameAndAudio({+ sampleRate: data.sampleRate,src: data.src,timeInSeconds: data.timeInSeconds,logLevel: data.logLevel,diff --git a/packages/media/src/video-extraction/extract-frame-via-broadcast-channel.ts b/packages/media/src/video-extraction/extract-frame-via-broadcast-channel.tsindex ca7087bee8..46642327fb 100644--- a/packages/media/src/video-extraction/extract-frame-via-broadcast-channel.ts+++ b/packages/media/src/video-extraction/extract-frame-via-broadcast-channel.ts@@ -31,6 +31,7 @@ export type ExtractFrameViaBroadcastChannelResult =addBroadcastChannelListener();export const extractFrameViaBroadcastChannel = async ({+ sampleRate,src,timeInSeconds,logLevel,@@ -49,6 +50,7 @@ export const extractFrameViaBroadcastChannel = async ({requestInit,mediaCache,}: {+ sampleRate: number;src: string;timeInSeconds: number;durationInSeconds: number;@@ -69,6 +71,7 @@ export const extractFrameViaBroadcastChannel = async ({}): Promise<ExtractFrameViaBroadcastChannelResult> => {if (isClientSideRendering || window.remotion_isMainTab) {return extractFrameAndAudio({+ sampleRate,logLevel,src,timeInSeconds,@@ -209,6 +212,7 @@ export const extractFrameViaBroadcastChannel = async ({const request: ExtractFrameRequest = {type: 'request',+ sampleRate,src,timeInSeconds,id: requestId,diff --git a/packages/media/src/video/video-for-rendering.tsx b/packages/media/src/video/video-for-rendering.tsxindex 03f80ff27b..5530f3ccbf 100644--- a/packages/media/src/video/video-for-rendering.tsx+++ b/packages/media/src/video/video-for-rendering.tsx@@ -23,7 +23,6 @@ import {} from 'remotion';import {useMaxMediaCacheSize, useRenderMediaCache} from '../caches';import {applyVolume} from '../convert-audiodata/apply-volume';-import {getTargetSampleRate} from '../convert-audiodata/resample-audiodata';import {frameForVolumeProp} from '../looped-frame';import {type MediaOnError, callOnErrorAndResolve} from '../on-error';import {ProResDecoderNotEnabledError} from '../prores-error';@@ -100,6 +99,8 @@ export const VideoForRendering: React.FC<InnerVideoProps> = ({throw new TypeError('No `src` was passed to <Video>.');}+ const audioContext = useContext(Internals.SharedAudioContext);+ const sampleRate = audioContext?.sampleRate ?? 48000;const frame = useCurrentFrame();const absoluteFrame = Internals.useTimelinePosition();@@ -191,6 +192,7 @@ export const VideoForRendering: React.FC<InnerVideoProps> = ({})();extractFrameViaBroadcastChannel({+ sampleRate,src,timeInSeconds: timestamp,durationInSeconds,@@ -398,8 +400,7 @@ export const VideoForRendering: React.FC<InnerVideoProps> = ({frame: absoluteFrame,startInVideo,timestamp: audio.timestamp,- duration:- (audio.numberOfFrames / getTargetSampleRate()) * 1_000_000,+ duration: audio.durationInMicroSeconds,toneFrequency,});}@@ -419,6 +420,7 @@ export const VideoForRendering: React.FC<InnerVideoProps> = ({unregisterRenderAsset(id);};}, [+ sampleRate,absoluteFrame,continueRender,delayRender,diff --git a/packages/studio/src/renderEntry.tsx b/packages/studio/src/renderEntry.tsxindex 5b83da6fd2..b5fc821597 100644--- a/packages/studio/src/renderEntry.tsx+++ b/packages/studio/src/renderEntry.tsx@@ -251,7 +251,7 @@ const renderContent = (Root: React.FC) => {logLevel={window.remotion_logLevel ?? 'info'}numberOfAudioTags={0}audioLatencyHint={window.remotion_audioLatencyHint ?? 'playback'}- previewSampleRate={window.remotion_previewSampleRate}+ previewSampleRate={window.remotion_sampleRate}_experimentalKeepAudioContextAlive={false}><Internals.RenderAssetManagerProvider collectAssets={null}>@@ -280,7 +280,7 @@ const renderContent = (Root: React.FC) => {logLevel={window.remotion_logLevel ?? 'info'}numberOfAudioTags={0}audioLatencyHint={window.remotion_audioLatencyHint ?? 'playback'}- previewSampleRate={window.remotion_previewSampleRate}+ previewSampleRate={window.remotion_sampleRate}_experimentalKeepAudioContextAlive={false}><Internals.RenderAssetManagerProvider collectAssets={null}>diff --git a/packages/web-renderer/src/render-media-on-web.tsx b/packages/web-renderer/src/render-media-on-web.tsxindex dffedc9d89..df547280df 100644--- a/packages/web-renderer/src/render-media-on-web.tsx+++ b/packages/web-renderer/src/render-media-on-web.tsx@@ -324,15 +324,6 @@ const internalRenderMediaOnWeb = async <return Promise.reject(new Error('renderMediaOnWeb() was cancelled'));}- // Media extraction reads this value before the audio context effect runs.- const previousSampleRate = window.remotion_sampleRate;- using _sampleRateScope = {- [Symbol.dispose]: () => {- window.remotion_sampleRate = previousSampleRate;- },- };- window.remotion_sampleRate = sampleRate;-using scaffold = createScaffold({width: resolved.width,height: resolved.height,diff --git a/packages/web-renderer/src/test/pitch-shift-audio.test.tsx b/packages/web-renderer/src/test/pitch-shift-audio.test.tsxindex 62d74cebf6..4062b59e67 100644--- a/packages/web-renderer/src/test/pitch-shift-audio.test.tsx+++ b/packages/web-renderer/src/test/pitch-shift-audio.test.tsx@@ -1,5 +1,8 @@import {Audio, Video} from '@remotion/media';+import {Player} from '@remotion/player';import {ALL_FORMATS, AudioBufferSink, BlobSource, Input} from 'mediabunny';+import {flushSync} from 'react-dom';+import {createRoot} from 'react-dom/client';import {useCurrentFrame} from 'remotion';import {expect, test} from 'vitest';import {renderMediaOnWeb} from '../render-media-on-web';@@ -28,15 +31,47 @@ const amplitudeAt = (};test.each([48000, 44100])(- 'renders pitch-shifted stereo audio at %i Hz with preserved timing',+ 'keeps %i Hz export audio correct when another Player mounts',async (sampleRate) => {+ const container = document.createElement('div');+ document.body.appendChild(container);+ const root = createRoot(container);+ using _preview = {+ [Symbol.dispose]: () => {+ root.unmount();+ container.remove();+ },+ };+ let mountedPreview = false;const result = await renderMediaOnWeb({licenseKey: 'free-license',composition: {...pitchShiftAudio, calculateMetadata: null},container: 'wav',outputTarget: 'arraybuffer',sampleRate,+ onProgress: ({renderedFrames}) => {+ if (mountedPreview || renderedFrames === 0) {+ return;+ }++ mountedPreview = true;+ // Mount another root during export, using a different sample rate.+ flushSync(() => {+ root.render(+ <Player+ acknowledgeRemotionLicense+ component={() => null}+ compositionHeight={100}+ compositionWidth={100}+ durationInFrames={60}+ fps={30}+ sampleRate={sampleRate === 48000 ? 44100 : 48000}+ />,+ );+ });+ },});+ expect(mountedPreview).toBe(true);using input = new Input({formats: ALL_FORMATS,source: new BlobSource(await result.getBlob()),