🎥 Make videos programmatically with React
Fix: honor preservePitch during server-side HTML5/Offthread rendering
Xavier Salaicese278de02f78b
diff --git a/packages/core/src/CompositionManager.tsx b/packages/core/src/CompositionManager.tsxindex ac7361ddf7..65564378c1 100644--- a/packages/core/src/CompositionManager.tsx+++ b/packages/core/src/CompositionManager.tsx@@ -158,6 +158,7 @@ export type AudioOrVideoAsset = {toneFrequency: number;audioStartFrame: number;audioStreamIndex: number;+ preservePitch: boolean;};export type InlineAudioAsset = {diff --git a/packages/core/src/audio/AudioForRendering.tsx b/packages/core/src/audio/AudioForRendering.tsxindex 7e086ebf97..5651340c6b 100644--- a/packages/core/src/audio/AudioForRendering.tsx+++ b/packages/core/src/audio/AudioForRendering.tsx@@ -52,7 +52,7 @@ const AudioForRenderingRefForwardingFunction: React.ForwardRefRenderFunction<loopVolumeCurveBehavior,pauseWhenBuffering,audioStreamIndex,- preservePitch: _preservePitch,+ preservePitch,...nativeProps} = props;@@ -123,6 +123,7 @@ const AudioForRenderingRefForwardingFunction: React.ForwardRefRenderFunction<toneFrequency: toneFrequency ?? 1,audioStartFrame,audioStreamIndex: audioStreamIndex ?? 0,+ preservePitch: preservePitch ?? true,});return () => unregisterRenderAsset(id);}, [@@ -141,6 +142,7 @@ const AudioForRenderingRefForwardingFunction: React.ForwardRefRenderFunction<audioStartFrame,sequencePlaybackRate,audioStreamIndex,+ preservePitch,]);const {src} = props;diff --git a/packages/core/src/video/OffthreadVideoForRendering.tsx b/packages/core/src/video/OffthreadVideoForRendering.tsxindex 61a00c9f37..13798cc9ac 100644--- a/packages/core/src/video/OffthreadVideoForRendering.tsx+++ b/packages/core/src/video/OffthreadVideoForRendering.tsx@@ -55,7 +55,7 @@ export const OffthreadVideoForRendering: React.FC<AllOffthreadVideoProps> = ({// https://discord.com/channels/809501355504959528/844143007183667220/1311639632496033813crossOrigin,audioStreamIndex,- preservePitch: _preservePitch,+ preservePitch,...props}) => {const absoluteFrame = useTimelinePosition();@@ -129,6 +129,7 @@ export const OffthreadVideoForRendering: React.FC<AllOffthreadVideoProps> = ({toneFrequency,audioStartFrame,audioStreamIndex,+ preservePitch: preservePitch ?? true,});return () => unregisterRenderAsset(id);@@ -146,6 +147,7 @@ export const OffthreadVideoForRendering: React.FC<AllOffthreadVideoProps> = ({audioStartFrame,sequencePlaybackRate,audioStreamIndex,+ preservePitch,]);const currentTime = useMemo(() => {diff --git a/packages/core/src/video/VideoForRendering.tsx b/packages/core/src/video/VideoForRendering.tsxindex 8a3841629f..525c634004 100644--- a/packages/core/src/video/VideoForRendering.tsx+++ b/packages/core/src/video/VideoForRendering.tsx@@ -56,7 +56,7 @@ const VideoForRenderingForwardFunction: React.ForwardRefRenderFunction<loopVolumeCurveBehavior,audioStreamIndex,onVideoFrame,- preservePitch: _preservePitch,+ preservePitch,...props},ref,@@ -136,6 +136,7 @@ const VideoForRenderingForwardFunction: React.ForwardRefRenderFunction<toneFrequency: toneFrequency ?? 1,audioStartFrame,audioStreamIndex: audioStreamIndex ?? 0,+ preservePitch: preservePitch ?? true,});return () => unregisterRenderAsset(id);@@ -153,6 +154,7 @@ const VideoForRenderingForwardFunction: React.ForwardRefRenderFunction<audioStartFrame,sequencePlaybackRate,audioStreamIndex,+ preservePitch,]);useImperativeHandle(ref, () => {diff --git a/packages/docs/docs/html5-audio.mdx b/packages/docs/docs/html5-audio.mdxindex f605ef2261..98316b8808 100644--- a/packages/docs/docs/html5-audio.mdx+++ b/packages/docs/docs/html5-audio.mdx@@ -138,7 +138,7 @@ Controls whether the audio pitch is preserved when [`playbackRate`](#playbackratAccepts `true`, `false`, or `undefined`. Default: `true`, matching the browser default for [`HTMLMediaElement.preservesPitch`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/preservesPitch).-`preservePitch` only affects preview playback. During server-side rendering, [`playbackRate`](#playbackrate) always preserves the audio pitch, regardless of this prop. Use [`toneFrequency`](/docs/audio/pitch) to change the pitch during server-side rendering.+`preservePitch` affects both preview playback and server-side rendering: setting it to `false` allows the pitch to shift (instead of staying constant) when [`playbackRate`](#playbackrate) is not `1`. Use [`toneFrequency`](/docs/audio/pitch) to change the pitch independently of `playbackRate`.### `muted?`<AvailableFrom v="2.0.0"/>diff --git a/packages/docs/docs/html5-video.mdx b/packages/docs/docs/html5-video.mdxindex f68cce7bf1..42fb02c5d3 100644--- a/packages/docs/docs/html5-video.mdx+++ b/packages/docs/docs/html5-video.mdx@@ -193,7 +193,7 @@ Controls whether the audio pitch is preserved when [`playbackRate`](#playbackratAccepts `true`, `false`, or `undefined`. Default: `true`, matching the browser default for [`HTMLMediaElement.preservesPitch`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/preservesPitch).-`preservePitch` only affects preview playback. During server-side rendering, [`playbackRate`](#playbackrate) always preserves the audio pitch, regardless of this prop. Use [`toneFrequency`](/docs/audio/pitch) to change the pitch during server-side rendering.+`preservePitch` affects both preview playback and server-side rendering: setting it to `false` allows the pitch to shift (instead of staying constant) when [`playbackRate`](#playbackrate) is not `1`. Use [`toneFrequency`](/docs/audio/pitch) to change the pitch independently of `playbackRate`.### `muted?`diff --git a/packages/docs/docs/offthreadvideo.mdx b/packages/docs/docs/offthreadvideo.mdxindex 4083c651ed..4481a1ef2a 100644--- a/packages/docs/docs/offthreadvideo.mdx+++ b/packages/docs/docs/offthreadvideo.mdx@@ -207,7 +207,7 @@ Controls whether the audio pitch is preserved when [`playbackRate`](#playbackratAccepts `true`, `false`, or `undefined`. Default: `true`, matching the browser default for [`HTMLMediaElement.preservesPitch`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/preservesPitch).-`preservePitch` only affects preview playback. During server-side rendering, [`playbackRate`](#playbackrate) always preserves the audio pitch, regardless of this prop. Use [`toneFrequency`](/docs/audio/pitch) to change the pitch during server-side rendering.+`preservePitch` affects both preview playback and server-side rendering: setting it to `false` allows the pitch to shift (instead of staying constant) when [`playbackRate`](#playbackrate) is not `1`. Use [`toneFrequency`](/docs/audio/pitch) to change the pitch independently of `playbackRate`.### `muted?`diff --git a/packages/renderer/src/assets/calculate-asset-positions.ts b/packages/renderer/src/assets/calculate-asset-positions.tsindex 970e80ef23..2b0addf03e 100644--- a/packages/renderer/src/assets/calculate-asset-positions.ts+++ b/packages/renderer/src/assets/calculate-asset-positions.ts@@ -107,6 +107,7 @@ export const calculateAssetPositions = (toneFrequency: asset.toneFrequency,audioStartFrame: asset.audioStartFrame,audioStreamIndex: asset.audioStreamIndex,+ preservePitch: asset.preservePitch,};assets.push(openAsset);openAssets.set(asset.id, openAsset);diff --git a/packages/renderer/src/stringify-ffmpeg-filter.ts b/packages/renderer/src/stringify-ffmpeg-filter.tsindex 10aae8c711..94b1f0a835 100644--- a/packages/renderer/src/stringify-ffmpeg-filter.ts+++ b/packages/renderer/src/stringify-ffmpeg-filter.ts@@ -41,6 +41,30 @@ const stringifyTrim = (trim: number) => {return asString;};+// When `preservePitch` is disabled, the speed change must not go through+// `atempo` (which keeps the pitch). Changing the sample rate and then+// resampling back to the target rate speeds up or slows down playback+// while letting the pitch shift along with it (tape / chipmunk effect).+const calculateSpeedChangeFilter = ({+ playbackRate,+ preservePitch,+ sampleRate,+}: {+ playbackRate: number;+ preservePitch: boolean;+ sampleRate: number;+}): string | null => {+ if (playbackRate === 1) {+ return null;+ }++ if (preservePitch) {+ return calculateATempo(playbackRate);+ }++ return `asetrate=${sampleRate}*${playbackRate},aresample=${sampleRate}`;+};+const trimAndSetTempo = ({assetDuration,asset,@@ -49,6 +73,7 @@ const trimAndSetTempo = ({fps,indent,logLevel,+ sampleRate,}: {assetDuration: number | null;trimLeftOffset: number;@@ -57,6 +82,7 @@ const trimAndSetTempo = ({fps: number;indent: boolean;logLevel: LogLevel;+ sampleRate: number;}): {actualTrimLeft: number;filter: (string | null)[];@@ -98,7 +124,11 @@ const trimAndSetTempo = ({return {filter: [- calculateATempo(asset.playbackRate),+ calculateSpeedChangeFilter({+ playbackRate: asset.playbackRate,+ preservePitch: asset.preservePitch,+ sampleRate,+ }),`atrim=${stringifyTrim(trimLeft)}:${stringifyTrim(trimRightOrAssetDuration)}`,],actualTrimLeft: trimLeft,@@ -175,6 +205,7 @@ export const stringifyFfmpegFilter = ({fps,indent,logLevel,+ sampleRate,});const volumeFilter = ffmpegVolumeExpression({diff --git a/packages/renderer/src/test/asset-calculation.test.tsx b/packages/renderer/src/test/asset-calculation.test.tsxindex 8ae1b04544..781ebead79 100644--- a/packages/renderer/src/test/asset-calculation.test.tsx+++ b/packages/renderer/src/test/asset-calculation.test.tsx@@ -136,6 +136,7 @@ test('Should calculate asset positions for a complete media workflow', async ()volume: 1,playbackRate: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -150,6 +151,7 @@ test('Should calculate asset positions for a complete media workflow', async ()volume: 1,playbackRate: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -164,6 +166,7 @@ test('Should calculate asset positions for a complete media workflow', async ()volume: 1,playbackRate: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -176,6 +179,7 @@ test('Should calculate asset positions for a complete media workflow', async ()volume: 1,playbackRate: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -190,6 +194,7 @@ test('Should calculate asset positions for a complete media workflow', async ()volume: 1,playbackRate: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -209,6 +214,7 @@ test('Should calculate asset positions for a complete media workflow', async ()).filter((f) => f > 0),toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -228,6 +234,7 @@ test('Should calculate asset positions for a complete media workflow', async ()}),),toneFrequency: 1,+ preservePitch: true,audioStartFrame: 100,audioStreamIndex: 0,},@@ -246,6 +253,7 @@ test('Should calculate asset positions for a complete media workflow', async ()}),),toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,} as const;@@ -271,6 +279,7 @@ test('Should calculate asset positions for a complete media workflow', async ()}),),toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},diff --git a/packages/renderer/src/test/asset-compression.test.ts b/packages/renderer/src/test/asset-compression.test.tsindex 7623ad9907..04adc348a8 100644--- a/packages/renderer/src/test/asset-compression.test.ts+++ b/packages/renderer/src/test/asset-compression.test.ts@@ -18,6 +18,7 @@ test('Should compress and uncompress assets', () => {type: 'video' as const,volume: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -32,6 +33,7 @@ test('Should compress and uncompress assets', () => {type: 'video' as const,volume: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -59,6 +61,7 @@ test('Should compress and uncompress assets', () => {type: 'video',volume: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -78,6 +81,7 @@ test('Should uncompress correctly when multiple assets share the same id and fratype: 'video' as const,volume: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -90,6 +94,7 @@ test('Should uncompress correctly when multiple assets share the same id and fratype: 'video' as const,volume: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -102,6 +107,7 @@ test('Should uncompress correctly when multiple assets share the same id and fratype: 'video' as const,volume: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},diff --git a/packages/renderer/src/test/calculate-asset-positions.test.ts b/packages/renderer/src/test/calculate-asset-positions.test.tsindex 83af2e32b5..f89b9087b0 100644--- a/packages/renderer/src/test/calculate-asset-positions.test.ts+++ b/packages/renderer/src/test/calculate-asset-positions.test.ts@@ -25,6 +25,7 @@ const makeAsset = ({playbackRate: 1,mediaFrame: mediaFrame ?? frame,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,};@@ -61,6 +62,7 @@ test('An asset that re-appears after a gap gets two positions', () => {volume: [0, 1, 2],playbackRate: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -74,6 +76,7 @@ test('An asset that re-appears after a gap gets two positions', () => {volume: [5, 6, 7],playbackRate: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},diff --git a/packages/renderer/src/test/dont-skip-assets.test.ts b/packages/renderer/src/test/dont-skip-assets.test.tsindex 5d326a2914..b0c849d270 100644--- a/packages/renderer/src/test/dont-skip-assets.test.ts+++ b/packages/renderer/src/test/dont-skip-assets.test.ts@@ -24,6 +24,7 @@ const mock: TRenderAsset[][] = new Array(2934)playbackRate: 1,mediaFrame: 0,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,}@@ -37,6 +38,7 @@ const mock: TRenderAsset[][] = new Array(2934)playbackRate: 1,mediaFrame: k,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -60,6 +62,7 @@ test('Dont skip assets', () => {volume: 1,playbackRate: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},@@ -73,6 +76,7 @@ test('Dont skip assets', () => {volume: 1,playbackRate: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},diff --git a/packages/renderer/src/test/ffmpeg-filters.test.ts b/packages/renderer/src/test/ffmpeg-filters.test.tsindex 949bea181a..2208cd6c22 100644--- a/packages/renderer/src/test/ffmpeg-filters.test.ts+++ b/packages/renderer/src/test/ffmpeg-filters.test.ts@@ -17,6 +17,7 @@ const baseAsset: MediaAsset = {id: '1',playbackRate: 1,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,};@@ -62,6 +63,37 @@ test('Should create a basic filter correctly', () => {pad_start: null,});});++test('preservePitch: false uses asetrate/aresample instead of atempo', () => {+ expect(+ stringifyFfmpegFilter({+ fps: 30,+ asset: {+ ...baseAsset,+ duration: 200,+ playbackRate: 2,+ preservePitch: false,+ },+ channels: 1,+ assetDuration: 10,+ chunkLengthInSeconds: 3.3333,+ trimLeftOffset: 0,+ trimRightOffset: 0,+ forSeamlessAacConcatenation: false,+ volume: flattenVolumeArray(baseAsset.volume),+ indent: false,+ logLevel: 'info',+ presentationTimeOffsetInSeconds: 0,+ sampleRate: 48000,+ }),+ ).toEqual({+ actualTrimLeft: 0,+ filter:+ '[0:a]aformat=sample_fmts=s16:sample_rates=48000,asetrate=48000*2,aresample=48000,atrim=0us:5000000us[a0]',+ pad_end: null,+ pad_start: null,+ });+});test('Trim the end', () => {const {chunkLengthInSeconds,@@ -338,6 +370,7 @@ test('Should calculate pad correctly with a lot of playbackRate', () => {startInVideo: 0,playbackRate: 16,toneFrequency: 1,+ preservePitch: true,audioStartFrame: 0,audioStreamIndex: 0,},