Skip to content

🎥 Make videos programmatically with React

Fix: honor preservePitch during server-side HTML5/Offthread rendering

Xavier Salaicese278de02f78b
diff --git a/packages/core/src/CompositionManager.tsx b/packages/core/src/CompositionManager.tsx
index ac7361ddf7..65564378c1 100644
--- a/packages/core/src/CompositionManager.tsx
+++ b/packages/core/src/CompositionManager.tsx
@@ -158,6 +158,7 @@ export type AudioOrVideoAsset = {
toneFrequency: number;
audioStartFrame: number;
audioStreamIndex: number;
+ preservePitch: boolean;
};
export type InlineAudioAsset = {
diff --git a/packages/core/src/audio/AudioForRendering.tsx b/packages/core/src/audio/AudioForRendering.tsx
index 7e086ebf97..5651340c6b 100644
--- a/packages/core/src/audio/AudioForRendering.tsx
+++ b/packages/core/src/audio/AudioForRendering.tsx
@@ -52,7 +52,7 @@ const AudioForRenderingRefForwardingFunction: React.ForwardRefRenderFunction<
loopVolumeCurveBehavior,
pauseWhenBuffering,
audioStreamIndex,
- preservePitch: _preservePitch,
+ preservePitch,
...nativeProps
} = props;
@@ -123,6 +123,7 @@ const AudioForRenderingRefForwardingFunction: React.ForwardRefRenderFunction<
toneFrequency: toneFrequency ?? 1,
audioStartFrame,
audioStreamIndex: audioStreamIndex ?? 0,
+ preservePitch: preservePitch ?? true,
});
return () => unregisterRenderAsset(id);
}, [
@@ -141,6 +142,7 @@ const AudioForRenderingRefForwardingFunction: React.ForwardRefRenderFunction<
audioStartFrame,
sequencePlaybackRate,
audioStreamIndex,
+ preservePitch,
]);
const {src} = props;
diff --git a/packages/core/src/video/OffthreadVideoForRendering.tsx b/packages/core/src/video/OffthreadVideoForRendering.tsx
index 61a00c9f37..13798cc9ac 100644
--- a/packages/core/src/video/OffthreadVideoForRendering.tsx
+++ b/packages/core/src/video/OffthreadVideoForRendering.tsx
@@ -55,7 +55,7 @@ export const OffthreadVideoForRendering: React.FC<AllOffthreadVideoProps> = ({
// https://discord.com/channels/809501355504959528/844143007183667220/1311639632496033813
crossOrigin,
audioStreamIndex,
- preservePitch: _preservePitch,
+ preservePitch,
...props
}) => {
const absoluteFrame = useTimelinePosition();
@@ -129,6 +129,7 @@ export const OffthreadVideoForRendering: React.FC<AllOffthreadVideoProps> = ({
toneFrequency,
audioStartFrame,
audioStreamIndex,
+ preservePitch: preservePitch ?? true,
});
return () => unregisterRenderAsset(id);
@@ -146,6 +147,7 @@ export const OffthreadVideoForRendering: React.FC<AllOffthreadVideoProps> = ({
audioStartFrame,
sequencePlaybackRate,
audioStreamIndex,
+ preservePitch,
]);
const currentTime = useMemo(() => {
diff --git a/packages/core/src/video/VideoForRendering.tsx b/packages/core/src/video/VideoForRendering.tsx
index 8a3841629f..525c634004 100644
--- a/packages/core/src/video/VideoForRendering.tsx
+++ b/packages/core/src/video/VideoForRendering.tsx
@@ -56,7 +56,7 @@ const VideoForRenderingForwardFunction: React.ForwardRefRenderFunction<
loopVolumeCurveBehavior,
audioStreamIndex,
onVideoFrame,
- preservePitch: _preservePitch,
+ preservePitch,
...props
},
ref,
@@ -136,6 +136,7 @@ const VideoForRenderingForwardFunction: React.ForwardRefRenderFunction<
toneFrequency: toneFrequency ?? 1,
audioStartFrame,
audioStreamIndex: audioStreamIndex ?? 0,
+ preservePitch: preservePitch ?? true,
});
return () => unregisterRenderAsset(id);
@@ -153,6 +154,7 @@ const VideoForRenderingForwardFunction: React.ForwardRefRenderFunction<
audioStartFrame,
sequencePlaybackRate,
audioStreamIndex,
+ preservePitch,
]);
useImperativeHandle(ref, () => {
diff --git a/packages/docs/docs/html5-audio.mdx b/packages/docs/docs/html5-audio.mdx
index f605ef2261..98316b8808 100644
--- a/packages/docs/docs/html5-audio.mdx
+++ b/packages/docs/docs/html5-audio.mdx
@@ -138,7 +138,7 @@ Controls whether the audio pitch is preserved when [`playbackRate`](#playbackrat
Accepts `true`, `false`, or `undefined`. Default: `true`, matching the browser default for [`HTMLMediaElement.preservesPitch`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/preservesPitch).
-`preservePitch` only affects preview playback. During server-side rendering, [`playbackRate`](#playbackrate) always preserves the audio pitch, regardless of this prop. Use [`toneFrequency`](/docs/audio/pitch) to change the pitch during server-side rendering.
+`preservePitch` affects both preview playback and server-side rendering: setting it to `false` allows the pitch to shift (instead of staying constant) when [`playbackRate`](#playbackrate) is not `1`. Use [`toneFrequency`](/docs/audio/pitch) to change the pitch independently of `playbackRate`.
### `muted?`<AvailableFrom v="2.0.0"/>
diff --git a/packages/docs/docs/html5-video.mdx b/packages/docs/docs/html5-video.mdx
index f68cce7bf1..42fb02c5d3 100644
--- a/packages/docs/docs/html5-video.mdx
+++ b/packages/docs/docs/html5-video.mdx
@@ -193,7 +193,7 @@ Controls whether the audio pitch is preserved when [`playbackRate`](#playbackrat
Accepts `true`, `false`, or `undefined`. Default: `true`, matching the browser default for [`HTMLMediaElement.preservesPitch`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/preservesPitch).
-`preservePitch` only affects preview playback. During server-side rendering, [`playbackRate`](#playbackrate) always preserves the audio pitch, regardless of this prop. Use [`toneFrequency`](/docs/audio/pitch) to change the pitch during server-side rendering.
+`preservePitch` affects both preview playback and server-side rendering: setting it to `false` allows the pitch to shift (instead of staying constant) when [`playbackRate`](#playbackrate) is not `1`. Use [`toneFrequency`](/docs/audio/pitch) to change the pitch independently of `playbackRate`.
### `muted?`
diff --git a/packages/docs/docs/offthreadvideo.mdx b/packages/docs/docs/offthreadvideo.mdx
index 4083c651ed..4481a1ef2a 100644
--- a/packages/docs/docs/offthreadvideo.mdx
+++ b/packages/docs/docs/offthreadvideo.mdx
@@ -207,7 +207,7 @@ Controls whether the audio pitch is preserved when [`playbackRate`](#playbackrat
Accepts `true`, `false`, or `undefined`. Default: `true`, matching the browser default for [`HTMLMediaElement.preservesPitch`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/preservesPitch).
-`preservePitch` only affects preview playback. During server-side rendering, [`playbackRate`](#playbackrate) always preserves the audio pitch, regardless of this prop. Use [`toneFrequency`](/docs/audio/pitch) to change the pitch during server-side rendering.
+`preservePitch` affects both preview playback and server-side rendering: setting it to `false` allows the pitch to shift (instead of staying constant) when [`playbackRate`](#playbackrate) is not `1`. Use [`toneFrequency`](/docs/audio/pitch) to change the pitch independently of `playbackRate`.
### `muted?`
diff --git a/packages/renderer/src/assets/calculate-asset-positions.ts b/packages/renderer/src/assets/calculate-asset-positions.ts
index 970e80ef23..2b0addf03e 100644
--- a/packages/renderer/src/assets/calculate-asset-positions.ts
+++ b/packages/renderer/src/assets/calculate-asset-positions.ts
@@ -107,6 +107,7 @@ export const calculateAssetPositions = (
toneFrequency: asset.toneFrequency,
audioStartFrame: asset.audioStartFrame,
audioStreamIndex: asset.audioStreamIndex,
+ preservePitch: asset.preservePitch,
};
assets.push(openAsset);
openAssets.set(asset.id, openAsset);
diff --git a/packages/renderer/src/stringify-ffmpeg-filter.ts b/packages/renderer/src/stringify-ffmpeg-filter.ts
index 10aae8c711..94b1f0a835 100644
--- a/packages/renderer/src/stringify-ffmpeg-filter.ts
+++ b/packages/renderer/src/stringify-ffmpeg-filter.ts
@@ -41,6 +41,30 @@ const stringifyTrim = (trim: number) => {
return asString;
};
+// When `preservePitch` is disabled, the speed change must not go through
+// `atempo` (which keeps the pitch). Changing the sample rate and then
+// resampling back to the target rate speeds up or slows down playback
+// while letting the pitch shift along with it (tape / chipmunk effect).
+const calculateSpeedChangeFilter = ({
+ playbackRate,
+ preservePitch,
+ sampleRate,
+}: {
+ playbackRate: number;
+ preservePitch: boolean;
+ sampleRate: number;
+}): string | null => {
+ if (playbackRate === 1) {
+ return null;
+ }
+
+ if (preservePitch) {
+ return calculateATempo(playbackRate);
+ }
+
+ return `asetrate=${sampleRate}*${playbackRate},aresample=${sampleRate}`;
+};
+
const trimAndSetTempo = ({
assetDuration,
asset,
@@ -49,6 +73,7 @@ const trimAndSetTempo = ({
fps,
indent,
logLevel,
+ sampleRate,
}: {
assetDuration: number | null;
trimLeftOffset: number;
@@ -57,6 +82,7 @@ const trimAndSetTempo = ({
fps: number;
indent: boolean;
logLevel: LogLevel;
+ sampleRate: number;
}): {
actualTrimLeft: number;
filter: (string | null)[];
@@ -98,7 +124,11 @@ const trimAndSetTempo = ({
return {
filter: [
- calculateATempo(asset.playbackRate),
+ calculateSpeedChangeFilter({
+ playbackRate: asset.playbackRate,
+ preservePitch: asset.preservePitch,
+ sampleRate,
+ }),
`atrim=${stringifyTrim(trimLeft)}:${stringifyTrim(trimRightOrAssetDuration)}`,
],
actualTrimLeft: trimLeft,
@@ -175,6 +205,7 @@ export const stringifyFfmpegFilter = ({
fps,
indent,
logLevel,
+ sampleRate,
});
const volumeFilter = ffmpegVolumeExpression({
diff --git a/packages/renderer/src/test/asset-calculation.test.tsx b/packages/renderer/src/test/asset-calculation.test.tsx
index 8ae1b04544..781ebead79 100644
--- a/packages/renderer/src/test/asset-calculation.test.tsx
+++ b/packages/renderer/src/test/asset-calculation.test.tsx
@@ -136,6 +136,7 @@ test('Should calculate asset positions for a complete media workflow', async ()
volume: 1,
playbackRate: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -150,6 +151,7 @@ test('Should calculate asset positions for a complete media workflow', async ()
volume: 1,
playbackRate: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -164,6 +166,7 @@ test('Should calculate asset positions for a complete media workflow', async ()
volume: 1,
playbackRate: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -176,6 +179,7 @@ test('Should calculate asset positions for a complete media workflow', async ()
volume: 1,
playbackRate: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -190,6 +194,7 @@ test('Should calculate asset positions for a complete media workflow', async ()
volume: 1,
playbackRate: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -209,6 +214,7 @@ test('Should calculate asset positions for a complete media workflow', async ()
)
.filter((f) => f > 0),
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -228,6 +234,7 @@ test('Should calculate asset positions for a complete media workflow', async ()
}),
),
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 100,
audioStreamIndex: 0,
},
@@ -246,6 +253,7 @@ test('Should calculate asset positions for a complete media workflow', async ()
}),
),
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
} as const;
@@ -271,6 +279,7 @@ test('Should calculate asset positions for a complete media workflow', async ()
}),
),
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
diff --git a/packages/renderer/src/test/asset-compression.test.ts b/packages/renderer/src/test/asset-compression.test.ts
index 7623ad9907..04adc348a8 100644
--- a/packages/renderer/src/test/asset-compression.test.ts
+++ b/packages/renderer/src/test/asset-compression.test.ts
@@ -18,6 +18,7 @@ test('Should compress and uncompress assets', () => {
type: 'video' as const,
volume: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -32,6 +33,7 @@ test('Should compress and uncompress assets', () => {
type: 'video' as const,
volume: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -59,6 +61,7 @@ test('Should compress and uncompress assets', () => {
type: 'video',
volume: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -78,6 +81,7 @@ test('Should uncompress correctly when multiple assets share the same id and fra
type: 'video' as const,
volume: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -90,6 +94,7 @@ test('Should uncompress correctly when multiple assets share the same id and fra
type: 'video' as const,
volume: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -102,6 +107,7 @@ test('Should uncompress correctly when multiple assets share the same id and fra
type: 'video' as const,
volume: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
diff --git a/packages/renderer/src/test/calculate-asset-positions.test.ts b/packages/renderer/src/test/calculate-asset-positions.test.ts
index 83af2e32b5..f89b9087b0 100644
--- a/packages/renderer/src/test/calculate-asset-positions.test.ts
+++ b/packages/renderer/src/test/calculate-asset-positions.test.ts
@@ -25,6 +25,7 @@ const makeAsset = ({
playbackRate: 1,
mediaFrame: mediaFrame ?? frame,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
};
@@ -61,6 +62,7 @@ test('An asset that re-appears after a gap gets two positions', () => {
volume: [0, 1, 2],
playbackRate: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -74,6 +76,7 @@ test('An asset that re-appears after a gap gets two positions', () => {
volume: [5, 6, 7],
playbackRate: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
diff --git a/packages/renderer/src/test/dont-skip-assets.test.ts b/packages/renderer/src/test/dont-skip-assets.test.ts
index 5d326a2914..b0c849d270 100644
--- a/packages/renderer/src/test/dont-skip-assets.test.ts
+++ b/packages/renderer/src/test/dont-skip-assets.test.ts
@@ -24,6 +24,7 @@ const mock: TRenderAsset[][] = new Array(2934)
playbackRate: 1,
mediaFrame: 0,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
}
@@ -37,6 +38,7 @@ const mock: TRenderAsset[][] = new Array(2934)
playbackRate: 1,
mediaFrame: k,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -60,6 +62,7 @@ test('Dont skip assets', () => {
volume: 1,
playbackRate: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
@@ -73,6 +76,7 @@ test('Dont skip assets', () => {
volume: 1,
playbackRate: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},
diff --git a/packages/renderer/src/test/ffmpeg-filters.test.ts b/packages/renderer/src/test/ffmpeg-filters.test.ts
index 949bea181a..2208cd6c22 100644
--- a/packages/renderer/src/test/ffmpeg-filters.test.ts
+++ b/packages/renderer/src/test/ffmpeg-filters.test.ts
@@ -17,6 +17,7 @@ const baseAsset: MediaAsset = {
id: '1',
playbackRate: 1,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
};
@@ -62,6 +63,37 @@ test('Should create a basic filter correctly', () => {
pad_start: null,
});
});
+
+test('preservePitch: false uses asetrate/aresample instead of atempo', () => {
+ expect(
+ stringifyFfmpegFilter({
+ fps: 30,
+ asset: {
+ ...baseAsset,
+ duration: 200,
+ playbackRate: 2,
+ preservePitch: false,
+ },
+ channels: 1,
+ assetDuration: 10,
+ chunkLengthInSeconds: 3.3333,
+ trimLeftOffset: 0,
+ trimRightOffset: 0,
+ forSeamlessAacConcatenation: false,
+ volume: flattenVolumeArray(baseAsset.volume),
+ indent: false,
+ logLevel: 'info',
+ presentationTimeOffsetInSeconds: 0,
+ sampleRate: 48000,
+ }),
+ ).toEqual({
+ actualTrimLeft: 0,
+ filter:
+ '[0:a]aformat=sample_fmts=s16:sample_rates=48000,asetrate=48000*2,aresample=48000,atrim=0us:5000000us[a0]',
+ pad_end: null,
+ pad_start: null,
+ });
+});
test('Trim the end', () => {
const {
chunkLengthInSeconds,
@@ -338,6 +370,7 @@ test('Should calculate pad correctly with a lot of playbackRate', () => {
startInVideo: 0,
playbackRate: 16,
toneFrequency: 1,
+ preservePitch: true,
audioStartFrame: 0,
audioStreamIndex: 0,
},