🎥 Make videos programmatically with React
`@remotion/media`: Fix audio sample-rate conversion (#10491)
Igor Samokhovets1a982c0bfce4
diff --git a/packages/example/src/AudioTesting/Issue10468.tsx b/packages/example/src/AudioTesting/Issue10468.tsxnew file mode 100644index 0000000000..28f92c3242--- /dev/null+++ b/packages/example/src/AudioTesting/Issue10468.tsx@@ -0,0 +1,32 @@+import {Audio as MediaAudio} from '@remotion/media';+import {ding} from '@remotion/sfx';+import {Html5Audio, Sequence} from 'remotion';++export type Issue10468Props = {+ readonly implementation: 'html5' | 'media';+ readonly src: string;+};++export const issue10468DefaultProps: Issue10468Props = {+ implementation: 'media',+ src: ding,+};++export const issue5758DefaultProps = {src: ding};++export const Issue10468: React.FC<Issue10468Props> = ({+ implementation,+ src,+}) => {+ const Component = implementation === 'html5' ? Html5Audio : MediaAudio;++ return (+ <Sequence from={81} durationInFrames={39}>+ <Component src={src} trimBefore={4} volume={0.3} />+ </Sequence>+ );+};++export const Issue5758: React.FC<{src: string}> = ({src}) => {+ return <MediaAudio src={src} />;+};diff --git a/packages/example/src/Root.tsx b/packages/example/src/Root.tsxindex 360e6e6c09..b4b27e0dd1 100644--- a/packages/example/src/Root.tsx+++ b/packages/example/src/Root.tsx@@ -209,6 +209,12 @@ import {AudioSmoothnessTrimAfterLoopComp} from './AudioSmoothness/TrimAfterLoop'import {AudioSmoothnessTrimButtonComp} from './AudioSmoothness/TrimButton';import Amplify from './AudioTesting/Amplify';import {Issue7568} from './AudioTesting/Issue7568';+import {+ Issue10468,+ Issue5758,+ issue10468DefaultProps,+ issue5758DefaultProps,+} from './AudioTesting/Issue10468';import {BrowserTest} from './BrowserTest';import {CanvasCapturePreview,@@ -1771,6 +1777,24 @@ export const Index: React.FC = () => {fps={30}durationInFrames={300}/>+ <Composition+ id="audio-issue-10468"+ component={Issue10468}+ width={1920}+ height={1080}+ fps={30}+ durationInFrames={120}+ defaultProps={issue10468DefaultProps}+ />+ <Composition+ id="audio-issue-5758"+ component={Issue5758}+ width={1920}+ height={1080}+ fps={24.87}+ durationInFrames={120}+ defaultProps={issue5758DefaultProps}+ /><Compositionid="audio-issue-7568"component={Issue7568}diff --git a/packages/it-tests/src/ssr/render-media-sample-rate.test.ts b/packages/it-tests/src/ssr/render-media-sample-rate.test.tsindex 04dbdbd68d..4af59107cc 100644--- a/packages/it-tests/src/ssr/render-media-sample-rate.test.ts+++ b/packages/it-tests/src/ssr/render-media-sample-rate.test.ts@@ -1,4 +1,5 @@import {expect, test} from 'bun:test';+import {mkdtempSync, readFileSync, rmSync} from 'node:fs';import os from 'os';import path from 'path';import {@@ -9,6 +10,74 @@ import {const exampleBuild = path.join(__dirname, '..', '..', '..', 'example', 'build');+const readPcmWav = (filePath: string) => {+ const file = readFileSync(filePath);+ let offset = 12;+ let channels: number | null = null;+ let sampleRate: number | null = null;+ let samples: Int16Array | null = null;++ while (offset + 8 <= file.length) {+ const chunk = file.toString('ascii', offset, offset + 4);+ const size = file.readUInt32LE(offset + 4);+ const dataStart = offset + 8;+ if (chunk === 'fmt ') {+ channels = file.readUInt16LE(dataStart + 2);+ sampleRate = file.readUInt32LE(dataStart + 4);+ expect(file.readUInt16LE(dataStart + 14)).toBe(16);+ }++ if (chunk === 'data') {+ const pcm = file.subarray(dataStart, dataStart + size);+ samples = new Int16Array(+ pcm.buffer.slice(pcm.byteOffset, pcm.byteOffset + pcm.byteLength),+ );+ break;+ }++ offset = dataStart + size + (size % 2);+ }++ if (channels === null || sampleRate === null || samples === null) {+ throw new Error(`Could not read PCM WAV file ${filePath}`);+ }++ return {channels, sampleRate, samples};+};++const comparePcm = ({+ reference,+ candidate,+}: {+ reference: Int16Array;+ candidate: Int16Array;+}) => {+ const referenceOnset = reference.findIndex((sample) => sample !== 0);+ const candidateOnset = candidate.findIndex((sample) => sample !== 0);+ expect(candidateOnset).toBe(referenceOnset);++ let signalPower = 0;+ let errorPower = 0;+ let maximumError = 0;+ for (+ let index = referenceOnset;+ index < Math.min(reference.length, candidate.length);+ index+++ ) {+ const expected = reference[index];+ const actual = candidate[index];+ const error = expected - actual;+ signalPower += expected * expected;+ errorPower += error * error;+ maximumError = Math.max(maximumError, Math.abs(error));+ }++ return {+ maximumError,+ signalToNoiseRatio: 10 * Math.log10(signalPower / errorPower),+ };+};+const getSampleRateFromFile = async (filePath: string): Promise<number> => {const result = await RenderInternals.callFf({bin: 'ffprobe',@@ -39,6 +108,58 @@ const getSampleRateFromFile = async (filePath: string): Promise<number> => {return parseInt(match[1], 10);};+const issue10468Source = readFileSync(+ path.join(__dirname, '..', '..', '..', 'remotion-media', 'ding.wav'),+);+const issue10468InputProps = {+ implementation: 'media',+ src: `data:audio/wav;base64,${issue10468Source.toString('base64')}`,+};++const issue5758Source = readFileSync(+ path.join(+ __dirname,+ '..',+ '..',+ '..',+ 'example',+ 'public',+ 'audio-48000hz.wav',+ ),+);+const issue5758InputProps = {+ src: `data:audio/wav;base64,${issue5758Source.toString('base64')}`,+};++const renderIssue10468Wav = async ({+ tmpDir,+ name,+ props,+}: {+ tmpDir: string;+ name: string;+ props: Record<string, unknown>;+}) => {+ const outputLocation = path.join(tmpDir, name);+ const composition = await selectComposition({+ id: 'audio-issue-10468',+ serveUrl: exampleBuild,+ inputProps: props,+ });+ await renderMedia({+ outputLocation,+ codec: 'wav',+ serveUrl: exampleBuild,+ composition,+ inputProps: props,+ sampleRate: 48000,+ concurrency: 1,+ logLevel: 'error',+ });++ return readPcmWav(outputLocation);+};+test('Render video with sampleRate 44100 should produce 44100 Hz audio',async () => {@@ -67,6 +188,76 @@ test({timeout: 30000},);+test(+ '@remotion/media sample-rate conversion should match Html5Audio',+ async () => {+ const tmpDir = mkdtempSync(path.join(os.tmpdir(), 'issue-10468-'));+ try {+ const reference = await renderIssue10468Wav({+ tmpDir,+ name: 'html5-reference.wav',+ props: {...issue10468InputProps, implementation: 'html5'},+ });+ const candidate = await renderIssue10468Wav({+ tmpDir,+ name: 'media.wav',+ props: issue10468InputProps,+ });++ expect(reference.sampleRate).toBe(48000);+ expect(candidate.sampleRate).toBe(48000);+ expect(reference.channels).toBe(2);+ expect(candidate.channels).toBe(2);+ const comparison = comparePcm({+ reference: reference.samples,+ candidate: candidate.samples,+ });+ expect(comparison.signalToNoiseRatio).toBeGreaterThan(30);+ expect(comparison.maximumError).toBeLessThanOrEqual(600);+ } finally {+ rmSync(tmpDir, {recursive: true, force: true});+ }+ },+ {timeout: 180000},+);++test(+ '@remotion/media should not leave audio gaps at 24.87 FPS (#5758)',+ async () => {+ const tmpDir = mkdtempSync(path.join(os.tmpdir(), 'issue-5758-'));+ try {+ const outputLocation = path.join(tmpDir, 'media.wav');+ const composition = await selectComposition({+ id: 'audio-issue-5758',+ serveUrl: exampleBuild,+ inputProps: issue5758InputProps,+ });+ await renderMedia({+ outputLocation,+ codec: 'wav',+ serveUrl: exampleBuild,+ composition,+ inputProps: issue5758InputProps,+ sampleRate: 48000,+ concurrency: 1,+ logLevel: 'error',+ });++ const rendered = readPcmWav(outputLocation);+ const onset = rendered.samples.findIndex((sample) => sample !== 0);+ expect(rendered.sampleRate).toBe(48000);+ expect(rendered.channels).toBe(2);+ expect(onset).toBeGreaterThan(-1);+ expect(+ rendered.samples.subarray(onset).findIndex((sample) => sample === 0),+ ).toBe(-1);+ } finally {+ rmSync(tmpDir, {recursive: true, force: true});+ }+ },+ {timeout: 180000},+);+test('Render video with default sampleRate should produce 48000 Hz audio',async () => {diff --git a/packages/media/src/audio-extraction/extract-audio.ts b/packages/media/src/audio-extraction/extract-audio.tsindex dfbd4f0a7d..b557c3dab1 100644--- a/packages/media/src/audio-extraction/extract-audio.ts+++ b/packages/media/src/audio-extraction/extract-audio.ts@@ -1,15 +1,13 @@import {type LogLevel} from 'remotion';import type {MediaCache} from '../caches';import {combineAudioDataAndClosePrevious} from '../convert-audiodata/combine-audiodata';-import type {PcmS16AudioData} from '../convert-audiodata/convert-audiodata';import {- convertAudioData,+ convertAudioDataToS16,fixFloatingPoint,+ resamplePcmS16AudioData,+ type PcmS16AudioData,+ type UnresampledPcmS16AudioData,} from '../convert-audiodata/convert-audiodata';-import {- TARGET_NUMBER_OF_CHANNELS,- getTargetSampleRate,-} from '../convert-audiodata/resample-audiodata';import {getTimeInSeconds} from '../get-time-in-seconds';import {isNetworkError,@@ -126,7 +124,7 @@ const extractAudioInternal = async ({mediaCache.audioManager.logOpenFrames();- const audioDataArray: PcmS16AudioData[] = [];+ const audioDataArray: UnresampledPcmS16AudioData[] = [];for (let i = 0; i < samples.length; i++) {const sample = samples[i];@@ -152,21 +150,23 @@ const extractAudioInternal = async ({// amount of samples to shave from start and endlet trimStartInSeconds = 0;let trimEndInSeconds = 0;- let leadingSilence: PcmS16AudioData | null = null;+ let leadingSilence: UnresampledPcmS16AudioData | null = null;if (isFirstSample) {trimStartInSeconds = fixFloatingPoint(timeInSeconds - sample.timestamp);if (trimStartInSeconds < 0) {const silenceFrames = Math.ceil(- fixFloatingPoint(-trimStartInSeconds * getTargetSampleRate()),+ fixFloatingPoint(-trimStartInSeconds * audioDataRaw.sampleRate),);leadingSilence = {- data: new Int16Array(silenceFrames * TARGET_NUMBER_OF_CHANNELS),+ data: new Int16Array(silenceFrames * audioDataRaw.numberOfChannels),+ numberOfChannels: audioDataRaw.numberOfChannels,numberOfFrames: silenceFrames,+ sampleRate: audioDataRaw.sampleRate,timestamp: timeInSeconds * 1_000_000,durationInMicroSeconds:- (silenceFrames / getTargetSampleRate()) * 1_000_000,+ (silenceFrames / audioDataRaw.sampleRate) * 1_000_000,};trimStartInSeconds = 0;}@@ -183,11 +183,10 @@ const extractAudioInternal = async ({);}- const audioData = convertAudioData({+ const audioData = convertAudioDataToS16({audioData: audioDataRaw,trimStartInSeconds,trimEndInSeconds,- playbackRate,audioDataTimestamp: sample.timestamp,isLast: isLastSample,});@@ -209,8 +208,13 @@ const extractAudioInternal = async ({}const combined = combineAudioDataAndClosePrevious(audioDataArray);+ const resampled = resamplePcmS16AudioData({+ audioData: combined,+ playbackRate,+ isLast: true,+ });- return {data: combined, durationInSeconds: mediaDurationInSeconds};+ return {data: resampled, durationInSeconds: mediaDurationInSeconds};} catch (err) {const error = err as Error;if (isNetworkError(error)) {diff --git a/packages/media/src/convert-audiodata/combine-audiodata.ts b/packages/media/src/convert-audiodata/combine-audiodata.tsindex 605357c250..140f47183b 100644--- a/packages/media/src/convert-audiodata/combine-audiodata.ts+++ b/packages/media/src/convert-audiodata/combine-audiodata.ts@@ -1,19 +1,28 @@-import {fixFloatingPoint, type PcmS16AudioData} from './convert-audiodata';-import {TARGET_NUMBER_OF_CHANNELS} from './resample-audiodata';+import {+ fixFloatingPoint,+ type UnresampledPcmS16AudioData,+} from './convert-audiodata';export const combineAudioDataAndClosePrevious = (- audioDataArray: PcmS16AudioData[],-): PcmS16AudioData => {+ audioDataArray: UnresampledPcmS16AudioData[],+): UnresampledPcmS16AudioData => {let numberOfFrames = 0;let durationInMicroSeconds = 0;- const {timestamp} = audioDataArray[0];+ const {numberOfChannels, sampleRate, timestamp} = audioDataArray[0];for (const audioData of audioDataArray) {+ if (+ audioData.numberOfChannels !== numberOfChannels ||+ audioData.sampleRate !== sampleRate+ ) {+ throw new Error('Cannot combine audio data with different formats');+ }+numberOfFrames += audioData.numberOfFrames;durationInMicroSeconds += audioData.durationInMicroSeconds;}- const arr = new Int16Array(numberOfFrames * TARGET_NUMBER_OF_CHANNELS);+ const arr = new Int16Array(numberOfFrames * numberOfChannels);let offset = 0;for (const audioData of audioDataArray) {@@ -23,7 +32,9 @@ export const combineAudioDataAndClosePrevious = (return {data: arr,+ numberOfChannels,numberOfFrames,+ sampleRate,timestamp: fixFloatingPoint(timestamp),durationInMicroSeconds: fixFloatingPoint(durationInMicroSeconds),};diff --git a/packages/media/src/convert-audiodata/convert-audiodata.ts b/packages/media/src/convert-audiodata/convert-audiodata.tsindex a6cd59f4dd..ffa617aeac 100644--- a/packages/media/src/convert-audiodata/convert-audiodata.ts+++ b/packages/media/src/convert-audiodata/convert-audiodata.ts@@ -22,6 +22,11 @@ export type PcmS16AudioData = {durationInMicroSeconds: number;};+export type UnresampledPcmS16AudioData = PcmS16AudioData & {+ numberOfChannels: number;+ sampleRate: number;+};+export const fixFloatingPoint = (value: number) => {const decimal = Math.abs(value % 1);@@ -41,20 +46,21 @@ const ceilButNotIfFloatingPointIssue = (value: number) => {return Math.ceil(fixed);};-export const convertAudioData = ({+export const convertAudioDataToS16 = ({audioData,trimStartInSeconds,trimEndInSeconds,- playbackRate,audioDataTimestamp,isLast,-}: ConvertAudioDataOptions): PcmS16AudioData => {+}: Omit<+ ConvertAudioDataOptions,+ 'playbackRate'+>): UnresampledPcmS16AudioData => {const {numberOfChannels: srcNumberOfChannels,sampleRate: currentSampleRate,numberOfFrames,} = audioData;- const ratio = currentSampleRate / getTargetSampleRate();// Always rounding down start timestamps and rounding up end durations// to ensure there are no gaps when the samples don't align@@ -73,16 +79,6 @@ export const convertAudioData = ({? ceilButNotIfFloatingPointIssue(unroundedFrameCount): Math.round(unroundedFrameCount);- const newNumberOfFrames = isLast- ? ceilButNotIfFloatingPointIssue(unroundedFrameCount / ratio / playbackRate)- : Math.round(unroundedFrameCount / ratio / playbackRate);-- if (newNumberOfFrames === 0) {- throw new Error(- 'Cannot resample - the given sample rate would result in less than 1 sample',- );- }-const srcChannels = new Int16Array(srcNumberOfChannels * frameCount);// https://github.com/remotion-dev/remotion/issues/6493@@ -126,32 +122,66 @@ export const convertAudioData = ({});}- const data = new Int16Array(newNumberOfFrames * TARGET_NUMBER_OF_CHANNELS);- const chunkSize = frameCount / newNumberOfFrames;-const timestampOffsetMicroseconds =(frameOffset / audioData.sampleRate) * 1_000_000;+ return {+ data: srcChannels,+ numberOfChannels: srcNumberOfChannels,+ numberOfFrames: frameCount,+ sampleRate: currentSampleRate,+ timestamp:+ audioDataTimestamp * 1_000_000 ++ fixFloatingPoint(timestampOffsetMicroseconds),+ durationInMicroSeconds: fixFloatingPoint(+ (frameCount / currentSampleRate) * 1_000_000,+ ),+ };+};++export const resamplePcmS16AudioData = ({+ audioData,+ playbackRate,+ isLast,+}: {+ audioData: UnresampledPcmS16AudioData;+ playbackRate: number;+ isLast: boolean;+}): PcmS16AudioData => {+ const ratio = audioData.sampleRate / getTargetSampleRate();+ const newNumberOfFrames = isLast+ ? ceilButNotIfFloatingPointIssue(+ audioData.numberOfFrames / ratio / playbackRate,+ )+ : Math.round(audioData.numberOfFrames / ratio / playbackRate);++ if (newNumberOfFrames === 0) {+ throw new Error(+ 'Cannot resample - the given sample rate would result in less than 1 sample',+ );+ }+if (- newNumberOfFrames === frameCount &&- TARGET_NUMBER_OF_CHANNELS === srcNumberOfChannels &&+ newNumberOfFrames === audioData.numberOfFrames &&+ TARGET_NUMBER_OF_CHANNELS === audioData.numberOfChannels &&playbackRate === 1) {return {- data: srcChannels,+ data: audioData.data,numberOfFrames: newNumberOfFrames,- timestamp:- audioDataTimestamp * 1_000_000 +- fixFloatingPoint(timestampOffsetMicroseconds),+ timestamp: audioData.timestamp,durationInMicroSeconds: fixFloatingPoint((newNumberOfFrames / getTargetSampleRate()) * 1_000_000,),};}+ const data = new Int16Array(newNumberOfFrames * TARGET_NUMBER_OF_CHANNELS);+ const chunkSize = audioData.numberOfFrames / newNumberOfFrames;+resampleAudioData({- srcNumberOfChannels,- sourceChannels: srcChannels,+ srcNumberOfChannels: audioData.numberOfChannels,+ sourceChannels: audioData.data,destination: data,targetFrames: newNumberOfFrames,chunkSize,@@ -160,9 +190,7 @@ export const convertAudioData = ({const newAudioData: PcmS16AudioData = {data,numberOfFrames: newNumberOfFrames,- timestamp:- audioDataTimestamp * 1_000_000 +- fixFloatingPoint(timestampOffsetMicroseconds),+ timestamp: audioData.timestamp,durationInMicroSeconds: fixFloatingPoint((newNumberOfFrames / getTargetSampleRate()) * 1_000_000,),@@ -170,3 +198,15 @@ export const convertAudioData = ({return newAudioData;};++export const convertAudioData = (+ options: ConvertAudioDataOptions,+): PcmS16AudioData => {+ const audioData = convertAudioDataToS16(options);++ return resamplePcmS16AudioData({+ audioData,+ playbackRate: options.playbackRate,+ isLast: options.isLast,+ });+};diff --git a/packages/media/src/test/browser.test.ts b/packages/media/src/test/browser.test.tsindex 4a62bf831c..560eb6f7a8 100644--- a/packages/media/src/test/browser.test.ts+++ b/packages/media/src/test/browser.test.ts@@ -57,8 +57,7 @@ test('Should be able to extract a frame', async () => {// bits = 16// sampleRate = 48000// 1 / 30 * 2 * 2 * 48000 = 6400- // we round down start and round up duration- expect(audio.data.byteLength).toBe(6404);+ expect(audio.data.byteLength).toBe(6400);const cacheStats = keyframeManager.getCacheStats();expect(cacheStats.count).toBe(1);diff --git a/packages/media/src/test/extract-audio-resampling.test.ts b/packages/media/src/test/extract-audio-resampling.test.tsnew file mode 100644index 0000000000..1f961b2297--- /dev/null+++ b/packages/media/src/test/extract-audio-resampling.test.ts@@ -0,0 +1,63 @@+import {assert, expect, test} from 'vitest';+import {extractAudio} from '../audio-extraction/extract-audio';+import {getMaxVideoCacheSize, globalMediaCache} from '../caches';+import {resampleAudioData} from '../convert-audiodata/resample-audiodata';++const FPS = 30;+const SOURCE_SAMPLE_RATE = 44100;+const TARGET_SAMPLE_RATE = 48000;+const NUMBER_OF_CHANNELS = 2;+const TRIM_BEFORE = 4;++test('resamples Mediabunny AudioSamples as one continuous frame', async () => {+ const src = new URL('../../../remotion-media/ding.wav', import.meta.url).href;+ const wav = await fetch(src).then((response) => response.arrayBuffer());+ const view = new DataView(wav);+ let dataOffset = 12;+ while (view.getUint32(dataOffset, false) !== 0x64617461) {+ dataOffset += 8 + view.getUint32(dataOffset + 4, true);+ }++ const source = new Int16Array(wav, dataOffset + 8);+ const extracted = await extractAudio({+ audioStreamIndex: 0,+ timeInSeconds: 0,+ durationInSeconds: 1 / FPS,+ playbackRate: 1,+ fps: FPS,+ logLevel: 'info',+ loop: false,+ src,+ trimBefore: TRIM_BEFORE,+ trimAfter: undefined,+ maxCacheSize: getMaxVideoCacheSize('info'),+ credentials: undefined,+ mediaCache: globalMediaCache,+ });++ assert(extracted && typeof extracted === 'object');+ assert(extracted.data);++ const targetFrames = TARGET_SAMPLE_RATE / FPS;+ const expected = new Int16Array(targetFrames * NUMBER_OF_CHANNELS);+ resampleAudioData({+ srcNumberOfChannels: NUMBER_OF_CHANNELS,+ sourceChannels: source.subarray(+ ((TRIM_BEFORE * SOURCE_SAMPLE_RATE) / FPS) * NUMBER_OF_CHANNELS,+ ),+ destination: expected,+ targetFrames,+ chunkSize: SOURCE_SAMPLE_RATE / TARGET_SAMPLE_RATE,+ });++ expect(extracted.data.data.length).toBe(expected.length);+ let maximumError = 0;+ for (let i = 0; i < expected.length; i++) {+ maximumError = Math.max(+ maximumError,+ Math.abs(extracted.data.data[i] - expected[i]),+ );+ }++ expect(maximumError).toBeLessThanOrEqual(2);+});