🎥 Make videos programmatically with React
`@remotion/renderer`: Preserve exact inline audio samples
JonnyBurgerdfb31437e656
diff --git a/packages/it-tests/src/ssr/render-media-sample-rate.test.ts b/packages/it-tests/src/ssr/render-media-sample-rate.test.tsindex b6bc79acef..83dcda311a 100644--- a/packages/it-tests/src/ssr/render-media-sample-rate.test.ts+++ b/packages/it-tests/src/ssr/render-media-sample-rate.test.ts@@ -10,6 +10,7 @@ import {import os from 'os';import path from 'path';import {+ combineChunks,renderMedia,RenderInternals,selectComposition,@@ -115,6 +116,25 @@ const getSampleRateFromFile = async (filePath: string): Promise<number> => {return parseInt(match[1], 10);};+const decodeAudioToPcmWav = async ({+ input,+ output,+}: {+ input: string;+ output: string;+}) => {+ await RenderInternals.callFf({+ bin: 'ffmpeg',+ args: ['-y', '-i', input, '-map', '0:a:0', '-c:a', 'pcm_s16le', output],+ indent: false,+ logLevel: 'error',+ binariesDirectory: null,+ cancelSignal: undefined,+ });++ return readPcmWav(output);+};+const issue10468Source = readFileSync(path.join(__dirname, '..', '..', '..', 'remotion-media', 'ding.wav'),);@@ -469,6 +489,109 @@ test({timeout: 300000},);+test(+ '@remotion/media keeps seamless AAC chunks sample-identical with 423 nested clips',+ async () => {+ const tmpDir = mkdtempSync(+ path.join(os.tmpdir(), 'inline-audio-seamless-aac-'),+ );+ try {+ const composition = await selectComposition({+ id: 'inline-audio-stress',+ serveUrl: exampleBuild,+ inputProps: inlineAudioStressInputProps,+ });+ const singleAac = path.join(tmpDir, 'single.aac');+ await renderMedia({+ outputLocation: singleAac,+ codec: 'aac',+ serveUrl: exampleBuild,+ composition,+ inputProps: inlineAudioStressInputProps,+ sampleRate: STRESS_SAMPLE_RATE,+ concurrency: 4,+ logLevel: 'error',+ });++ const framesPerChunk = 200;+ const audioFiles: string[] = [];+ for (+ let chunkStart = 0;+ chunkStart < STRESS_DURATION_IN_FRAMES;+ chunkStart += framesPerChunk+ ) {+ const chunk = path.join(tmpDir, `chunk-${chunkStart}.aac`);+ audioFiles.push(chunk);+ await renderMedia({+ outputLocation: chunk,+ codec: 'aac',+ serveUrl: exampleBuild,+ composition,+ inputProps: inlineAudioStressInputProps,+ frameRange: [+ chunkStart,+ Math.min(+ chunkStart + framesPerChunk - 1,+ STRESS_DURATION_IN_FRAMES - 1,+ ),+ ],+ compositionStart: 0,+ forSeamlessAacConcatenation: true,+ sampleRate: STRESS_SAMPLE_RATE,+ concurrency: 4,+ logLevel: 'error',+ });+ }++ const combinedAac = path.join(tmpDir, 'combined.aac');+ await combineChunks({+ outputLocation: combinedAac,+ audioFiles,+ videoFiles: [],+ codec: 'aac',+ audioCodec: 'aac',+ fps: STRESS_FPS,+ framesPerChunk,+ preferLossless: false,+ compositionDurationInFrames: STRESS_DURATION_IN_FRAMES,+ sampleRate: STRESS_SAMPLE_RATE,+ logLevel: 'error',+ });++ const single = await decodeAudioToPcmWav({+ input: singleAac,+ output: path.join(tmpDir, 'single.wav'),+ });+ const combined = await decodeAudioToPcmWav({+ input: combinedAac,+ output: path.join(tmpDir, 'combined.wav'),+ });+ expect(combined.sampleRate).toBe(single.sampleRate);+ expect(combined.channels).toBe(single.channels);+ expect(combined.samples.length).toBeGreaterThanOrEqual(+ single.samples.length,+ );++ let firstDifferentSample = -1;+ for (let index = 0; index < single.samples.length; index++) {+ if (single.samples[index] !== combined.samples[index]) {+ firstDifferentSample = index;+ break;+ }+ }++ expect(firstDifferentSample).toBe(-1);+ // AAC concatenation may retain up to two encoder frames of tail padding.+ expect(+ combined.samples.length - single.samples.length,+ ).toBeLessThanOrEqual(2 * 1024 * single.channels);+ } finally {+ rmSync(tmpDir, {recursive: true, force: true});+ }+ },+ {timeout: 300000},+);+test('Render video with default sampleRate should produce 48000 Hz audio',async () => {diff --git a/packages/renderer/src/assets/inline-audio-mixing.ts b/packages/renderer/src/assets/inline-audio-mixing.tsindex b357016e75..5effb608b2 100644--- a/packages/renderer/src/assets/inline-audio-mixing.ts+++ b/packages/renderer/src/assets/inline-audio-mixing.ts@@ -42,8 +42,8 @@ const WAV_HEADER_SIZE = 44;export type InlineAudioTrack = {outName: string;- startInSeconds: number;- durationInSeconds: number;+ startInSamples: number;+ durationInSamples: number;};export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {@@ -51,7 +51,7 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {const openFiles: Record<string, number> = {};const writtenHeaders: Record<string, boolean> = {};const toneFrequencies: Record<string, number> = {};- const startTimesInSeconds: Record<string, number> = {};+ const startTimesInSamples: Record<string, number> = {};const writtenDataSizes: Record<string, number> = {};const cleanup = () => {@@ -69,12 +69,11 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {return Object.keys(writtenHeaders).map((outName) => ({outName,- startInSeconds: startTimesInSeconds[outName],- durationInSeconds:- writtenDataSizes[outName] /- (NUMBER_OF_CHANNELS * sampleRate * BYTES_PER_SAMPLE),+ startInSamples: startTimesInSamples[outName],+ durationInSamples:+ writtenDataSizes[outName] / (NUMBER_OF_CHANNELS * BYTES_PER_SAMPLE),}))- .sort((a, b) => a.startInSeconds - b.startInSeconds);+ .sort((a, b) => a.startInSamples - b.startInSamples);};const getFilePath = (asset: InlineAudioAsset) => {@@ -198,21 +197,25 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {const assetStartInVideo = asset.startInVideo ?? firstFrame;const firstFrameForAsset = Math.max(assetStartInVideo, firstFrame);- const startsAtRenderStart = firstFrameForAsset === firstFrame;- const startInSeconds = Math.max(+ const startInSamples = Math.max(0,- (firstFrameForAsset - firstFrame) / fps - trimLeftOffset,+ Math.floor(+ correctFloatingPointError(+ ((firstFrameForAsset - firstFrame) / fps - trimLeftOffset) *+ sampleRate,+ ),+ ),);if (- startTimesInSeconds[filePath] !== undefined &&- Math.abs(startTimesInSeconds[filePath] - startInSeconds) > 0.00001+ startTimesInSamples[filePath] !== undefined &&+ startTimesInSamples[filePath] !== startInSamples) {throw new Error(- `The start time for inline audio asset ${asset.id} changed from ${startTimesInSeconds[filePath]} to ${startInSeconds}`,+ `The start time for inline audio asset ${asset.id} changed from ${startTimesInSamples[filePath]} to ${startInSamples} samples`,);}- startTimesInSeconds[filePath] = startInSeconds;+ startTimesInSamples[filePath] = startInSamples;let arr = new Int16Array(asset.audio);const isFirst = asset.frame === firstFrame;@@ -240,16 +243,18 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {);}- const positionInSeconds =- (asset.frame - firstFrameForAsset) / fps -- (isFirst || !startsAtRenderStart ? 0 : trimLeftOffset);+ const positionInRenderInSeconds =+ (asset.frame - firstFrame) / fps - (isFirst ? 0 : trimLeftOffset);// Always rounding down to ensure there are no gaps when the samples don't align// In @remotion/media, we also round down the sample start timestamp and round up the end timestamp// This might lead to overlapping, hopefully aligning perfectly!// Test case: https://github.com/remotion-dev/remotion/issues/5758const position =- Math.floor(correctFloatingPointError(positionInSeconds * sampleRate)) *+ (Math.floor(+ correctFloatingPointError(positionInRenderInSeconds * sampleRate),+ ) -+ startInSamples) *NUMBER_OF_CHANNELS *BYTES_PER_SAMPLE;diff --git a/packages/renderer/src/create-audio.ts b/packages/renderer/src/create-audio.tsindex 0263f5e3a7..d8b96e59b6 100644--- a/packages/renderer/src/create-audio.ts+++ b/packages/renderer/src/create-audio.ts@@ -161,9 +161,10 @@ export const createAudio = async ({? [inlineAudioTrackToPreprocessedAudioTrack({track: mergedInlineAudio,- relativeToInSeconds: 0,- padToDurationInSeconds: chunkLengthInSeconds,- sampleRate,+ relativeToInSamples: 0,+ padToDurationInSamples: Math.round(+ chunkLengthInSeconds * sampleRate,+ ),}),]: []),diff --git a/packages/renderer/src/merge-inline-audio-tracks.ts b/packages/renderer/src/merge-inline-audio-tracks.tsindex 5727ced39e..ff3e97421e 100644--- a/packages/renderer/src/merge-inline-audio-tracks.ts+++ b/packages/renderer/src/merge-inline-audio-tracks.ts@@ -12,27 +12,23 @@ const MAX_INLINE_AUDIO_INPUTS = 10;export const inlineAudioTrackToPreprocessedAudioTrack = ({track,- relativeToInSeconds,- padToDurationInSeconds,- sampleRate,+ relativeToInSamples,+ padToDurationInSamples,}: {track: InlineAudioTrack;- relativeToInSeconds: number;- padToDurationInSeconds: number | null;- sampleRate: number;+ relativeToInSamples: number;+ padToDurationInSamples: number | null;}): PreprocessedAudioTrack => {const delayInSamples = Math.max(0,- Math.round((track.startInSeconds - relativeToInSeconds) * sampleRate),+ track.startInSamples - relativeToInSamples,);const padAtEndInSamples =- padToDurationInSeconds === null+ padToDurationInSamples === null? 0: Math.max(0,- Math.round(padToDurationInSeconds * sampleRate) -- delayInSamples -- Math.round(track.durationInSeconds * sampleRate),+ padToDurationInSamples - delayInSamples - track.durationInSamples,);return {@@ -72,7 +68,7 @@ export const mergeInlineAudioTracks = async ({sampleRate: number;}): Promise<InlineAudioTrack | null> => {let currentTracks = tracks.sort(- (a, b) => a.startInSeconds - b.startInSeconds,+ (a, b) => a.startInSamples - b.startInSamples,);let level = 0;@@ -84,12 +80,11 @@ export const mergeInlineAudioTracks = async ({return group[0];}- const [{startInSeconds}] = group;+ const [{startInSamples}] = group;const durationInSamples = Math.max(...group.map((track) => {return (- Math.round((track.startInSeconds - startInSeconds) * sampleRate) +- Math.round(track.durationInSeconds * sampleRate)+ track.startInSamples - startInSamples + track.durationInSamples);}),);@@ -102,9 +97,8 @@ export const mergeInlineAudioTracks = async ({files: group.map((track) =>inlineAudioTrackToPreprocessedAudioTrack({track,- relativeToInSeconds: startInSeconds,- padToDurationInSeconds: null,- sampleRate,+ relativeToInSamples: startInSamples,+ padToDurationInSamples: null,}),),outName,@@ -126,8 +120,8 @@ export const mergeInlineAudioTracks = async ({return {outName,- startInSeconds,- durationInSeconds: durationInSamples / sampleRate,+ startInSamples,+ durationInSamples,};}),);diff --git a/packages/renderer/src/test/inline-audio-mixing.test.ts b/packages/renderer/src/test/inline-audio-mixing.test.tsindex 6bc835bc80..4924b40cf8 100644--- a/packages/renderer/src/test/inline-audio-mixing.test.ts+++ b/packages/renderer/src/test/inline-audio-mixing.test.ts@@ -5,7 +5,10 @@ import path from 'node:path';import type {InlineAudioAsset} from 'remotion/no-react';import {cleanDownloadMap, makeDownloadMap} from '../assets/download-map';import {makeInlineAudioMixing} from '../assets/inline-audio-mixing';-import {mergeInlineAudioTracks} from '../merge-inline-audio-tracks';+import {+ inlineAudioTrackToPreprocessedAudioTrack,+ mergeInlineAudioTracks,+} from '../merge-inline-audio-tracks';const SAMPLE_RATE = 48_000;const FPS = 30;@@ -67,8 +70,8 @@ test('writes late inline audio relative to its clip origin', async () => {const [track] = mixing.getListOfAssets();const wav = readFileSync(track.outName);- expect(track.startInSeconds).toBe(10);- expect(track.durationInSeconds).toBeCloseTo(2 / FPS, 8);+ expect(track.startInSamples).toBe(10 * SAMPLE_RATE);+ expect(track.durationInSamples).toBe(2 * SAMPLES_PER_FRAME);expect(wav.length).toBe(44 + SAMPLES_PER_FRAME * 2 * 2 * 2);expect(wav.readUInt32LE(4)).toBe(wav.length - 8);expect(wav.readUInt32LE(40)).toBe(wav.length - 44);@@ -94,8 +97,50 @@ test('keeps a composition-relative fallback for an unknown clip origin', async (await finish(mixing);const [track] = mixing.getListOfAssets();- expect(track.startInSeconds).toBe(0);- expect(track.durationInSeconds).toBeCloseTo(4 / FPS, 8);+ expect(track.startInSamples).toBe(0);+ expect(track.durationInSamples).toBe(4 * SAMPLES_PER_FRAME);+ } finally {+ mixing.cleanup();+ rmSync(dir, {recursive: true, force: true});+ }+});++test('preserves legacy sample placement at fractional frame rates', async () => {+ const dir = mkdtempSync(path.join(os.tmpdir(), 'inline-audio-fractional-'));+ const mixing = makeInlineAudioMixing(dir, SAMPLE_RATE);++ try {+ // Write out of order to also exercise sparse compact-file placement.+ mixing.addAsset({+ asset: makeAsset({id: 'fractional', frame: 83, startInVideo: 81}),+ fps: 24.87,+ totalNumberOfFrames: 41,+ firstFrame: 79,+ trimLeftOffset: 0.0021487736228385534,+ trimRightOffset: 0,+ });+ mixing.addAsset({+ asset: makeAsset({id: 'fractional', frame: 81, startInVideo: 81}),+ fps: 24.87,+ totalNumberOfFrames: 41,+ firstFrame: 79,+ trimLeftOffset: 0.0021487736228385534,+ trimRightOffset: 0,+ });+ await finish(mixing);++ const [track] = mixing.getListOfAssets();+ const wav = readFileSync(track.outName);+ const preprocessed = inlineAudioTrackToPreprocessedAudioTrack({+ track,+ relativeToInSamples: 0,+ padToDurationInSamples: null,+ });+ expect(preprocessed.filter.pad_start).toBe('adelay=3756S|3756S|3756S');+ // Legacy absolute placement for frame 83 is sample 7617. Relative to the+ // compact track's sample 3756 origin, its first sample must be at 3861.+ expect(wav.readInt16LE(44 + 3860 * 2 * 2)).toBe(0);+ expect(wav.readInt16LE(44 + 3861 * 2 * 2)).toBe(1000);} finally {mixing.cleanup();rmSync(dir, {recursive: true, force: true});@@ -148,10 +193,9 @@ test(});expect(merged).not.toBeNull();- expect(merged?.startInSeconds).toBe(startFrames[0] / FPS);- expect(merged?.durationInSeconds).toBeCloseTo(- (startFrames.at(-1)! - startFrames[0] + 1) / FPS,- 8,+ expect(merged?.startInSamples).toBe((startFrames[0] * SAMPLE_RATE) / FPS);+ expect(merged?.durationInSamples).toBe(+ ((startFrames.at(-1)! - startFrames[0] + 1) * SAMPLE_RATE) / FPS,);expect(statSync(merged!.outName).size).toBeLessThan(4_000_000);} finally {