Skip to content

🎥 Make videos programmatically with React

`@remotion/renderer`: Preserve exact inline audio samples

JonnyBurgerdfb31437e656
diff --git a/packages/it-tests/src/ssr/render-media-sample-rate.test.ts b/packages/it-tests/src/ssr/render-media-sample-rate.test.ts
index b6bc79acef..83dcda311a 100644
--- a/packages/it-tests/src/ssr/render-media-sample-rate.test.ts
+++ b/packages/it-tests/src/ssr/render-media-sample-rate.test.ts
@@ -10,6 +10,7 @@ import {
import os from 'os';
import path from 'path';
import {
+ combineChunks,
renderMedia,
RenderInternals,
selectComposition,
@@ -115,6 +116,25 @@ const getSampleRateFromFile = async (filePath: string): Promise<number> => {
return parseInt(match[1], 10);
};
+const decodeAudioToPcmWav = async ({
+ input,
+ output,
+}: {
+ input: string;
+ output: string;
+}) => {
+ await RenderInternals.callFf({
+ bin: 'ffmpeg',
+ args: ['-y', '-i', input, '-map', '0:a:0', '-c:a', 'pcm_s16le', output],
+ indent: false,
+ logLevel: 'error',
+ binariesDirectory: null,
+ cancelSignal: undefined,
+ });
+
+ return readPcmWav(output);
+};
+
const issue10468Source = readFileSync(
path.join(__dirname, '..', '..', '..', 'remotion-media', 'ding.wav'),
);
@@ -469,6 +489,109 @@ test(
{timeout: 300000},
);
+test(
+ '@remotion/media keeps seamless AAC chunks sample-identical with 423 nested clips',
+ async () => {
+ const tmpDir = mkdtempSync(
+ path.join(os.tmpdir(), 'inline-audio-seamless-aac-'),
+ );
+ try {
+ const composition = await selectComposition({
+ id: 'inline-audio-stress',
+ serveUrl: exampleBuild,
+ inputProps: inlineAudioStressInputProps,
+ });
+ const singleAac = path.join(tmpDir, 'single.aac');
+ await renderMedia({
+ outputLocation: singleAac,
+ codec: 'aac',
+ serveUrl: exampleBuild,
+ composition,
+ inputProps: inlineAudioStressInputProps,
+ sampleRate: STRESS_SAMPLE_RATE,
+ concurrency: 4,
+ logLevel: 'error',
+ });
+
+ const framesPerChunk = 200;
+ const audioFiles: string[] = [];
+ for (
+ let chunkStart = 0;
+ chunkStart < STRESS_DURATION_IN_FRAMES;
+ chunkStart += framesPerChunk
+ ) {
+ const chunk = path.join(tmpDir, `chunk-${chunkStart}.aac`);
+ audioFiles.push(chunk);
+ await renderMedia({
+ outputLocation: chunk,
+ codec: 'aac',
+ serveUrl: exampleBuild,
+ composition,
+ inputProps: inlineAudioStressInputProps,
+ frameRange: [
+ chunkStart,
+ Math.min(
+ chunkStart + framesPerChunk - 1,
+ STRESS_DURATION_IN_FRAMES - 1,
+ ),
+ ],
+ compositionStart: 0,
+ forSeamlessAacConcatenation: true,
+ sampleRate: STRESS_SAMPLE_RATE,
+ concurrency: 4,
+ logLevel: 'error',
+ });
+ }
+
+ const combinedAac = path.join(tmpDir, 'combined.aac');
+ await combineChunks({
+ outputLocation: combinedAac,
+ audioFiles,
+ videoFiles: [],
+ codec: 'aac',
+ audioCodec: 'aac',
+ fps: STRESS_FPS,
+ framesPerChunk,
+ preferLossless: false,
+ compositionDurationInFrames: STRESS_DURATION_IN_FRAMES,
+ sampleRate: STRESS_SAMPLE_RATE,
+ logLevel: 'error',
+ });
+
+ const single = await decodeAudioToPcmWav({
+ input: singleAac,
+ output: path.join(tmpDir, 'single.wav'),
+ });
+ const combined = await decodeAudioToPcmWav({
+ input: combinedAac,
+ output: path.join(tmpDir, 'combined.wav'),
+ });
+ expect(combined.sampleRate).toBe(single.sampleRate);
+ expect(combined.channels).toBe(single.channels);
+ expect(combined.samples.length).toBeGreaterThanOrEqual(
+ single.samples.length,
+ );
+
+ let firstDifferentSample = -1;
+ for (let index = 0; index < single.samples.length; index++) {
+ if (single.samples[index] !== combined.samples[index]) {
+ firstDifferentSample = index;
+ break;
+ }
+ }
+
+ expect(firstDifferentSample).toBe(-1);
+ // AAC concatenation may retain up to two encoder frames of tail padding.
+ expect(
+ combined.samples.length - single.samples.length,
+ ).toBeLessThanOrEqual(2 * 1024 * single.channels);
+ } finally {
+ rmSync(tmpDir, {recursive: true, force: true});
+ }
+ },
+ {timeout: 300000},
+);
+
test(
'Render video with default sampleRate should produce 48000 Hz audio',
async () => {
diff --git a/packages/renderer/src/assets/inline-audio-mixing.ts b/packages/renderer/src/assets/inline-audio-mixing.ts
index b357016e75..5effb608b2 100644
--- a/packages/renderer/src/assets/inline-audio-mixing.ts
+++ b/packages/renderer/src/assets/inline-audio-mixing.ts
@@ -42,8 +42,8 @@ const WAV_HEADER_SIZE = 44;
export type InlineAudioTrack = {
outName: string;
- startInSeconds: number;
- durationInSeconds: number;
+ startInSamples: number;
+ durationInSamples: number;
};
export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {
@@ -51,7 +51,7 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {
const openFiles: Record<string, number> = {};
const writtenHeaders: Record<string, boolean> = {};
const toneFrequencies: Record<string, number> = {};
- const startTimesInSeconds: Record<string, number> = {};
+ const startTimesInSamples: Record<string, number> = {};
const writtenDataSizes: Record<string, number> = {};
const cleanup = () => {
@@ -69,12 +69,11 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {
return Object.keys(writtenHeaders)
.map((outName) => ({
outName,
- startInSeconds: startTimesInSeconds[outName],
- durationInSeconds:
- writtenDataSizes[outName] /
- (NUMBER_OF_CHANNELS * sampleRate * BYTES_PER_SAMPLE),
+ startInSamples: startTimesInSamples[outName],
+ durationInSamples:
+ writtenDataSizes[outName] / (NUMBER_OF_CHANNELS * BYTES_PER_SAMPLE),
}))
- .sort((a, b) => a.startInSeconds - b.startInSeconds);
+ .sort((a, b) => a.startInSamples - b.startInSamples);
};
const getFilePath = (asset: InlineAudioAsset) => {
@@ -198,21 +197,25 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {
const assetStartInVideo = asset.startInVideo ?? firstFrame;
const firstFrameForAsset = Math.max(assetStartInVideo, firstFrame);
- const startsAtRenderStart = firstFrameForAsset === firstFrame;
- const startInSeconds = Math.max(
+ const startInSamples = Math.max(
0,
- (firstFrameForAsset - firstFrame) / fps - trimLeftOffset,
+ Math.floor(
+ correctFloatingPointError(
+ ((firstFrameForAsset - firstFrame) / fps - trimLeftOffset) *
+ sampleRate,
+ ),
+ ),
);
if (
- startTimesInSeconds[filePath] !== undefined &&
- Math.abs(startTimesInSeconds[filePath] - startInSeconds) > 0.00001
+ startTimesInSamples[filePath] !== undefined &&
+ startTimesInSamples[filePath] !== startInSamples
) {
throw new Error(
- `The start time for inline audio asset ${asset.id} changed from ${startTimesInSeconds[filePath]} to ${startInSeconds}`,
+ `The start time for inline audio asset ${asset.id} changed from ${startTimesInSamples[filePath]} to ${startInSamples} samples`,
);
}
- startTimesInSeconds[filePath] = startInSeconds;
+ startTimesInSamples[filePath] = startInSamples;
let arr = new Int16Array(asset.audio);
const isFirst = asset.frame === firstFrame;
@@ -240,16 +243,18 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {
);
}
- const positionInSeconds =
- (asset.frame - firstFrameForAsset) / fps -
- (isFirst || !startsAtRenderStart ? 0 : trimLeftOffset);
+ const positionInRenderInSeconds =
+ (asset.frame - firstFrame) / fps - (isFirst ? 0 : trimLeftOffset);
// Always rounding down to ensure there are no gaps when the samples don't align
// In @remotion/media, we also round down the sample start timestamp and round up the end timestamp
// This might lead to overlapping, hopefully aligning perfectly!
// Test case: https://github.com/remotion-dev/remotion/issues/5758
const position =
- Math.floor(correctFloatingPointError(positionInSeconds * sampleRate)) *
+ (Math.floor(
+ correctFloatingPointError(positionInRenderInSeconds * sampleRate),
+ ) -
+ startInSamples) *
NUMBER_OF_CHANNELS *
BYTES_PER_SAMPLE;
diff --git a/packages/renderer/src/create-audio.ts b/packages/renderer/src/create-audio.ts
index 0263f5e3a7..d8b96e59b6 100644
--- a/packages/renderer/src/create-audio.ts
+++ b/packages/renderer/src/create-audio.ts
@@ -161,9 +161,10 @@ export const createAudio = async ({
? [
inlineAudioTrackToPreprocessedAudioTrack({
track: mergedInlineAudio,
- relativeToInSeconds: 0,
- padToDurationInSeconds: chunkLengthInSeconds,
- sampleRate,
+ relativeToInSamples: 0,
+ padToDurationInSamples: Math.round(
+ chunkLengthInSeconds * sampleRate,
+ ),
}),
]
: []),
diff --git a/packages/renderer/src/merge-inline-audio-tracks.ts b/packages/renderer/src/merge-inline-audio-tracks.ts
index 5727ced39e..ff3e97421e 100644
--- a/packages/renderer/src/merge-inline-audio-tracks.ts
+++ b/packages/renderer/src/merge-inline-audio-tracks.ts
@@ -12,27 +12,23 @@ const MAX_INLINE_AUDIO_INPUTS = 10;
export const inlineAudioTrackToPreprocessedAudioTrack = ({
track,
- relativeToInSeconds,
- padToDurationInSeconds,
- sampleRate,
+ relativeToInSamples,
+ padToDurationInSamples,
}: {
track: InlineAudioTrack;
- relativeToInSeconds: number;
- padToDurationInSeconds: number | null;
- sampleRate: number;
+ relativeToInSamples: number;
+ padToDurationInSamples: number | null;
}): PreprocessedAudioTrack => {
const delayInSamples = Math.max(
0,
- Math.round((track.startInSeconds - relativeToInSeconds) * sampleRate),
+ track.startInSamples - relativeToInSamples,
);
const padAtEndInSamples =
- padToDurationInSeconds === null
+ padToDurationInSamples === null
? 0
: Math.max(
0,
- Math.round(padToDurationInSeconds * sampleRate) -
- delayInSamples -
- Math.round(track.durationInSeconds * sampleRate),
+ padToDurationInSamples - delayInSamples - track.durationInSamples,
);
return {
@@ -72,7 +68,7 @@ export const mergeInlineAudioTracks = async ({
sampleRate: number;
}): Promise<InlineAudioTrack | null> => {
let currentTracks = tracks.sort(
- (a, b) => a.startInSeconds - b.startInSeconds,
+ (a, b) => a.startInSamples - b.startInSamples,
);
let level = 0;
@@ -84,12 +80,11 @@ export const mergeInlineAudioTracks = async ({
return group[0];
}
- const [{startInSeconds}] = group;
+ const [{startInSamples}] = group;
const durationInSamples = Math.max(
...group.map((track) => {
return (
- Math.round((track.startInSeconds - startInSeconds) * sampleRate) +
- Math.round(track.durationInSeconds * sampleRate)
+ track.startInSamples - startInSamples + track.durationInSamples
);
}),
);
@@ -102,9 +97,8 @@ export const mergeInlineAudioTracks = async ({
files: group.map((track) =>
inlineAudioTrackToPreprocessedAudioTrack({
track,
- relativeToInSeconds: startInSeconds,
- padToDurationInSeconds: null,
- sampleRate,
+ relativeToInSamples: startInSamples,
+ padToDurationInSamples: null,
}),
),
outName,
@@ -126,8 +120,8 @@ export const mergeInlineAudioTracks = async ({
return {
outName,
- startInSeconds,
- durationInSeconds: durationInSamples / sampleRate,
+ startInSamples,
+ durationInSamples,
};
}),
);
diff --git a/packages/renderer/src/test/inline-audio-mixing.test.ts b/packages/renderer/src/test/inline-audio-mixing.test.ts
index 6bc835bc80..4924b40cf8 100644
--- a/packages/renderer/src/test/inline-audio-mixing.test.ts
+++ b/packages/renderer/src/test/inline-audio-mixing.test.ts
@@ -5,7 +5,10 @@ import path from 'node:path';
import type {InlineAudioAsset} from 'remotion/no-react';
import {cleanDownloadMap, makeDownloadMap} from '../assets/download-map';
import {makeInlineAudioMixing} from '../assets/inline-audio-mixing';
-import {mergeInlineAudioTracks} from '../merge-inline-audio-tracks';
+import {
+ inlineAudioTrackToPreprocessedAudioTrack,
+ mergeInlineAudioTracks,
+} from '../merge-inline-audio-tracks';
const SAMPLE_RATE = 48_000;
const FPS = 30;
@@ -67,8 +70,8 @@ test('writes late inline audio relative to its clip origin', async () => {
const [track] = mixing.getListOfAssets();
const wav = readFileSync(track.outName);
- expect(track.startInSeconds).toBe(10);
- expect(track.durationInSeconds).toBeCloseTo(2 / FPS, 8);
+ expect(track.startInSamples).toBe(10 * SAMPLE_RATE);
+ expect(track.durationInSamples).toBe(2 * SAMPLES_PER_FRAME);
expect(wav.length).toBe(44 + SAMPLES_PER_FRAME * 2 * 2 * 2);
expect(wav.readUInt32LE(4)).toBe(wav.length - 8);
expect(wav.readUInt32LE(40)).toBe(wav.length - 44);
@@ -94,8 +97,50 @@ test('keeps a composition-relative fallback for an unknown clip origin', async (
await finish(mixing);
const [track] = mixing.getListOfAssets();
- expect(track.startInSeconds).toBe(0);
- expect(track.durationInSeconds).toBeCloseTo(4 / FPS, 8);
+ expect(track.startInSamples).toBe(0);
+ expect(track.durationInSamples).toBe(4 * SAMPLES_PER_FRAME);
+ } finally {
+ mixing.cleanup();
+ rmSync(dir, {recursive: true, force: true});
+ }
+});
+
+test('preserves legacy sample placement at fractional frame rates', async () => {
+ const dir = mkdtempSync(path.join(os.tmpdir(), 'inline-audio-fractional-'));
+ const mixing = makeInlineAudioMixing(dir, SAMPLE_RATE);
+
+ try {
+ // Write out of order to also exercise sparse compact-file placement.
+ mixing.addAsset({
+ asset: makeAsset({id: 'fractional', frame: 83, startInVideo: 81}),
+ fps: 24.87,
+ totalNumberOfFrames: 41,
+ firstFrame: 79,
+ trimLeftOffset: 0.0021487736228385534,
+ trimRightOffset: 0,
+ });
+ mixing.addAsset({
+ asset: makeAsset({id: 'fractional', frame: 81, startInVideo: 81}),
+ fps: 24.87,
+ totalNumberOfFrames: 41,
+ firstFrame: 79,
+ trimLeftOffset: 0.0021487736228385534,
+ trimRightOffset: 0,
+ });
+ await finish(mixing);
+
+ const [track] = mixing.getListOfAssets();
+ const wav = readFileSync(track.outName);
+ const preprocessed = inlineAudioTrackToPreprocessedAudioTrack({
+ track,
+ relativeToInSamples: 0,
+ padToDurationInSamples: null,
+ });
+ expect(preprocessed.filter.pad_start).toBe('adelay=3756S|3756S|3756S');
+ // Legacy absolute placement for frame 83 is sample 7617. Relative to the
+ // compact track's sample 3756 origin, its first sample must be at 3861.
+ expect(wav.readInt16LE(44 + 3860 * 2 * 2)).toBe(0);
+ expect(wav.readInt16LE(44 + 3861 * 2 * 2)).toBe(1000);
} finally {
mixing.cleanup();
rmSync(dir, {recursive: true, force: true});
@@ -148,10 +193,9 @@ test(
});
expect(merged).not.toBeNull();
- expect(merged?.startInSeconds).toBe(startFrames[0] / FPS);
- expect(merged?.durationInSeconds).toBeCloseTo(
- (startFrames.at(-1)! - startFrames[0] + 1) / FPS,
- 8,
+ expect(merged?.startInSamples).toBe((startFrames[0] * SAMPLE_RATE) / FPS);
+ expect(merged?.durationInSamples).toBe(
+ ((startFrames.at(-1)! - startFrames[0] + 1) * SAMPLE_RATE) / FPS,
);
expect(statSync(merged!.outName).size).toBeLessThan(4_000_000);
} finally {