Skip to content

🎥 Make videos programmatically with React

@remotion/renderer: Compensate libfdk_aac priming when muxing ADTS into MP4/MOV

Nicolas Coutureau6ee2cd285284
diff --git a/packages/renderer/src/aac-priming.ts b/packages/renderer/src/aac-priming.ts
new file mode 100644
index 0000000000..f08fb72379
--- /dev/null
+++ b/packages/renderer/src/aac-priming.ts
@@ -0,0 +1,36 @@
+import type {AudioCodec} from './options/audio-codec';
+
+// libfdk_aac (AAC-LC) prepends 2048 samples of encoder delay ("priming") to
+// its output. The audio is encoded to ADTS first (compressAudio, combineAudio),
+// and ADTS cannot signal that delay, so stream-copying it into an MP4 or MOV
+// produced an edit list starting at media time 0: players then played the
+// priming as content and the audio was 2048 / sampleRate late (42.67ms at
+// 48kHz). Shifting the ADTS input back by the delay makes the muxer write an
+// edit list that skips the priming, like encoding straight to MP4 does.
+export const LIBFDK_AAC_LC_ENCODER_DELAY_IN_SAMPLES = 2048;
+
+export const getAacPrimingInputArgs = ({
+ audioCodec,
+ sampleRate,
+ outputExtension,
+}: {
+ audioCodec: AudioCodec | null;
+ sampleRate: number;
+ outputExtension: string | null;
+}): string[] => {
+ if (audioCodec !== 'aac') {
+ return [];
+ }
+
+ // Only containers with edit lists can carry the shift
+ const extension = outputExtension?.toLowerCase();
+ if (extension !== 'mp4' && extension !== 'mov') {
+ return [];
+ }
+
+ const delayInMicroseconds = Math.round(
+ (LIBFDK_AAC_LC_ENCODER_DELAY_IN_SAMPLES / sampleRate) * 1_000_000,
+ );
+
+ return ['-itsoffset', `-${delayInMicroseconds}us`];
+};
diff --git a/packages/renderer/src/combine-chunks.ts b/packages/renderer/src/combine-chunks.ts
index 26b217f265..be286df3f3 100644
--- a/packages/renderer/src/combine-chunks.ts
+++ b/packages/renderer/src/combine-chunks.ts
@@ -237,6 +237,8 @@ export const internalCombineChunks = async ({
cancelSignal,
metadata,
numberOfGifLoops,
+ audioCodec: resolvedAudioCodec,
+ sampleRate,
});
onProgress({totalProgress: 1, frames: numberOfFrames});
rmSync(filelistDir, {recursive: true});
diff --git a/packages/renderer/src/mux-video-and-audio.ts b/packages/renderer/src/mux-video-and-audio.ts
index 230bd6c1bb..ad0d00ec69 100644
--- a/packages/renderer/src/mux-video-and-audio.ts
+++ b/packages/renderer/src/mux-video-and-audio.ts
@@ -1,3 +1,4 @@
+import {getAacPrimingInputArgs} from './aac-priming';
import {callFf} from './call-ffmpeg';
import {convertNumberOfGifLoopsToFfmpegSyntax} from './convert-number-of-gif-loops-to-ffmpeg';
import {getExtensionOfFilename} from './get-extension-of-filename';
@@ -6,6 +7,7 @@ import type {LogLevel} from './log-level';
import {Log} from './logger';
import type {CancelSignal} from './make-cancel-signal';
import {makeMetadataArgs} from './make-metadata-args';
+import type {AudioCodec} from './options/audio-codec';
import {parseFfmpegProgress} from './parse-ffmpeg-progress';
import {truthy} from './truthy';
@@ -21,6 +23,8 @@ export const muxVideoAndAudio = async ({
cancelSignal,
metadata,
numberOfGifLoops,
+ audioCodec,
+ sampleRate,
}: {
videoOutput: string | null;
audioOutput: string | null;
@@ -33,6 +37,8 @@ export const muxVideoAndAudio = async ({
cancelSignal: CancelSignal | undefined;
metadata?: Record<string, string> | null;
numberOfGifLoops: number | null;
+ audioCodec: AudioCodec | null;
+ sampleRate: number;
}) => {
const startTime = Date.now();
Log.verbose({indent, logLevel}, 'Muxing video and audio together');
@@ -45,6 +51,13 @@ export const muxVideoAndAudio = async ({
'-hide_banner',
videoOutput ? '-i' : null,
videoOutput,
+ ...(audioOutput
+ ? getAacPrimingInputArgs({
+ audioCodec,
+ sampleRate,
+ outputExtension,
+ })
+ : []),
audioOutput ? '-i' : null,
audioOutput,
videoOutput ? '-c:v' : null,
diff --git a/packages/renderer/src/stitch-frames-to-video.ts b/packages/renderer/src/stitch-frames-to-video.ts
index aaca541b82..d9e4d71d48 100644
--- a/packages/renderer/src/stitch-frames-to-video.ts
+++ b/packages/renderer/src/stitch-frames-to-video.ts
@@ -1,6 +1,7 @@
import {cpSync, promises, rmSync} from 'node:fs';
import path from 'node:path';
import type {_InternalTypes} from 'remotion';
+import {getAacPrimingInputArgs} from './aac-priming';
import type {RenderMediaOnDownload} from './assets/download-and-map-assets-to-file';
import type {RenderAssetInfo} from './assets/download-map';
import {cleanDownloadMap} from './assets/download-map';
@@ -408,7 +409,19 @@ const innerStitchFramesToVideo = async (
? ['-filter_complex', 'split[v],palettegen,[v]paletteuse']
: null,
]),
- audio && !separateAudioTo ? ['-i', audio, '-c:a', 'copy'] : ['-an'],
+ audio && !separateAudioTo
+ ? [
+ ...getAacPrimingInputArgs({
+ audioCodec: resolvedAudioCodec,
+ sampleRate,
+ outputExtension,
+ }),
+ '-i',
+ audio,
+ '-c:a',
+ 'copy',
+ ]
+ : ['-an'],
numberOfGifLoops === null
? null
: ['-loop', convertNumberOfGifLoopsToFfmpegSyntax(numberOfGifLoops)],
diff --git a/packages/renderer/src/test/aac-priming.test.ts b/packages/renderer/src/test/aac-priming.test.ts
new file mode 100644
index 0000000000..e4f964d3ff
--- /dev/null
+++ b/packages/renderer/src/test/aac-priming.test.ts
@@ -0,0 +1,41 @@
+import {describe, expect, test} from 'bun:test';
+import {getAacPrimingInputArgs} from '../aac-priming';
+
+describe('AAC priming compensation', () => {
+ test('Shifts AAC input back by 2048 samples at 48kHz', () => {
+ expect(
+ getAacPrimingInputArgs({
+ audioCodec: 'aac',
+ sampleRate: 48000,
+ outputExtension: 'mp4',
+ }),
+ ).toEqual(['-itsoffset', '-42667us']);
+ });
+ test('Scales with the sample rate', () => {
+ expect(
+ getAacPrimingInputArgs({
+ audioCodec: 'aac',
+ sampleRate: 44100,
+ outputExtension: 'mov',
+ }),
+ ).toEqual(['-itsoffset', '-46440us']);
+ });
+ test('Leaves other audio codecs untouched', () => {
+ expect(
+ getAacPrimingInputArgs({
+ audioCodec: 'opus',
+ sampleRate: 48000,
+ outputExtension: 'mp4',
+ }),
+ ).toEqual([]);
+ });
+ test('Leaves containers without edit lists untouched', () => {
+ expect(
+ getAacPrimingInputArgs({
+ audioCodec: 'aac',
+ sampleRate: 48000,
+ outputExtension: 'mkv',
+ }),
+ ).toEqual([]);
+ });
+});