🎥 Make videos programmatically with React
@remotion/renderer: Compensate libfdk_aac priming when muxing ADTS into MP4/MOV
Nicolas Coutureau6ee2cd285284
diff --git a/packages/renderer/src/aac-priming.ts b/packages/renderer/src/aac-priming.tsnew file mode 100644index 0000000000..f08fb72379--- /dev/null+++ b/packages/renderer/src/aac-priming.ts@@ -0,0 +1,36 @@+import type {AudioCodec} from './options/audio-codec';++// libfdk_aac (AAC-LC) prepends 2048 samples of encoder delay ("priming") to+// its output. The audio is encoded to ADTS first (compressAudio, combineAudio),+// and ADTS cannot signal that delay, so stream-copying it into an MP4 or MOV+// produced an edit list starting at media time 0: players then played the+// priming as content and the audio was 2048 / sampleRate late (42.67ms at+// 48kHz). Shifting the ADTS input back by the delay makes the muxer write an+// edit list that skips the priming, like encoding straight to MP4 does.+export const LIBFDK_AAC_LC_ENCODER_DELAY_IN_SAMPLES = 2048;++export const getAacPrimingInputArgs = ({+ audioCodec,+ sampleRate,+ outputExtension,+}: {+ audioCodec: AudioCodec | null;+ sampleRate: number;+ outputExtension: string | null;+}): string[] => {+ if (audioCodec !== 'aac') {+ return [];+ }++ // Only containers with edit lists can carry the shift+ const extension = outputExtension?.toLowerCase();+ if (extension !== 'mp4' && extension !== 'mov') {+ return [];+ }++ const delayInMicroseconds = Math.round(+ (LIBFDK_AAC_LC_ENCODER_DELAY_IN_SAMPLES / sampleRate) * 1_000_000,+ );++ return ['-itsoffset', `-${delayInMicroseconds}us`];+};diff --git a/packages/renderer/src/combine-chunks.ts b/packages/renderer/src/combine-chunks.tsindex 26b217f265..be286df3f3 100644--- a/packages/renderer/src/combine-chunks.ts+++ b/packages/renderer/src/combine-chunks.ts@@ -237,6 +237,8 @@ export const internalCombineChunks = async ({cancelSignal,metadata,numberOfGifLoops,+ audioCodec: resolvedAudioCodec,+ sampleRate,});onProgress({totalProgress: 1, frames: numberOfFrames});rmSync(filelistDir, {recursive: true});diff --git a/packages/renderer/src/mux-video-and-audio.ts b/packages/renderer/src/mux-video-and-audio.tsindex 230bd6c1bb..ad0d00ec69 100644--- a/packages/renderer/src/mux-video-and-audio.ts+++ b/packages/renderer/src/mux-video-and-audio.ts@@ -1,3 +1,4 @@+import {getAacPrimingInputArgs} from './aac-priming';import {callFf} from './call-ffmpeg';import {convertNumberOfGifLoopsToFfmpegSyntax} from './convert-number-of-gif-loops-to-ffmpeg';import {getExtensionOfFilename} from './get-extension-of-filename';@@ -6,6 +7,7 @@ import type {LogLevel} from './log-level';import {Log} from './logger';import type {CancelSignal} from './make-cancel-signal';import {makeMetadataArgs} from './make-metadata-args';+import type {AudioCodec} from './options/audio-codec';import {parseFfmpegProgress} from './parse-ffmpeg-progress';import {truthy} from './truthy';@@ -21,6 +23,8 @@ export const muxVideoAndAudio = async ({cancelSignal,metadata,numberOfGifLoops,+ audioCodec,+ sampleRate,}: {videoOutput: string | null;audioOutput: string | null;@@ -33,6 +37,8 @@ export const muxVideoAndAudio = async ({cancelSignal: CancelSignal | undefined;metadata?: Record<string, string> | null;numberOfGifLoops: number | null;+ audioCodec: AudioCodec | null;+ sampleRate: number;}) => {const startTime = Date.now();Log.verbose({indent, logLevel}, 'Muxing video and audio together');@@ -45,6 +51,13 @@ export const muxVideoAndAudio = async ({'-hide_banner',videoOutput ? '-i' : null,videoOutput,+ ...(audioOutput+ ? getAacPrimingInputArgs({+ audioCodec,+ sampleRate,+ outputExtension,+ })+ : []),audioOutput ? '-i' : null,audioOutput,videoOutput ? '-c:v' : null,diff --git a/packages/renderer/src/stitch-frames-to-video.ts b/packages/renderer/src/stitch-frames-to-video.tsindex aaca541b82..d9e4d71d48 100644--- a/packages/renderer/src/stitch-frames-to-video.ts+++ b/packages/renderer/src/stitch-frames-to-video.ts@@ -1,6 +1,7 @@import {cpSync, promises, rmSync} from 'node:fs';import path from 'node:path';import type {_InternalTypes} from 'remotion';+import {getAacPrimingInputArgs} from './aac-priming';import type {RenderMediaOnDownload} from './assets/download-and-map-assets-to-file';import type {RenderAssetInfo} from './assets/download-map';import {cleanDownloadMap} from './assets/download-map';@@ -408,7 +409,19 @@ const innerStitchFramesToVideo = async (? ['-filter_complex', 'split[v],palettegen,[v]paletteuse']: null,]),- audio && !separateAudioTo ? ['-i', audio, '-c:a', 'copy'] : ['-an'],+ audio && !separateAudioTo+ ? [+ ...getAacPrimingInputArgs({+ audioCodec: resolvedAudioCodec,+ sampleRate,+ outputExtension,+ }),+ '-i',+ audio,+ '-c:a',+ 'copy',+ ]+ : ['-an'],numberOfGifLoops === null? null: ['-loop', convertNumberOfGifLoopsToFfmpegSyntax(numberOfGifLoops)],diff --git a/packages/renderer/src/test/aac-priming.test.ts b/packages/renderer/src/test/aac-priming.test.tsnew file mode 100644index 0000000000..e4f964d3ff--- /dev/null+++ b/packages/renderer/src/test/aac-priming.test.ts@@ -0,0 +1,41 @@+import {describe, expect, test} from 'bun:test';+import {getAacPrimingInputArgs} from '../aac-priming';++describe('AAC priming compensation', () => {+ test('Shifts AAC input back by 2048 samples at 48kHz', () => {+ expect(+ getAacPrimingInputArgs({+ audioCodec: 'aac',+ sampleRate: 48000,+ outputExtension: 'mp4',+ }),+ ).toEqual(['-itsoffset', '-42667us']);+ });+ test('Scales with the sample rate', () => {+ expect(+ getAacPrimingInputArgs({+ audioCodec: 'aac',+ sampleRate: 44100,+ outputExtension: 'mov',+ }),+ ).toEqual(['-itsoffset', '-46440us']);+ });+ test('Leaves other audio codecs untouched', () => {+ expect(+ getAacPrimingInputArgs({+ audioCodec: 'opus',+ sampleRate: 48000,+ outputExtension: 'mp4',+ }),+ ).toEqual([]);+ });+ test('Leaves containers without edit lists untouched', () => {+ expect(+ getAacPrimingInputArgs({+ audioCodec: 'aac',+ sampleRate: 48000,+ outputExtension: 'mkv',+ }),+ ).toEqual([]);+ });+});