🎥 Make videos programmatically with React
`@remotion/renderer`: Use clip-relative WAV files for inline audio
JonnyBurger942a6ff38295
diff --git a/packages/core/src/CompositionManager.tsx b/packages/core/src/CompositionManager.tsxindex 44e6289fe8..09c0205069 100644--- a/packages/core/src/CompositionManager.tsx+++ b/packages/core/src/CompositionManager.tsx@@ -162,6 +162,7 @@ export type InlineAudioAsset = {id: string;audio: Int16Array | number[];frame: number;+ startInVideo?: number;timestamp: number;duration: number;toneFrequency: number;diff --git a/packages/media/src/audio/audio-for-rendering.tsx b/packages/media/src/audio/audio-for-rendering.tsxindex 7c50878511..7b1fb75edc 100644--- a/packages/media/src/audio/audio-for-rendering.tsx+++ b/packages/media/src/audio/audio-for-rendering.tsx@@ -68,6 +68,9 @@ export const AudioForRendering: React.FC<AudioProps> = ({const [initialRequestInit] = useState(requestInit);const sequenceContext = useContext(Internals.SequenceContext);+ const startInVideo = sequenceContext+ ? sequenceContext.cumulatedFrom + sequenceContext.relativeFrom+ : 0;// Generate a string that's as unique as possible for this asset// but at the same time the same on all threads@@ -238,6 +241,7 @@ export const AudioForRendering: React.FC<AudioProps> = ({? audio.data: Array.from(audio.data),frame: absoluteFrame,+ startInVideo,timestamp: audio.timestamp,duration:(audio.numberOfFrames / getTargetSampleRate()) * 1_000_000,@@ -277,6 +281,7 @@ export const AudioForRendering: React.FC<AudioProps> = ({playbackRate,registerRenderAsset,src,+ startInVideo,startsAt,unregisterRenderAsset,volumeProp,diff --git a/packages/media/src/video/video-for-rendering.tsx b/packages/media/src/video/video-for-rendering.tsxindex 1de4f585c7..03f80ff27b 100644--- a/packages/media/src/video/video-for-rendering.tsx+++ b/packages/media/src/video/video-for-rendering.tsx@@ -109,6 +109,9 @@ export const VideoForRendering: React.FC<InnerVideoProps> = ({);const startsAt = Internals.useMediaStartsAt();const sequenceContext = useContext(Internals.SequenceContext);+ const startInVideo = sequenceContext+ ? sequenceContext.cumulatedFrom + sequenceContext.relativeFrom+ : 0;// Generate a string that's as unique as possible for this asset// but at the same time the same on all threads@@ -393,6 +396,7 @@ export const VideoForRendering: React.FC<InnerVideoProps> = ({? audio.data: Array.from(audio.data),frame: absoluteFrame,+ startInVideo,timestamp: audio.timestamp,duration:(audio.numberOfFrames / getTargetSampleRate()) * 1_000_000,@@ -432,6 +436,7 @@ export const VideoForRendering: React.FC<InnerVideoProps> = ({playbackRate,registerRenderAsset,src,+ startInVideo,startsAt,unregisterRenderAsset,volumeProp,diff --git a/packages/renderer/src/assets/inline-audio-mixing.ts b/packages/renderer/src/assets/inline-audio-mixing.tsindex b8a69d6969..b357016e75 100644--- a/packages/renderer/src/assets/inline-audio-mixing.ts+++ b/packages/renderer/src/assets/inline-audio-mixing.ts@@ -38,13 +38,21 @@ const correctFloatingPointError = (value: number): number => {const BIT_DEPTH = 16;const BYTES_PER_SAMPLE = BIT_DEPTH / 8;const NUMBER_OF_CHANNELS = 2;+const WAV_HEADER_SIZE = 44;++export type InlineAudioTrack = {+ outName: string;+ startInSeconds: number;+ durationInSeconds: number;+};export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {const folderToAdd = makeAndReturn(dir, 'remotion-inline-audio-mixing');- // asset id -> file descriptorconst openFiles: Record<string, number> = {};const writtenHeaders: Record<string, boolean> = {};const toneFrequencies: Record<string, number> = {};+ const startTimesInSeconds: Record<string, number> = {};+ const writtenDataSizes: Record<string, number> = {};const cleanup = () => {for (const fileName of Object.keys(openFiles)) {@@ -57,29 +65,25 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {deleteDirectory(folderToAdd);};- const getListOfAssets = () => {- return Object.keys(openFiles);+ const getListOfAssets = (): InlineAudioTrack[] => {+ return Object.keys(writtenHeaders)+ .map((outName) => ({+ outName,+ startInSeconds: startTimesInSeconds[outName],+ durationInSeconds:+ writtenDataSizes[outName] /+ (NUMBER_OF_CHANNELS * sampleRate * BYTES_PER_SAMPLE),+ }))+ .sort((a, b) => a.startInSeconds - b.startInSeconds);};const getFilePath = (asset: InlineAudioAsset) => {return path.join(folderToAdd, `${asset.id}.wav`);};- const ensureAsset = ({- asset,- fps,- totalNumberOfFrames,- trimLeftOffset,- trimRightOffset,- }: {- asset: InlineAudioAsset;- fps: number;- totalNumberOfFrames: number;- trimLeftOffset: number;- trimRightOffset: number;- }) => {+ const ensureAsset = (asset: InlineAudioAsset) => {const filePath = getFilePath(asset);- if (!openFiles[filePath]) {+ if (openFiles[filePath] === undefined) {openFiles[filePath] = fs.openSync(filePath, 'w');}@@ -88,43 +92,21 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {}writtenHeaders[filePath] = true;-- const expectedDataSize = Math.round(- (totalNumberOfFrames / fps - trimLeftOffset + trimRightOffset) *- NUMBER_OF_CHANNELS *- sampleRate *- BYTES_PER_SAMPLE,- );-- const expectedSize = 40 + expectedDataSize;+ writtenDataSizes[filePath] = 0;const fd = openFiles[filePath];writeSync(fd, new Uint8Array([0x52, 0x49, 0x46, 0x46]), 0, 4, 0); // "RIFF"- writeSync(- fd,- new Uint8Array(numberTo32BiIntLittleEndian(expectedSize)),- 0,- 4,- 4,- ); // Remaining size+ writeSync(fd, numberTo32BiIntLittleEndian(36), 0, 4, 4); // Remaining sizewriteSync(fd, new Uint8Array([0x57, 0x41, 0x56, 0x45]), 0, 4, 8); // "WAVE"writeSync(fd, new Uint8Array([0x66, 0x6d, 0x74, 0x20]), 0, 4, 12); // "fmt "writeSync(fd, new Uint8Array([BIT_DEPTH, 0x00, 0x00, 0x00]), 0, 4, 16); // fmt chunk size = 16writeSync(fd, new Uint8Array([0x01, 0x00]), 0, 2, 20); // Audio format (PCM) = 1, set 3 if float32 would be truewriteSync(fd, new Uint8Array([NUMBER_OF_CHANNELS, 0x00]), 0, 2, 22); // Number of channels+ writeSync(fd, numberTo32BiIntLittleEndian(sampleRate), 0, 4, 24); // Sample ratewriteSync(fd,- new Uint8Array(numberTo32BiIntLittleEndian(sampleRate)),- 0,- 4,- 24,- ); // Sample rate- writeSync(- fd,- new Uint8Array(- numberTo32BiIntLittleEndian(- sampleRate * NUMBER_OF_CHANNELS * BYTES_PER_SAMPLE,- ),+ numberTo32BiIntLittleEndian(+ sampleRate * NUMBER_OF_CHANNELS * BYTES_PER_SAMPLE,),0,4,@@ -132,22 +114,14 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {); // Byte ratewriteSync(fd,- new Uint8Array(- numberTo16BitLittleEndian(NUMBER_OF_CHANNELS * BYTES_PER_SAMPLE),- ),+ numberTo16BitLittleEndian(NUMBER_OF_CHANNELS * BYTES_PER_SAMPLE),0,2,32,); // Block alignwriteSync(fd, numberTo16BitLittleEndian(BIT_DEPTH), 0, 2, 34); // Bits per samplewriteSync(fd, new Uint8Array([0x64, 0x61, 0x74, 0x61]), 0, 4, 36); // "data"- writeSync(- fd,- new Uint8Array(numberTo32BiIntLittleEndian(expectedDataSize)),- 0,- 4,- 40,- ); // Remaining size+ writeSync(fd, numberTo32BiIntLittleEndian(0), 0, 4, 40); // Data size};const finish = async ({@@ -163,7 +137,15 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {cancelSignal: CancelSignal | undefined;sampleRate: number;}) => {- for (const fileName of Object.keys(openFiles)) {+ for (const fileName of Object.keys(writtenHeaders)) {+ const fd = openFiles[fileName];+ const dataSize = Math.max(0, fs.fstatSync(fd).size - WAV_HEADER_SIZE);+ writtenDataSizes[fileName] = dataSize;+ writeSync(fd, numberTo32BiIntLittleEndian(36 + dataSize), 0, 4, 4);+ writeSync(fd, numberTo32BiIntLittleEndian(dataSize), 0, 4, 40);+ fs.closeSync(fd);+ delete openFiles[fileName];+const frequency = toneFrequencies[fileName];if (frequency === 1) {continue;@@ -180,10 +162,6 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {cancelSignal,sampleRate: finishSampleRate,});- try {- fs.closeSync(openFiles[fileName]);- } catch {}-fs.renameSync(tmpFile, fileName);}};@@ -203,16 +181,9 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {trimLeftOffset: number;trimRightOffset: number;}) => {- ensureAsset({- asset,- fps,- totalNumberOfFrames,- trimLeftOffset,- trimRightOffset,- });+ ensureAsset(asset);const filePath = getFilePath(asset);-if (toneFrequencies[filePath] !== undefined &&toneFrequencies[filePath] !== asset.toneFrequency@@ -225,6 +196,24 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {const fileDescriptor = openFiles[filePath];toneFrequencies[filePath] = asset.toneFrequency;+ const assetStartInVideo = asset.startInVideo ?? firstFrame;+ const firstFrameForAsset = Math.max(assetStartInVideo, firstFrame);+ const startsAtRenderStart = firstFrameForAsset === firstFrame;+ const startInSeconds = Math.max(+ 0,+ (firstFrameForAsset - firstFrame) / fps - trimLeftOffset,+ );+ if (+ startTimesInSeconds[filePath] !== undefined &&+ Math.abs(startTimesInSeconds[filePath] - startInSeconds) > 0.00001+ ) {+ throw new Error(+ `The start time for inline audio asset ${asset.id} changed from ${startTimesInSeconds[filePath]} to ${startInSeconds}`,+ );+ }++ startTimesInSeconds[filePath] = startInSeconds;+let arr = new Int16Array(asset.audio);const isFirst = asset.frame === firstFrame;const isLast = asset.frame === totalNumberOfFrames + firstFrame - 1;@@ -252,7 +241,8 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {}const positionInSeconds =- (asset.frame - firstFrame) / fps - (isFirst ? 0 : trimLeftOffset);+ (asset.frame - firstFrameForAsset) / fps -+ (isFirst || !startsAtRenderStart ? 0 : trimLeftOffset);// Always rounding down to ensure there are no gaps when the samples don't align// In @remotion/media, we also round down the sample start timestamp and round up the end timestamp@@ -264,16 +254,15 @@ export const makeInlineAudioMixing = (dir: string, sampleRate: number) => {BYTES_PER_SAMPLE;writeSync(- // fsfileDescriptor,- // dataarr,- // offset of data0,- // lengtharr.byteLength,- // position- 44 + position,+ WAV_HEADER_SIZE + position,+ );+ writtenDataSizes[filePath] = Math.max(+ writtenDataSizes[filePath],+ position + arr.byteLength,);};diff --git a/packages/renderer/src/create-audio.ts b/packages/renderer/src/create-audio.tsindex ccc1fc2048..0263f5e3a7 100644--- a/packages/renderer/src/create-audio.ts+++ b/packages/renderer/src/create-audio.ts@@ -11,6 +11,10 @@ import type {LogLevel} from './log-level';import {Log} from './logger';import type {CancelSignal} from './make-cancel-signal';import {mergeAudioTrack} from './merge-audio-track';+import {+ inlineAudioTrackToPreprocessedAudioTrack,+ mergeInlineAudioTracks,+} from './merge-inline-audio-tracks';import type {AudioCodec} from './options/audio-codec';import {getExtensionFromAudioCodec} from './options/audio-codec';import type {PreprocessedAudioTrack} from './preprocess-audio-track';@@ -138,17 +142,31 @@ export const createAudio = async ({});const inlinedAudio = downloadMap.inlineAudioMixing.getListOfAssets();+ const mergedInlineAudio = await mergeInlineAudioTracks({+ tracks: inlinedAudio,+ downloadMap,+ remotionRoot,+ indent,+ logLevel,+ binariesDirectory,+ cancelSignal,+ fps,+ chunkLengthInSeconds,+ sampleRate,+ });const preprocessed: PreprocessedAudioTrack[] = [...audioTracks.filter(truthy),- ...inlinedAudio.map((asset) => ({- outName: asset,- filter: {- filter: null,- pad_start: null,- pad_end: null,- },- })),+ ...(mergedInlineAudio+ ? [+ inlineAudioTrackToPreprocessedAudioTrack({+ track: mergedInlineAudio,+ relativeToInSeconds: 0,+ padToDurationInSeconds: chunkLengthInSeconds,+ sampleRate,+ }),+ ]+ : []),];const merged = path.join(downloadMap.audioPreprocessing, 'merged.wav');const extension = getExtensionFromAudioCodec(audioCodec);diff --git a/packages/renderer/src/merge-inline-audio-tracks.ts b/packages/renderer/src/merge-inline-audio-tracks.tsnew file mode 100644index 0000000000..5727ced39e--- /dev/null+++ b/packages/renderer/src/merge-inline-audio-tracks.ts@@ -0,0 +1,138 @@+import path from 'node:path';+import type {DownloadMap} from './assets/download-map';+import type {InlineAudioTrack} from './assets/inline-audio-mixing';+import {chunk} from './chunk';+import {deleteDirectory} from './delete-directory';+import type {LogLevel} from './log-level';+import type {CancelSignal} from './make-cancel-signal';+import {mergeAudioTrack} from './merge-audio-track';+import type {PreprocessedAudioTrack} from './preprocess-audio-track';++const MAX_INLINE_AUDIO_INPUTS = 10;++export const inlineAudioTrackToPreprocessedAudioTrack = ({+ track,+ relativeToInSeconds,+ padToDurationInSeconds,+ sampleRate,+}: {+ track: InlineAudioTrack;+ relativeToInSeconds: number;+ padToDurationInSeconds: number | null;+ sampleRate: number;+}): PreprocessedAudioTrack => {+ const delayInSamples = Math.max(+ 0,+ Math.round((track.startInSeconds - relativeToInSeconds) * sampleRate),+ );+ const padAtEndInSamples =+ padToDurationInSeconds === null+ ? 0+ : Math.max(+ 0,+ Math.round(padToDurationInSeconds * sampleRate) -+ delayInSamples -+ Math.round(track.durationInSeconds * sampleRate),+ );++ return {+ outName: track.outName,+ filter: {+ pad_start:+ delayInSamples === 0+ ? null+ : `adelay=${new Array(3).fill(`${delayInSamples}S`).join('|')}`,+ pad_end:+ padAtEndInSamples === 0 ? null : `apad=pad_len=${padAtEndInSamples}`,+ },+ };+};++export const mergeInlineAudioTracks = async ({+ tracks,+ downloadMap,+ remotionRoot,+ indent,+ logLevel,+ binariesDirectory,+ cancelSignal,+ fps,+ chunkLengthInSeconds,+ sampleRate,+}: {+ tracks: InlineAudioTrack[];+ downloadMap: DownloadMap;+ remotionRoot: string;+ indent: boolean;+ logLevel: LogLevel;+ binariesDirectory: string | null;+ cancelSignal: CancelSignal | undefined;+ fps: number;+ chunkLengthInSeconds: number;+ sampleRate: number;+}): Promise<InlineAudioTrack | null> => {+ let currentTracks = tracks.sort(+ (a, b) => a.startInSeconds - b.startInSeconds,+ );+ let level = 0;++ while (currentTracks.length > 1) {+ const groups = chunk(currentTracks, MAX_INLINE_AUDIO_INPUTS);+ currentTracks = await Promise.all(+ groups.map(async (group, index): Promise<InlineAudioTrack> => {+ if (group.length === 1) {+ return group[0];+ }++ const [{startInSeconds}] = group;+ const durationInSamples = Math.max(+ ...group.map((track) => {+ return (+ Math.round((track.startInSeconds - startInSeconds) * sampleRate) ++ Math.round(track.durationInSeconds * sampleRate)+ );+ }),+ );+ const outName = path.join(+ downloadMap.audioMixing,+ `inline-${level}-${index}.wav`,+ );++ await mergeAudioTrack({+ files: group.map((track) =>+ inlineAudioTrackToPreprocessedAudioTrack({+ track,+ relativeToInSeconds: startInSeconds,+ padToDurationInSeconds: null,+ sampleRate,+ }),+ ),+ outName,+ downloadMap,+ remotionRoot,+ indent,+ logLevel,+ binariesDirectory,+ cancelSignal,+ onProgress: () => undefined,+ fps,+ chunkLengthInSeconds,+ sampleRate,+ });++ for (const track of group) {+ deleteDirectory(track.outName);+ }++ return {+ outName,+ startInSeconds,+ durationInSeconds: durationInSamples / sampleRate,+ };+ }),+ );+ level++;+ }++ return currentTracks[0] ?? null;+};diff --git a/packages/renderer/src/test/inline-audio-mixing.test.ts b/packages/renderer/src/test/inline-audio-mixing.test.tsnew file mode 100644index 0000000000..3f4a479843--- /dev/null+++ b/packages/renderer/src/test/inline-audio-mixing.test.ts@@ -0,0 +1,149 @@+import {expect, test} from 'bun:test';+import {mkdtempSync, readFileSync, rmSync, statSync} from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import type {InlineAudioAsset} from 'remotion/no-react';+import {cleanDownloadMap, makeDownloadMap} from '../assets/download-map';+import {makeInlineAudioMixing} from '../assets/inline-audio-mixing';+import {mergeInlineAudioTracks} from '../merge-inline-audio-tracks';++const SAMPLE_RATE = 48_000;+const FPS = 30;+const SAMPLES_PER_FRAME = SAMPLE_RATE / FPS;++const makeAsset = ({+ id,+ frame,+ startInVideo,+}: {+ id: string;+ frame: number;+ startInVideo: number | undefined;+}): InlineAudioAsset => ({+ type: 'inline-audio',+ id,+ audio: new Int16Array(SAMPLES_PER_FRAME * 2).fill(1000),+ frame,+ startInVideo,+ timestamp: 0,+ duration: 1_000_000 / FPS,+ toneFrequency: 1,+});++const finish = (mixing: ReturnType<typeof makeInlineAudioMixing>) => {+ return mixing.finish({+ indent: false,+ logLevel: 'error',+ binariesDirectory: null,+ cancelSignal: undefined,+ sampleRate: SAMPLE_RATE,+ });+};++test('writes late inline audio relative to its clip origin', async () => {+ const dir = mkdtempSync(path.join(os.tmpdir(), 'inline-audio-relative-'));+ const mixing = makeInlineAudioMixing(dir, SAMPLE_RATE);++ try {+ // Deliberately write the second frame first to ensure the origin does not+ // depend on renderer concurrency or completion order.+ mixing.addAsset({+ asset: makeAsset({id: 'late', frame: 301, startInVideo: 300}),+ fps: FPS,+ totalNumberOfFrames: 330,+ firstFrame: 0,+ trimLeftOffset: 0,+ trimRightOffset: 0,+ });+ mixing.addAsset({+ asset: makeAsset({id: 'late', frame: 300, startInVideo: 300}),+ fps: FPS,+ totalNumberOfFrames: 330,+ firstFrame: 0,+ trimLeftOffset: 0,+ trimRightOffset: 0,+ });+ await finish(mixing);++ const [track] = mixing.getListOfAssets();+ const wav = readFileSync(track.outName);+ expect(track.startInSeconds).toBe(10);+ expect(track.durationInSeconds).toBeCloseTo(2 / FPS, 8);+ expect(wav.length).toBe(44 + SAMPLES_PER_FRAME * 2 * 2 * 2);+ expect(wav.readUInt32LE(4)).toBe(wav.length - 8);+ expect(wav.readUInt32LE(40)).toBe(wav.length - 44);+ } finally {+ mixing.cleanup();+ rmSync(dir, {recursive: true, force: true});+ }+});++test('keeps a composition-relative fallback for an unknown clip origin', async () => {+ const dir = mkdtempSync(path.join(os.tmpdir(), 'inline-audio-fallback-'));+ const mixing = makeInlineAudioMixing(dir, SAMPLE_RATE);++ try {+ mixing.addAsset({+ asset: makeAsset({id: 'unknown', frame: 3, startInVideo: undefined}),+ fps: FPS,+ totalNumberOfFrames: 4,+ firstFrame: 0,+ trimLeftOffset: 0,+ trimRightOffset: 0,+ });+ await finish(mixing);++ const [track] = mixing.getListOfAssets();+ expect(track.startInSeconds).toBe(0);+ expect(track.durationInSeconds).toBeCloseTo(4 / FPS, 8);+ } finally {+ mixing.cleanup();+ rmSync(dir, {recursive: true, force: true});+ }+});++test(+ 'pre-merges many sequential clips without materializing their timeline offsets',+ async () => {+ const downloadMap = makeDownloadMap(SAMPLE_RATE);++ try {+ for (let index = 0; index < 12; index++) {+ downloadMap.inlineAudioMixing.addAsset({+ asset: makeAsset({+ id: `clip-${index}`,+ frame: 300 + index,+ startInVideo: 300 + index,+ }),+ fps: FPS,+ totalNumberOfFrames: 360,+ firstFrame: 0,+ trimLeftOffset: 0,+ trimRightOffset: 0,+ });+ }++ await finish(downloadMap.inlineAudioMixing);+ const merged = await mergeInlineAudioTracks({+ tracks: downloadMap.inlineAudioMixing.getListOfAssets(),+ downloadMap,+ remotionRoot: process.cwd(),+ indent: false,+ logLevel: 'error',+ binariesDirectory: null,+ cancelSignal: undefined,+ fps: FPS,+ chunkLengthInSeconds: 12,+ sampleRate: SAMPLE_RATE,+ });++ expect(merged).not.toBeNull();+ expect(merged?.startInSeconds).toBe(10);+ expect(merged?.durationInSeconds).toBeCloseTo(12 / FPS, 8);+ expect(statSync(merged!.outName).size).toBeLessThan(100_000);+ } finally {+ cleanDownloadMap(downloadMap);+ }+ },+ {timeout: 30_000},+);