🎥 Make videos programmatically with React
Internal tests: Stress test clip-relative inline audio
JonnyBurgerebfc573f9745
diff --git a/packages/example/src/AudioTesting/InlineAudioStress.tsx b/packages/example/src/AudioTesting/InlineAudioStress.tsxnew file mode 100644index 0000000000..b94eb80e6a--- /dev/null+++ b/packages/example/src/AudioTesting/InlineAudioStress.tsx@@ -0,0 +1,91 @@+import {Audio} from '@remotion/media';+import {Sequence, useCurrentFrame} from 'remotion';++export type InlineAudioStressClip = {+ readonly outerFrom: number;+ readonly middleFrom: number;+ readonly innerFrom: number;+ readonly durationInFrames: number;+};++export type InlineAudioStressRange = {+ readonly from: number;+ readonly durationInFrames: number;+};++export type InlineAudioStressProps = {+ readonly src: string;+ readonly clips: InlineAudioStressClip[];+ readonly intermittentOuterFrom: number;+ readonly intermittentInnerFrom: number;+ readonly intermittentDurationInFrames: number;+ readonly intermittentRanges: InlineAudioStressRange[];+};++const IntermittentAudio: React.FC<{+ readonly src: string;+ readonly ranges: InlineAudioStressRange[];+}> = ({src, ranges}) => {+ const frame = useCurrentFrame();+ const isMounted = ranges.some(+ (range) =>+ frame >= range.from && frame < range.from + range.durationInFrames,+ );++ return isMounted ? <Audio src={src} loop volume={0.125} /> : null;+};++export const InlineAudioStress: React.FC<InlineAudioStressProps> = ({+ src,+ clips,+ intermittentOuterFrom,+ intermittentInnerFrom,+ intermittentDurationInFrames,+ intermittentRanges,+}) => {+ return (+ <>+ {clips.map((clip, index) => (+ <Sequence+ key={index}+ from={clip.outerFrom}+ durationInFrames={+ clip.middleFrom + clip.innerFrom + clip.durationInFrames+ }+ >+ <Sequence+ from={clip.middleFrom}+ durationInFrames={clip.innerFrom + clip.durationInFrames}+ >+ <Sequence+ from={clip.innerFrom}+ durationInFrames={clip.durationInFrames}+ >+ <Audio src={src} loop volume={0.25} />+ </Sequence>+ </Sequence>+ </Sequence>+ ))}+ <Sequence+ from={intermittentOuterFrom}+ durationInFrames={intermittentInnerFrom + intermittentDurationInFrames}+ >+ <Sequence+ from={intermittentInnerFrom}+ durationInFrames={intermittentDurationInFrames}+ >+ <IntermittentAudio src={src} ranges={intermittentRanges} />+ </Sequence>+ </Sequence>+ </>+ );+};++export const inlineAudioStressDefaultProps: InlineAudioStressProps = {+ src: '',+ clips: [],+ intermittentOuterFrom: 0,+ intermittentInnerFrom: 0,+ intermittentDurationInFrames: 1,+ intermittentRanges: [],+};diff --git a/packages/example/src/Root.tsx b/packages/example/src/Root.tsxindex e9d275fc8d..58d364f59d 100644--- a/packages/example/src/Root.tsx+++ b/packages/example/src/Root.tsx@@ -213,6 +213,10 @@ import {AudioSmoothnessSlicedVideoComp} from './AudioSmoothness/SlicedVideo';import {AudioSmoothnessTrimAfterLoopComp} from './AudioSmoothness/TrimAfterLoop';import {AudioSmoothnessTrimButtonComp} from './AudioSmoothness/TrimButton';import Amplify from './AudioTesting/Amplify';+import {+ InlineAudioStress,+ inlineAudioStressDefaultProps,+} from './AudioTesting/InlineAudioStress';import {Issue7568} from './AudioTesting/Issue7568';import {Issue10468,@@ -1805,6 +1809,15 @@ export const Index: React.FC = () => {durationInFrames={120}defaultProps={issue5758DefaultProps}/>+ <Composition+ id="inline-audio-stress"+ component={InlineAudioStress}+ width={16}+ height={16}+ fps={30}+ durationInFrames={620}+ defaultProps={inlineAudioStressDefaultProps}+ /><Compositionid="audio-issue-7568"component={Issue7568}diff --git a/packages/it-tests/src/ssr/render-media-sample-rate.test.ts b/packages/it-tests/src/ssr/render-media-sample-rate.test.tsindex 4af59107cc..b6bc79acef 100644--- a/packages/it-tests/src/ssr/render-media-sample-rate.test.ts+++ b/packages/it-tests/src/ssr/render-media-sample-rate.test.ts@@ -1,5 +1,12 @@import {expect, test} from 'bun:test';-import {mkdtempSync, readFileSync, rmSync} from 'node:fs';+import {+ existsSync,+ mkdtempSync,+ readFileSync,+ readdirSync,+ rmSync,+ statSync,+} from 'node:fs';import os from 'os';import path from 'path';import {@@ -131,6 +138,120 @@ const issue5758InputProps = {src: `data:audio/wav;base64,${issue5758Source.toString('base64')}`,};+const STRESS_FPS = 30;+const STRESS_SAMPLE_RATE = 48_000;+const STRESS_DURATION_IN_FRAMES = 620;+const STRESS_CLIP_COUNT = 423;+const STRESS_SOURCE_AMPLITUDE = 8000;+const STRESS_INTERMITTENT_START = 28;+const stressIntermittentRanges = [+ {from: 0, durationInFrames: 37},+ {from: 52, durationInFrames: 39},+ {from: 143, durationInFrames: 37},+ {from: 260, durationInFrames: 70},+ {from: 411, durationInFrames: 89},+];++let nextStressClipStart = 20;+const stressClips = new Array(STRESS_CLIP_COUNT).fill(null).map((_, index) => {+ if (index % 17 === 0) {+ nextStressClipStart += 4;+ } else if (index % 7 === 0) {+ nextStressClipStart++;+ }++ const middleFrom = (index * 7) % 8;+ const innerFrom = (index * 11) % 9;+ const clip = {+ outerFrom: nextStressClipStart - middleFrom - innerFrom,+ middleFrom,+ innerFrom,+ durationInFrames: 1,+ };+ nextStressClipStart++;+ return clip;+});++const stressClipStarts = new Set(+ stressClips.map((clip) => clip.outerFrom + clip.middleFrom + clip.innerFrom),+);++const makeConstantPcmWavDataUrl = () => {+ const channels = 2;+ const sourceDurationInFrames = 30;+ const samplesPerChannel =+ (STRESS_SAMPLE_RATE / STRESS_FPS) * sourceDurationInFrames;+ const dataSize = samplesPerChannel * channels * 2;+ const wav = Buffer.alloc(44 + dataSize);+ wav.write('RIFF', 0);+ wav.writeUInt32LE(36 + dataSize, 4);+ wav.write('WAVE', 8);+ wav.write('fmt ', 12);+ wav.writeUInt32LE(16, 16);+ wav.writeUInt16LE(1, 20);+ wav.writeUInt16LE(channels, 22);+ wav.writeUInt32LE(STRESS_SAMPLE_RATE, 24);+ wav.writeUInt32LE(STRESS_SAMPLE_RATE * channels * 2, 28);+ wav.writeUInt16LE(channels * 2, 32);+ wav.writeUInt16LE(16, 34);+ wav.write('data', 36);+ wav.writeUInt32LE(dataSize, 40);+ for (let offset = 44; offset < wav.length; offset += 2) {+ wav.writeInt16LE(STRESS_SOURCE_AMPLITUDE, offset);+ }++ return `data:audio/wav;base64,${wav.toString('base64')}`;+};++const inlineAudioStressInputProps = {+ src: makeConstantPcmWavDataUrl(),+ clips: stressClips,+ intermittentOuterFrom: 11,+ intermittentInnerFrom: 17,+ intermittentDurationInFrames: 560,+ intermittentRanges: stressIntermittentRanges,+};++const getInlineAudioTempSize = (+ directory: string,+ insideInlineAudioDirectory: boolean,+): number => {+ if (!existsSync(directory)) {+ return 0;+ }++ try {+ return readdirSync(directory, {withFileTypes: true}).reduce(+ (total, entry) => {+ const entryPath = path.join(directory, entry.name);+ if (entry.isFile()) {+ return insideInlineAudioDirectory+ ? total + statSync(entryPath).size+ : total;+ }++ if (!entry.isDirectory()) {+ return total;+ }++ return (+ total ++ getInlineAudioTempSize(+ entryPath,+ insideInlineAudioDirectory ||+ entry.name === 'remotion-inline-audio-mixing',+ )+ );+ },+ 0,+ );+ } catch {+ // The renderer can delete a temporary file between readdirSync() and+ // statSync() while the size monitor is running.+ return 0;+ }+};+const renderIssue10468Wav = async ({tmpDir,name,@@ -258,6 +379,96 @@ test({timeout: 180000},);+test(+ '@remotion/media keeps 423 nested and intermittently mounted audio clips sample-aligned',+ async () => {+ const tmpDir = mkdtempSync(path.join(os.tmpdir(), 'inline-audio-stress-'));+ const previousTmpDir = process.env.TMPDIR;+ process.env.TMPDIR = tmpDir;+ let peakInlineAudioTempSize = 0;+ const tempSizeMonitor = setInterval(() => {+ peakInlineAudioTempSize = Math.max(+ peakInlineAudioTempSize,+ getInlineAudioTempSize(tmpDir, false),+ );+ }, 5);+ try {+ const outputLocation = path.join(tmpDir, 'stress.wav');+ const composition = await selectComposition({+ id: 'inline-audio-stress',+ serveUrl: exampleBuild,+ inputProps: inlineAudioStressInputProps,+ });+ await renderMedia({+ outputLocation,+ codec: 'wav',+ serveUrl: exampleBuild,+ composition,+ inputProps: inlineAudioStressInputProps,+ sampleRate: STRESS_SAMPLE_RATE,+ concurrency: 4,+ logLevel: 'error',+ });++ const rendered = readPcmWav(outputLocation);+ expect(rendered.sampleRate).toBe(STRESS_SAMPLE_RATE);+ expect(rendered.channels).toBe(2);+ const samplesPerFrame = STRESS_SAMPLE_RATE / STRESS_FPS;+ const expectedSampleCount =+ STRESS_DURATION_IN_FRAMES * samplesPerFrame * rendered.channels;+ expect(rendered.samples.length).toBe(expectedSampleCount);++ const observedLevels = new Set<number>();+ let maximumMeanError = 0;+ for (let frame = 0; frame < STRESS_DURATION_IN_FRAMES; frame++) {+ const intermittentFrame = frame - STRESS_INTERMITTENT_START;+ const intermittentIsMounted = stressIntermittentRanges.some(+ (range) =>+ intermittentFrame >= range.from &&+ intermittentFrame < range.from + range.durationInFrames,+ );+ const expectedAmplitude =+ (stressClipStarts.has(frame) ? 2000 : 0) ++ (intermittentIsMounted ? 1000 : 0);+ observedLevels.add(expectedAmplitude);++ // Ignore a few samples at frame boundaries so this assertion is about+ // timeline placement rather than decoder edge interpolation.+ const firstSample = (frame * samplesPerFrame + 64) * rendered.channels;+ const lastSample =+ ((frame + 1) * samplesPerFrame - 64) * rendered.channels;+ let sum = 0;+ for (let sample = firstSample; sample < lastSample; sample++) {+ sum += Math.abs(rendered.samples[sample]);+ }++ const mean = sum / (lastSample - firstSample);+ maximumMeanError = Math.max(+ maximumMeanError,+ Math.abs(mean - expectedAmplitude),+ );+ }++ expect([...observedLevels].sort((a, b) => a - b)).toEqual([+ 0, 1000, 2000, 3000,+ ]);+ expect(maximumMeanError).toBeLessThanOrEqual(20);+ expect(peakInlineAudioTempSize).toBeGreaterThan(2_000_000);+ expect(peakInlineAudioTempSize).toBeLessThan(10_000_000);+ } finally {+ clearInterval(tempSizeMonitor);+ if (previousTmpDir === undefined) {+ delete process.env.TMPDIR;+ } else {+ process.env.TMPDIR = previousTmpDir;+ }++ rmSync(tmpDir, {recursive: true, force: true});+ }+ },+ {timeout: 300000},+);+test('Render video with default sampleRate should produce 48000 Hz audio',async () => {diff --git a/packages/renderer/src/test/inline-audio-mixing.test.ts b/packages/renderer/src/test/inline-audio-mixing.test.tsindex 3f4a479843..db721c197f 100644--- a/packages/renderer/src/test/inline-audio-mixing.test.ts+++ b/packages/renderer/src/test/inline-audio-mixing.test.ts@@ -103,24 +103,34 @@ test('keeps a composition-relative fallback for an unknown clip origin', async (});test(- 'pre-merges many sequential clips without materializing their timeline offsets',+ 'pre-merges 423 offset clips without materializing their timeline offsets',async () => {const downloadMap = makeDownloadMap(SAMPLE_RATE);try {- for (let index = 0; index < 12; index++) {+ let nextFrame = 300;+ const startFrames: number[] = [];+ for (let index = 0; index < 423; index++) {+ if (index % 17 === 0) {+ nextFrame += 4;+ } else if (index % 7 === 0) {+ nextFrame++;+ }++ startFrames.push(nextFrame);downloadMap.inlineAudioMixing.addAsset({asset: makeAsset({id: `clip-${index}`,- frame: 300 + index,- startInVideo: 300 + index,+ frame: nextFrame,+ startInVideo: nextFrame,}),fps: FPS,- totalNumberOfFrames: 360,+ totalNumberOfFrames: 1000,firstFrame: 0,trimLeftOffset: 0,trimRightOffset: 0,});+ nextFrame++;}await finish(downloadMap.inlineAudioMixing);@@ -138,12 +148,15 @@ test(});expect(merged).not.toBeNull();- expect(merged?.startInSeconds).toBe(10);- expect(merged?.durationInSeconds).toBeCloseTo(12 / FPS, 8);- expect(statSync(merged!.outName).size).toBeLessThan(100_000);+ expect(merged?.startInSeconds).toBe(startFrames[0] / FPS);+ expect(merged?.durationInSeconds).toBeCloseTo(+ (startFrames.at(-1)! - startFrames[0] + 1) / FPS,+ 8,+ );+ expect(statSync(merged!.outName).size).toBeLessThan(4_000_000);} finally {cleanDownloadMap(downloadMap);}},- {timeout: 30_000},+ {timeout: 120_000},);