Skip to content

🎥 Make videos programmatically with React

Internal tests: Stress test clip-relative inline audio

JonnyBurgerebfc573f9745
diff --git a/packages/example/src/AudioTesting/InlineAudioStress.tsx b/packages/example/src/AudioTesting/InlineAudioStress.tsx
new file mode 100644
index 0000000000..b94eb80e6a
--- /dev/null
+++ b/packages/example/src/AudioTesting/InlineAudioStress.tsx
@@ -0,0 +1,91 @@
+import {Audio} from '@remotion/media';
+import {Sequence, useCurrentFrame} from 'remotion';
+
+export type InlineAudioStressClip = {
+ readonly outerFrom: number;
+ readonly middleFrom: number;
+ readonly innerFrom: number;
+ readonly durationInFrames: number;
+};
+
+export type InlineAudioStressRange = {
+ readonly from: number;
+ readonly durationInFrames: number;
+};
+
+export type InlineAudioStressProps = {
+ readonly src: string;
+ readonly clips: InlineAudioStressClip[];
+ readonly intermittentOuterFrom: number;
+ readonly intermittentInnerFrom: number;
+ readonly intermittentDurationInFrames: number;
+ readonly intermittentRanges: InlineAudioStressRange[];
+};
+
+const IntermittentAudio: React.FC<{
+ readonly src: string;
+ readonly ranges: InlineAudioStressRange[];
+}> = ({src, ranges}) => {
+ const frame = useCurrentFrame();
+ const isMounted = ranges.some(
+ (range) =>
+ frame >= range.from && frame < range.from + range.durationInFrames,
+ );
+
+ return isMounted ? <Audio src={src} loop volume={0.125} /> : null;
+};
+
+export const InlineAudioStress: React.FC<InlineAudioStressProps> = ({
+ src,
+ clips,
+ intermittentOuterFrom,
+ intermittentInnerFrom,
+ intermittentDurationInFrames,
+ intermittentRanges,
+}) => {
+ return (
+ <>
+ {clips.map((clip, index) => (
+ <Sequence
+ key={index}
+ from={clip.outerFrom}
+ durationInFrames={
+ clip.middleFrom + clip.innerFrom + clip.durationInFrames
+ }
+ >
+ <Sequence
+ from={clip.middleFrom}
+ durationInFrames={clip.innerFrom + clip.durationInFrames}
+ >
+ <Sequence
+ from={clip.innerFrom}
+ durationInFrames={clip.durationInFrames}
+ >
+ <Audio src={src} loop volume={0.25} />
+ </Sequence>
+ </Sequence>
+ </Sequence>
+ ))}
+ <Sequence
+ from={intermittentOuterFrom}
+ durationInFrames={intermittentInnerFrom + intermittentDurationInFrames}
+ >
+ <Sequence
+ from={intermittentInnerFrom}
+ durationInFrames={intermittentDurationInFrames}
+ >
+ <IntermittentAudio src={src} ranges={intermittentRanges} />
+ </Sequence>
+ </Sequence>
+ </>
+ );
+};
+
+export const inlineAudioStressDefaultProps: InlineAudioStressProps = {
+ src: '',
+ clips: [],
+ intermittentOuterFrom: 0,
+ intermittentInnerFrom: 0,
+ intermittentDurationInFrames: 1,
+ intermittentRanges: [],
+};
diff --git a/packages/example/src/Root.tsx b/packages/example/src/Root.tsx
index e9d275fc8d..58d364f59d 100644
--- a/packages/example/src/Root.tsx
+++ b/packages/example/src/Root.tsx
@@ -213,6 +213,10 @@ import {AudioSmoothnessSlicedVideoComp} from './AudioSmoothness/SlicedVideo';
import {AudioSmoothnessTrimAfterLoopComp} from './AudioSmoothness/TrimAfterLoop';
import {AudioSmoothnessTrimButtonComp} from './AudioSmoothness/TrimButton';
import Amplify from './AudioTesting/Amplify';
+import {
+ InlineAudioStress,
+ inlineAudioStressDefaultProps,
+} from './AudioTesting/InlineAudioStress';
import {Issue7568} from './AudioTesting/Issue7568';
import {
Issue10468,
@@ -1805,6 +1809,15 @@ export const Index: React.FC = () => {
durationInFrames={120}
defaultProps={issue5758DefaultProps}
/>
+ <Composition
+ id="inline-audio-stress"
+ component={InlineAudioStress}
+ width={16}
+ height={16}
+ fps={30}
+ durationInFrames={620}
+ defaultProps={inlineAudioStressDefaultProps}
+ />
<Composition
id="audio-issue-7568"
component={Issue7568}
diff --git a/packages/it-tests/src/ssr/render-media-sample-rate.test.ts b/packages/it-tests/src/ssr/render-media-sample-rate.test.ts
index 4af59107cc..b6bc79acef 100644
--- a/packages/it-tests/src/ssr/render-media-sample-rate.test.ts
+++ b/packages/it-tests/src/ssr/render-media-sample-rate.test.ts
@@ -1,5 +1,12 @@
import {expect, test} from 'bun:test';
-import {mkdtempSync, readFileSync, rmSync} from 'node:fs';
+import {
+ existsSync,
+ mkdtempSync,
+ readFileSync,
+ readdirSync,
+ rmSync,
+ statSync,
+} from 'node:fs';
import os from 'os';
import path from 'path';
import {
@@ -131,6 +138,120 @@ const issue5758InputProps = {
src: `data:audio/wav;base64,${issue5758Source.toString('base64')}`,
};
+const STRESS_FPS = 30;
+const STRESS_SAMPLE_RATE = 48_000;
+const STRESS_DURATION_IN_FRAMES = 620;
+const STRESS_CLIP_COUNT = 423;
+const STRESS_SOURCE_AMPLITUDE = 8000;
+const STRESS_INTERMITTENT_START = 28;
+const stressIntermittentRanges = [
+ {from: 0, durationInFrames: 37},
+ {from: 52, durationInFrames: 39},
+ {from: 143, durationInFrames: 37},
+ {from: 260, durationInFrames: 70},
+ {from: 411, durationInFrames: 89},
+];
+
+let nextStressClipStart = 20;
+const stressClips = new Array(STRESS_CLIP_COUNT).fill(null).map((_, index) => {
+ if (index % 17 === 0) {
+ nextStressClipStart += 4;
+ } else if (index % 7 === 0) {
+ nextStressClipStart++;
+ }
+
+ const middleFrom = (index * 7) % 8;
+ const innerFrom = (index * 11) % 9;
+ const clip = {
+ outerFrom: nextStressClipStart - middleFrom - innerFrom,
+ middleFrom,
+ innerFrom,
+ durationInFrames: 1,
+ };
+ nextStressClipStart++;
+ return clip;
+});
+
+const stressClipStarts = new Set(
+ stressClips.map((clip) => clip.outerFrom + clip.middleFrom + clip.innerFrom),
+);
+
+const makeConstantPcmWavDataUrl = () => {
+ const channels = 2;
+ const sourceDurationInFrames = 30;
+ const samplesPerChannel =
+ (STRESS_SAMPLE_RATE / STRESS_FPS) * sourceDurationInFrames;
+ const dataSize = samplesPerChannel * channels * 2;
+ const wav = Buffer.alloc(44 + dataSize);
+ wav.write('RIFF', 0);
+ wav.writeUInt32LE(36 + dataSize, 4);
+ wav.write('WAVE', 8);
+ wav.write('fmt ', 12);
+ wav.writeUInt32LE(16, 16);
+ wav.writeUInt16LE(1, 20);
+ wav.writeUInt16LE(channels, 22);
+ wav.writeUInt32LE(STRESS_SAMPLE_RATE, 24);
+ wav.writeUInt32LE(STRESS_SAMPLE_RATE * channels * 2, 28);
+ wav.writeUInt16LE(channels * 2, 32);
+ wav.writeUInt16LE(16, 34);
+ wav.write('data', 36);
+ wav.writeUInt32LE(dataSize, 40);
+ for (let offset = 44; offset < wav.length; offset += 2) {
+ wav.writeInt16LE(STRESS_SOURCE_AMPLITUDE, offset);
+ }
+
+ return `data:audio/wav;base64,${wav.toString('base64')}`;
+};
+
+const inlineAudioStressInputProps = {
+ src: makeConstantPcmWavDataUrl(),
+ clips: stressClips,
+ intermittentOuterFrom: 11,
+ intermittentInnerFrom: 17,
+ intermittentDurationInFrames: 560,
+ intermittentRanges: stressIntermittentRanges,
+};
+
+const getInlineAudioTempSize = (
+ directory: string,
+ insideInlineAudioDirectory: boolean,
+): number => {
+ if (!existsSync(directory)) {
+ return 0;
+ }
+
+ try {
+ return readdirSync(directory, {withFileTypes: true}).reduce(
+ (total, entry) => {
+ const entryPath = path.join(directory, entry.name);
+ if (entry.isFile()) {
+ return insideInlineAudioDirectory
+ ? total + statSync(entryPath).size
+ : total;
+ }
+
+ if (!entry.isDirectory()) {
+ return total;
+ }
+
+ return (
+ total +
+ getInlineAudioTempSize(
+ entryPath,
+ insideInlineAudioDirectory ||
+ entry.name === 'remotion-inline-audio-mixing',
+ )
+ );
+ },
+ 0,
+ );
+ } catch {
+ // The renderer can delete a temporary file between readdirSync() and
+ // statSync() while the size monitor is running.
+ return 0;
+ }
+};
+
const renderIssue10468Wav = async ({
tmpDir,
name,
@@ -258,6 +379,96 @@ test(
{timeout: 180000},
);
+test(
+ '@remotion/media keeps 423 nested and intermittently mounted audio clips sample-aligned',
+ async () => {
+ const tmpDir = mkdtempSync(path.join(os.tmpdir(), 'inline-audio-stress-'));
+ const previousTmpDir = process.env.TMPDIR;
+ process.env.TMPDIR = tmpDir;
+ let peakInlineAudioTempSize = 0;
+ const tempSizeMonitor = setInterval(() => {
+ peakInlineAudioTempSize = Math.max(
+ peakInlineAudioTempSize,
+ getInlineAudioTempSize(tmpDir, false),
+ );
+ }, 5);
+ try {
+ const outputLocation = path.join(tmpDir, 'stress.wav');
+ const composition = await selectComposition({
+ id: 'inline-audio-stress',
+ serveUrl: exampleBuild,
+ inputProps: inlineAudioStressInputProps,
+ });
+ await renderMedia({
+ outputLocation,
+ codec: 'wav',
+ serveUrl: exampleBuild,
+ composition,
+ inputProps: inlineAudioStressInputProps,
+ sampleRate: STRESS_SAMPLE_RATE,
+ concurrency: 4,
+ logLevel: 'error',
+ });
+
+ const rendered = readPcmWav(outputLocation);
+ expect(rendered.sampleRate).toBe(STRESS_SAMPLE_RATE);
+ expect(rendered.channels).toBe(2);
+ const samplesPerFrame = STRESS_SAMPLE_RATE / STRESS_FPS;
+ const expectedSampleCount =
+ STRESS_DURATION_IN_FRAMES * samplesPerFrame * rendered.channels;
+ expect(rendered.samples.length).toBe(expectedSampleCount);
+
+ const observedLevels = new Set<number>();
+ let maximumMeanError = 0;
+ for (let frame = 0; frame < STRESS_DURATION_IN_FRAMES; frame++) {
+ const intermittentFrame = frame - STRESS_INTERMITTENT_START;
+ const intermittentIsMounted = stressIntermittentRanges.some(
+ (range) =>
+ intermittentFrame >= range.from &&
+ intermittentFrame < range.from + range.durationInFrames,
+ );
+ const expectedAmplitude =
+ (stressClipStarts.has(frame) ? 2000 : 0) +
+ (intermittentIsMounted ? 1000 : 0);
+ observedLevels.add(expectedAmplitude);
+
+ // Ignore a few samples at frame boundaries so this assertion is about
+ // timeline placement rather than decoder edge interpolation.
+ const firstSample = (frame * samplesPerFrame + 64) * rendered.channels;
+ const lastSample =
+ ((frame + 1) * samplesPerFrame - 64) * rendered.channels;
+ let sum = 0;
+ for (let sample = firstSample; sample < lastSample; sample++) {
+ sum += Math.abs(rendered.samples[sample]);
+ }
+
+ const mean = sum / (lastSample - firstSample);
+ maximumMeanError = Math.max(
+ maximumMeanError,
+ Math.abs(mean - expectedAmplitude),
+ );
+ }
+
+ expect([...observedLevels].sort((a, b) => a - b)).toEqual([
+ 0, 1000, 2000, 3000,
+ ]);
+ expect(maximumMeanError).toBeLessThanOrEqual(20);
+ expect(peakInlineAudioTempSize).toBeGreaterThan(2_000_000);
+ expect(peakInlineAudioTempSize).toBeLessThan(10_000_000);
+ } finally {
+ clearInterval(tempSizeMonitor);
+ if (previousTmpDir === undefined) {
+ delete process.env.TMPDIR;
+ } else {
+ process.env.TMPDIR = previousTmpDir;
+ }
+
+ rmSync(tmpDir, {recursive: true, force: true});
+ }
+ },
+ {timeout: 300000},
+);
+
test(
'Render video with default sampleRate should produce 48000 Hz audio',
async () => {
diff --git a/packages/renderer/src/test/inline-audio-mixing.test.ts b/packages/renderer/src/test/inline-audio-mixing.test.ts
index 3f4a479843..db721c197f 100644
--- a/packages/renderer/src/test/inline-audio-mixing.test.ts
+++ b/packages/renderer/src/test/inline-audio-mixing.test.ts
@@ -103,24 +103,34 @@ test('keeps a composition-relative fallback for an unknown clip origin', async (
});
test(
- 'pre-merges many sequential clips without materializing their timeline offsets',
+ 'pre-merges 423 offset clips without materializing their timeline offsets',
async () => {
const downloadMap = makeDownloadMap(SAMPLE_RATE);
try {
- for (let index = 0; index < 12; index++) {
+ let nextFrame = 300;
+ const startFrames: number[] = [];
+ for (let index = 0; index < 423; index++) {
+ if (index % 17 === 0) {
+ nextFrame += 4;
+ } else if (index % 7 === 0) {
+ nextFrame++;
+ }
+
+ startFrames.push(nextFrame);
downloadMap.inlineAudioMixing.addAsset({
asset: makeAsset({
id: `clip-${index}`,
- frame: 300 + index,
- startInVideo: 300 + index,
+ frame: nextFrame,
+ startInVideo: nextFrame,
}),
fps: FPS,
- totalNumberOfFrames: 360,
+ totalNumberOfFrames: 1000,
firstFrame: 0,
trimLeftOffset: 0,
trimRightOffset: 0,
});
+ nextFrame++;
}
await finish(downloadMap.inlineAudioMixing);
@@ -138,12 +148,15 @@ test(
});
expect(merged).not.toBeNull();
- expect(merged?.startInSeconds).toBe(10);
- expect(merged?.durationInSeconds).toBeCloseTo(12 / FPS, 8);
- expect(statSync(merged!.outName).size).toBeLessThan(100_000);
+ expect(merged?.startInSeconds).toBe(startFrames[0] / FPS);
+ expect(merged?.durationInSeconds).toBeCloseTo(
+ (startFrames.at(-1)! - startFrames[0] + 1) / FPS,
+ 8,
+ );
+ expect(statSync(merged!.outName).size).toBeLessThan(4_000_000);
} finally {
cleanDownloadMap(downloadMap);
}
},
- {timeout: 30_000},
+ {timeout: 120_000},
);