Skip to content

🎥 Make videos programmatically with React

`@remotion/media`: Fix audio extraction across trimmed loop boundaries

Jonny Burgerf4f60a07191d
diff --git a/packages/media/src/audio-extraction/extract-audio.ts b/packages/media/src/audio-extraction/extract-audio.ts
index 078f76be32..6e0385a6ee 100644
--- a/packages/media/src/audio-extraction/extract-audio.ts
+++ b/packages/media/src/audio-extraction/extract-audio.ts
@@ -107,102 +107,164 @@ const extractAudioInternal = async ({
}
try {
- const sampleIterator = await mediaCache.audioManager.getIterator({
- src,
- timeInSeconds,
- audioSampleSink: audio.sampleSink,
- isMatroska,
- actualMatroskaTimestamps,
- logLevel,
- maxCacheSize,
- });
-
const durationInSeconds = durationNotYetApplyingPlaybackRate * playbackRate;
-
- const samples = await sampleIterator.getSamples(
- timeInSeconds,
- durationInSeconds,
- );
-
- mediaCache.audioManager.logOpenFrames();
+ const loopStartInSeconds = (trimBefore ?? 0) / fps;
+ const loopEndInSeconds =
+ trimAfter === undefined
+ ? (mediaDurationInSeconds ?? Infinity)
+ : trimAfter / fps;
+ if (loop && loopEndInSeconds <= loopStartInSeconds) {
+ throw new Error('Cannot loop audio over an empty range');
+ }
const audioDataArray: UnresampledPcmS16AudioData[] = [];
- for (let i = 0; i < samples.length; i++) {
- const sample = samples[i];
-
- // Less than 1 sample would be included - we did not need it after all!
- if (
- Math.abs(sample.timestamp - (timeInSeconds + durationInSeconds)) *
- sample.sampleRate <
- 1
- ) {
+ let remainingDurationInSeconds = durationInSeconds;
+ let segmentStartInSeconds = timeInSeconds;
+ let pendingSilenceInSeconds = 0;
+ while (remainingDurationInSeconds > 0.0000000001) {
+ const segmentDurationInSeconds = loop
+ ? Math.min(
+ remainingDurationInSeconds,
+ loopEndInSeconds - segmentStartInSeconds,
+ )
+ : remainingDurationInSeconds;
+ if (segmentDurationInSeconds <= 0) {
+ segmentStartInSeconds = loopStartInSeconds;
continue;
}
- // Less than 1 sample would be included - we did not need it after all!
- if (sample.timestamp + sample.duration <= timeInSeconds) {
- continue;
- }
+ const sampleIterator = await mediaCache.audioManager.getIterator({
+ src,
+ timeInSeconds: segmentStartInSeconds,
+ audioSampleSink: audio.sampleSink,
+ isMatroska,
+ actualMatroskaTimestamps,
+ logLevel,
+ maxCacheSize,
+ });
+ const samples = await sampleIterator.getSamples(
+ segmentStartInSeconds,
+ segmentDurationInSeconds,
+ );
+ mediaCache.audioManager.logOpenFrames();
+
+ const segmentAudioData: UnresampledPcmS16AudioData[] = [];
+ for (let i = 0; i < samples.length; i++) {
+ const sample = samples[i];
+ const segmentEndInSeconds =
+ segmentStartInSeconds + segmentDurationInSeconds;
+
+ // Less than 1 sample would be included - we did not need it after all!
+ if (
+ Math.abs(sample.timestamp - segmentEndInSeconds) * sample.sampleRate <
+ 1
+ ) {
+ continue;
+ }
- const isFirstSample = i === 0;
- const isLastSample = i === samples.length - 1;
+ // Less than 1 sample would be included - we did not need it after all!
+ if (sample.timestamp + sample.duration <= segmentStartInSeconds) {
+ continue;
+ }
- const audioDataRaw = sample.toAudioData();
+ const isFirstSample = i === 0;
+ const isLastSample = i === samples.length - 1;
- // amount of samples to shave from start and end
- let trimStartInSeconds = 0;
- let trimEndInSeconds = 0;
- let leadingSilence: UnresampledPcmS16AudioData | null = null;
+ const audioDataRaw = sample.toAudioData();
- if (isFirstSample) {
- trimStartInSeconds = fixFloatingPoint(timeInSeconds - sample.timestamp);
+ // amount of samples to shave from start and end
+ let trimStartInSeconds = 0;
+ let trimEndInSeconds = 0;
+ let leadingSilence: UnresampledPcmS16AudioData | null = null;
- if (trimStartInSeconds < 0) {
- const silenceFrames = Math.ceil(
- fixFloatingPoint(-trimStartInSeconds * audioDataRaw.sampleRate),
+ if (isFirstSample) {
+ trimStartInSeconds = fixFloatingPoint(
+ segmentStartInSeconds - sample.timestamp,
);
- leadingSilence = {
- data: new Int16Array(silenceFrames * audioDataRaw.numberOfChannels),
- numberOfChannels: audioDataRaw.numberOfChannels,
- numberOfFrames: silenceFrames,
- sampleRate: audioDataRaw.sampleRate,
- timestamp: timeInSeconds * 1_000_000,
- durationInMicroSeconds:
- (silenceFrames / audioDataRaw.sampleRate) * 1_000_000,
- };
- trimStartInSeconds = 0;
+
+ if (trimStartInSeconds < 0) {
+ const silenceFrames = Math.ceil(
+ fixFloatingPoint(-trimStartInSeconds * audioDataRaw.sampleRate),
+ );
+ leadingSilence = {
+ data: new Int16Array(
+ silenceFrames * audioDataRaw.numberOfChannels,
+ ),
+ numberOfChannels: audioDataRaw.numberOfChannels,
+ numberOfFrames: silenceFrames,
+ sampleRate: audioDataRaw.sampleRate,
+ timestamp: segmentStartInSeconds * 1_000_000,
+ durationInMicroSeconds:
+ (silenceFrames / audioDataRaw.sampleRate) * 1_000_000,
+ };
+ trimStartInSeconds = 0;
+ }
}
- }
- if (isLastSample) {
- trimEndInSeconds =
- // clamp to 0 in case the audio ends early
- Math.max(
- 0,
- sample.timestamp +
- sample.duration -
- (timeInSeconds + durationInSeconds),
- );
- }
+ if (isLastSample) {
+ trimEndInSeconds =
+ // clamp to 0 in case the audio ends early
+ Math.max(
+ 0,
+ sample.timestamp + sample.duration - segmentEndInSeconds,
+ );
+ }
- const audioData = convertAudioDataToS16({
- audioData: audioDataRaw,
- trimStartInSeconds,
- trimEndInSeconds,
- audioDataTimestamp: sample.timestamp,
- isLast: isLastSample,
- });
- audioDataRaw.close();
+ const audioData = convertAudioDataToS16({
+ audioData: audioDataRaw,
+ trimStartInSeconds,
+ trimEndInSeconds,
+ audioDataTimestamp: sample.timestamp,
+ isLast: isLastSample,
+ });
+ audioDataRaw.close();
+
+ if (audioData.numberOfFrames === 0) {
+ continue;
+ }
- if (audioData.numberOfFrames === 0) {
- continue;
+ if (leadingSilence) {
+ segmentAudioData.push(leadingSilence);
+ }
+
+ segmentAudioData.push(audioData);
}
- if (leadingSilence) {
- audioDataArray.push(leadingSilence);
+ if (segmentAudioData.length > 0) {
+ const {numberOfChannels, sampleRate: sourceSampleRate} =
+ segmentAudioData[0];
+ // Preserve silence at the end of a loop pass before appending audio
+ // from the next pass.
+ const silenceFrames = Math.ceil(
+ fixFloatingPoint(pendingSilenceInSeconds * sourceSampleRate),
+ );
+ if (silenceFrames > 0) {
+ audioDataArray.push({
+ data: new Int16Array(silenceFrames * numberOfChannels),
+ numberOfChannels,
+ numberOfFrames: silenceFrames,
+ sampleRate: sourceSampleRate,
+ timestamp: timeInSeconds * 1_000_000,
+ durationInMicroSeconds:
+ (silenceFrames / sourceSampleRate) * 1_000_000,
+ });
+ }
+
+ audioDataArray.push(...segmentAudioData);
+ const segmentFrames = segmentAudioData.reduce(
+ (sum, data) => sum + data.numberOfFrames,
+ 0,
+ );
+ pendingSilenceInSeconds = Math.max(
+ 0,
+ segmentDurationInSeconds - segmentFrames / sourceSampleRate,
+ );
+ } else {
+ pendingSilenceInSeconds += segmentDurationInSeconds;
}
- audioDataArray.push(audioData);
+ remainingDurationInSeconds -= segmentDurationInSeconds;
+ segmentStartInSeconds = loopStartInSeconds;
}
if (audioDataArray.length === 0) {