Skip to content

🎥 Make videos programmatically with React

`@remotion/studio`: Fix out-of-memory crash when tiling looped waveforms (#10588)

Jonny Burger9342fc58159b
diff --git a/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx b/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx
index 40f4200aff..f00aa11c3f 100644
--- a/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx
+++ b/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx
@@ -8,6 +8,7 @@ import {
frameDatabase,
getAspectRatioFromCache,
getFrameDatabaseKeyPrefix,
+ getLoopDisplaySegments,
getTimestampFromFrameDatabaseKey,
makeFrameDatabaseKey,
resizeVideoFrame,
@@ -504,7 +505,11 @@ const getVisibleSegments = ({
readonly displayOffsetInFrames: number;
readonly loopDisplay: LoopDisplay | undefined;
}) => {
- if (!loopDisplay || loopDisplay.numberOfTimes <= 1) {
+ if (
+ !loopDisplay ||
+ loopDisplay.numberOfTimes <= 1 ||
+ loopDisplay.durationInFrames <= 0
+ ) {
return [
{
key: 'single',
@@ -515,37 +520,18 @@ const getVisibleSegments = ({
];
}
- const segments: {
- key: string;
- displayOffsetInFrames: number;
- durationInFrames: number;
- sourceOffsetInFrames: number;
- }[] = [];
- let processed = 0;
- while (processed < displayDurationInFrames) {
- const absoluteOffset = displayOffsetInFrames + processed;
- const sourceOffsetInFrames =
- ((absoluteOffset % loopDisplay.durationInFrames) +
- loopDisplay.durationInFrames) %
- loopDisplay.durationInFrames;
- const durationInFrames = Math.min(
- displayDurationInFrames - processed,
- loopDisplay.durationInFrames - sourceOffsetInFrames,
- );
- if (durationInFrames <= 0) {
- break;
- }
-
- segments.push({
- key: `loop-${Math.floor(absoluteOffset / loopDisplay.durationInFrames)}`,
- displayOffsetInFrames: absoluteOffset,
- durationInFrames,
- sourceOffsetInFrames,
- });
- processed += durationInFrames;
- }
-
- return segments;
+ return getLoopDisplaySegments({
+ displayDurationInFrames,
+ displayOffsetInFrames,
+ loopDurationInFrames: loopDisplay.durationInFrames,
+ }).map((segment) => {
+ return {
+ key: `loop-${segment.loopIndex}`,
+ displayOffsetInFrames: segment.absoluteOffsetInFrames,
+ durationInFrames: segment.durationInFrames,
+ sourceOffsetInFrames: segment.loopOffsetInFrames,
+ };
+ });
};
const TimelineVideoInfoInner: React.FC<{
diff --git a/packages/timeline-utils/src/audio-waveform/get-visible-waveform-volume.ts b/packages/timeline-utils/src/audio-waveform/get-visible-waveform-volume.ts
index 39d6e417a3..97c735b44a 100644
--- a/packages/timeline-utils/src/audio-waveform/get-visible-waveform-volume.ts
+++ b/packages/timeline-utils/src/audio-waveform/get-visible-waveform-volume.ts
@@ -1,4 +1,8 @@
-import {shouldTileLoopDisplay, type TimelineLoopDisplay} from '../loop-display';
+import {
+ getLoopDisplaySegments,
+ shouldTileLoopDisplay,
+ type TimelineLoopDisplay,
+} from '../loop-display';
import type {WaveformVolume} from './draw-peaks';
export const getVisibleWaveformVolume = ({
@@ -35,34 +39,24 @@ export const getVisibleWaveformVolume = ({
return volume.slice(start, end);
}
- const result: number[] = [];
- let processed = 0;
- while (processed < displayDurationInFrames) {
- const absoluteOffset = displayOffsetInFrames + processed;
- const loopOffset =
- ((absoluteOffset % loopDisplay.durationInFrames) +
- loopDisplay.durationInFrames) %
- loopDisplay.durationInFrames;
- const segmentDuration = Math.min(
- displayDurationInFrames - processed,
- loopDisplay.durationInFrames - loopOffset,
- );
- if (segmentDuration <= 0) {
- break;
- }
+ const segments = getLoopDisplaySegments({
+ displayDurationInFrames,
+ displayOffsetInFrames,
+ loopDurationInFrames: loopDisplay.durationInFrames,
+ });
- const start = Math.max(0, Math.floor(loopOffset));
+ const result: number[] = [];
+ for (const segment of segments) {
+ const start = Math.max(0, Math.floor(segment.loopOffsetInFrames));
const end = Math.min(
volume.length,
- Math.ceil(loopOffset + segmentDuration),
+ Math.ceil(segment.loopOffsetInFrames + segment.durationInFrames),
);
// Do not `push(...slice)`: Chrome throws RangeError: Invalid array length
// once a loop segment is longer than ~65k frames.
for (let index = start; index < end; index++) {
result.push(volume[index]);
}
-
- processed += segmentDuration;
}
return result;
diff --git a/packages/timeline-utils/src/audio-waveform/slice-visible-waveform-peaks.ts b/packages/timeline-utils/src/audio-waveform/slice-visible-waveform-peaks.ts
index 0ccfdd33ba..bc4ec1d493 100644
--- a/packages/timeline-utils/src/audio-waveform/slice-visible-waveform-peaks.ts
+++ b/packages/timeline-utils/src/audio-waveform/slice-visible-waveform-peaks.ts
@@ -1,5 +1,5 @@
import type {TimelineLoopDisplay} from '../loop-display';
-import {shouldTileLoopDisplay} from '../loop-display';
+import {getLoopDisplaySegments, shouldTileLoopDisplay} from '../loop-display';
import {sliceWaveformPeaks} from './slice-waveform-peaks';
export const sliceVisibleWaveformPeaks = ({
@@ -21,7 +21,10 @@ export const sliceVisibleWaveformPeaks = ({
readonly playbackRate: number;
readonly startFrom: number;
}) => {
- if (!shouldTileLoopDisplay(loopDisplay)) {
+ if (
+ !shouldTileLoopDisplay(loopDisplay) ||
+ loopDisplay.durationInFrames <= 0
+ ) {
return sliceWaveformPeaks({
durationInFrames: Math.min(
displayDurationInFrames,
@@ -34,33 +37,24 @@ export const sliceVisibleWaveformPeaks = ({
});
}
+ const segments = getLoopDisplaySegments({
+ displayDurationInFrames,
+ displayOffsetInFrames,
+ loopDurationInFrames: loopDisplay.durationInFrames,
+ });
+
const parts: Float32Array[] = [];
- let processed = 0;
let totalLength = 0;
- while (processed < displayDurationInFrames) {
- const absoluteOffset = displayOffsetInFrames + processed;
- const loopOffset =
- ((absoluteOffset % loopDisplay.durationInFrames) +
- loopDisplay.durationInFrames) %
- loopDisplay.durationInFrames;
- const segmentDuration = Math.min(
- displayDurationInFrames - processed,
- loopDisplay.durationInFrames - loopOffset,
- );
- if (segmentDuration <= 0) {
- break;
- }
-
+ for (const segment of segments) {
const part = sliceWaveformPeaks({
- durationInFrames: segmentDuration,
+ durationInFrames: segment.durationInFrames,
fps,
peaks,
playbackRate,
- startFrom: startFrom + loopOffset * playbackRate,
+ startFrom: startFrom + segment.loopOffsetInFrames * playbackRate,
});
parts.push(part);
totalLength += part.length;
- processed += segmentDuration;
}
if (parts.length === 1) {
diff --git a/packages/timeline-utils/src/index.ts b/packages/timeline-utils/src/index.ts
index a7223014dd..4726eec0db 100644
--- a/packages/timeline-utils/src/index.ts
+++ b/packages/timeline-utils/src/index.ts
@@ -5,17 +5,15 @@ export type {
} from './audio-waveform/audio-waveform-worker-types';
export {TARGET_SAMPLE_RATE} from './audio-waveform/constants';
export {drawBars, type WaveformVolume} from './audio-waveform/draw-peaks';
+export {getVisibleWaveformVolume} from './audio-waveform/get-visible-waveform-volume';
export {loadWaveformPeaks} from './audio-waveform/load-waveform-peaks';
export {makeAudioWaveformWorker} from './audio-waveform/make-audio-waveform-worker';
-export {sliceWaveformPeaks} from './audio-waveform/slice-waveform-peaks';
-export {getVisibleWaveformVolume} from './audio-waveform/get-visible-waveform-volume';
export {sliceVisibleWaveformPeaks} from './audio-waveform/slice-visible-waveform-peaks';
+export {sliceWaveformPeaks} from './audio-waveform/slice-waveform-peaks';
export {
createWaveformPeakProcessor,
emitWaveformProgress,
} from './audio-waveform/waveform-peak-processor';
-export {drawRepeatingImageThumbnail} from './image-thumbnail/draw-repeating-image-thumbnail';
-export {getScaledImageThumbnailDimensions} from './image-thumbnail/get-scaled-image-thumbnail-dimensions';
export {clampTimestampsToDuration} from './clamp-timestamps-to-duration';
export {extractFrames} from './extract-frames';
export type {
@@ -32,16 +30,22 @@ export {
makeFrameDatabaseKey,
} from './frame-database';
export type {FrameDatabaseKey} from './frame-database';
-export {getLoopDisplayWidth, shouldTileLoopDisplay} from './loop-display';
-export type {TimelineLoopDisplay} from './loop-display';
+export {drawRepeatingImageThumbnail} from './image-thumbnail/draw-repeating-image-thumbnail';
+export {getScaledImageThumbnailDimensions} from './image-thumbnail/get-scaled-image-thumbnail-dimensions';
+export {
+ getLoopDisplaySegments,
+ getLoopDisplayWidth,
+ shouldTileLoopDisplay,
+} from './loop-display';
+export type {LoopDisplaySegment, TimelineLoopDisplay} from './loop-display';
export {
+ WEBCODECS_TIMESCALE,
calculateTimestampSlots,
drawSlot,
ensureSlots,
fillFrameWhereItFits,
fillWithCachedFrames,
getDurationOfOneFrame,
- WEBCODECS_TIMESCALE,
} from './render-frame-strip';
export {renderFrameStripToCanvas} from './render-frame-strip-to-canvas';
export type {RenderFrameStripToCanvasOptions} from './render-frame-strip-to-canvas';
diff --git a/packages/timeline-utils/src/loop-display.ts b/packages/timeline-utils/src/loop-display.ts
index bfcdb0066e..b6f4f64fc1 100644
--- a/packages/timeline-utils/src/loop-display.ts
+++ b/packages/timeline-utils/src/loop-display.ts
@@ -10,6 +10,73 @@ export const shouldTileLoopDisplay = (
return loopDisplay !== undefined && loopDisplay.numberOfTimes > 1;
};
+export type LoopDisplaySegment = {
+ loopIndex: number;
+ // Offset of the segment relative to the start of the tiled display window origin
+ absoluteOffsetInFrames: number;
+ // Offset within the loop iteration
+ loopOffsetInFrames: number;
+ durationInFrames: number;
+};
+
+// Computes the visible loop segments by loop index instead of accumulating
+// floating-point segment durations. With fractional display offsets (e.g. from
+// horizontal timeline virtualization), an accumulating `processed += segment`
+// loop can stall once the residue is smaller than the ULP of `processed`,
+// growing arrays unboundedly until the tab runs out of memory.
+export const getLoopDisplaySegments = ({
+ displayDurationInFrames,
+ displayOffsetInFrames,
+ loopDurationInFrames,
+}: {
+ readonly displayDurationInFrames: number;
+ readonly displayOffsetInFrames: number;
+ readonly loopDurationInFrames: number;
+}): LoopDisplaySegment[] => {
+ if (
+ !Number.isFinite(displayDurationInFrames) ||
+ displayDurationInFrames <= 0 ||
+ !Number.isFinite(displayOffsetInFrames) ||
+ !Number.isFinite(loopDurationInFrames) ||
+ loopDurationInFrames <= 0
+ ) {
+ return [];
+ }
+
+ const displayEndInFrames = displayOffsetInFrames + displayDurationInFrames;
+ const firstLoopIndex = Math.floor(
+ displayOffsetInFrames / loopDurationInFrames,
+ );
+ const lastLoopIndex = Math.floor(displayEndInFrames / loopDurationInFrames);
+
+ const segments: LoopDisplaySegment[] = [];
+ for (
+ let loopIndex = firstLoopIndex;
+ loopIndex <= lastLoopIndex;
+ loopIndex++
+ ) {
+ const loopStartInFrames = loopIndex * loopDurationInFrames;
+ const segmentStart = Math.max(displayOffsetInFrames, loopStartInFrames);
+ const segmentEnd = Math.min(
+ displayEndInFrames,
+ loopStartInFrames + loopDurationInFrames,
+ );
+ const durationInFrames = segmentEnd - segmentStart;
+ if (durationInFrames <= 0) {
+ continue;
+ }
+
+ segments.push({
+ loopIndex,
+ absoluteOffsetInFrames: segmentStart,
+ loopOffsetInFrames: segmentStart - loopStartInFrames,
+ durationInFrames,
+ });
+ }
+
+ return segments;
+};
+
export const getLoopDisplayWidth = ({
visualizationWidth,
loopDisplay,
diff --git a/packages/timeline-utils/src/test/get-loop-display-segments.test.ts b/packages/timeline-utils/src/test/get-loop-display-segments.test.ts
new file mode 100644
index 0000000000..27e4f4e528
--- /dev/null
+++ b/packages/timeline-utils/src/test/get-loop-display-segments.test.ts
@@ -0,0 +1,135 @@
+import {expect, test} from 'bun:test';
+import {getLoopDisplaySegments} from '../loop-display';
+
+test('splits an integer display window at loop boundaries', () => {
+ expect(
+ getLoopDisplaySegments({
+ displayDurationInFrames: 5,
+ displayOffsetInFrames: 8,
+ loopDurationInFrames: 10,
+ }),
+ ).toEqual([
+ {
+ loopIndex: 0,
+ absoluteOffsetInFrames: 8,
+ loopOffsetInFrames: 8,
+ durationInFrames: 2,
+ },
+ {
+ loopIndex: 1,
+ absoluteOffsetInFrames: 10,
+ loopOffsetInFrames: 0,
+ durationInFrames: 3,
+ },
+ ]);
+});
+
+test('handles fractional display windows', () => {
+ expect(
+ getLoopDisplaySegments({
+ displayDurationInFrames: 5.25,
+ displayOffsetInFrames: 8.5,
+ loopDurationInFrames: 10,
+ }),
+ ).toEqual([
+ {
+ loopIndex: 0,
+ absoluteOffsetInFrames: 8.5,
+ loopOffsetInFrames: 8.5,
+ durationInFrames: 1.5,
+ },
+ {
+ loopIndex: 1,
+ absoluteOffsetInFrames: 10,
+ loopOffsetInFrames: 0,
+ durationInFrames: 3.75,
+ },
+ ]);
+});
+
+test('does not emit an empty segment when the window ends on a boundary', () => {
+ const segments = getLoopDisplaySegments({
+ displayDurationInFrames: 10,
+ displayOffsetInFrames: 10,
+ loopDurationInFrames: 10,
+ });
+ expect(segments).toHaveLength(1);
+ expect(segments[0].loopIndex).toBe(1);
+ expect(segments[0].durationInFrames).toBe(10);
+});
+
+test('stays bounded for fractional offsets that stalled the accumulating loop', () => {
+ // Regression: `processed += segmentDuration` used to no-op once the
+ // floating-point residue was smaller than the ULP of `processed`,
+ // growing arrays until an out-of-memory crash.
+ const displayOffsetInFrames = 11256.685714285799;
+ const displayDurationInFrames = 33426.30571428571;
+ // Stalls the previous accumulating implementation after 12 iterations
+ const loopDurationInFrames = 99.999;
+ const segments = getLoopDisplaySegments({
+ displayDurationInFrames,
+ displayOffsetInFrames,
+ loopDurationInFrames,
+ });
+
+ expect(segments.length).toBeLessThanOrEqual(
+ Math.ceil(displayDurationInFrames / loopDurationInFrames) + 1,
+ );
+ const total = segments.reduce(
+ (sum, segment) => sum + segment.durationInFrames,
+ 0,
+ );
+ expect(total).toBeCloseTo(displayDurationInFrames, 6);
+ for (const segment of segments) {
+ expect(segment.durationInFrames).toBeGreaterThan(0);
+ expect(segment.loopOffsetInFrames).toBeGreaterThanOrEqual(0);
+ expect(segment.loopOffsetInFrames).toBeLessThan(loopDurationInFrames);
+ }
+});
+
+test('handles negative display offsets', () => {
+ expect(
+ getLoopDisplaySegments({
+ displayDurationInFrames: 8,
+ displayOffsetInFrames: -3,
+ loopDurationInFrames: 10,
+ }),
+ ).toEqual([
+ {
+ loopIndex: -1,
+ absoluteOffsetInFrames: -3,
+ loopOffsetInFrames: 7,
+ durationInFrames: 3,
+ },
+ {
+ loopIndex: 0,
+ absoluteOffsetInFrames: 0,
+ loopOffsetInFrames: 0,
+ durationInFrames: 5,
+ },
+ ]);
+});
+
+test('returns no segments for degenerate inputs', () => {
+ expect(
+ getLoopDisplaySegments({
+ displayDurationInFrames: 0,
+ displayOffsetInFrames: 0,
+ loopDurationInFrames: 10,
+ }),
+ ).toEqual([]);
+ expect(
+ getLoopDisplaySegments({
+ displayDurationInFrames: 10,
+ displayOffsetInFrames: 0,
+ loopDurationInFrames: 0,
+ }),
+ ).toEqual([]);
+ expect(
+ getLoopDisplaySegments({
+ displayDurationInFrames: Infinity,
+ displayOffsetInFrames: 0,
+ loopDurationInFrames: 10,
+ }),
+ ).toEqual([]);
+});
diff --git a/packages/timeline-utils/src/test/get-visible-waveform-volume.test.ts b/packages/timeline-utils/src/test/get-visible-waveform-volume.test.ts
index bdf6663ab6..050654388e 100644
--- a/packages/timeline-utils/src/test/get-visible-waveform-volume.test.ts
+++ b/packages/timeline-utils/src/test/get-visible-waveform-volume.test.ts
@@ -66,6 +66,28 @@ test('tiles a long looped volume curve', () => {
expect((visible as number[])[loopDurationInFrames - 1]).toBe(0.5);
});
+test('terminates with fractional display windows from timeline virtualization', () => {
+ const volume = Array.from({length: 100}, (_, index) => index / 100);
+ const visible = getVisibleWaveformVolume({
+ displayDurationInFrames: 33426.30571428571,
+ displayOffsetInFrames: 11256.685714285799,
+ loopDisplay: {
+ // A fractional loop duration whose residue stalls the previous
+ // `processed += segmentDuration` implementation below 1 ULP
+ durationInFrames: 99.999,
+ numberOfTimes: 600,
+ startOffset: 0,
+ },
+ volume,
+ });
+
+ expect(Array.isArray(visible)).toBe(true);
+ const values = visible as number[];
+ expect(values.length).toBeGreaterThan(33426);
+ // Bounded by display duration plus per-segment rounding
+ expect(values.length).toBeLessThan(34000);
+});
+
test('stops tiling if a loop segment would not advance', () => {
expect(
getVisibleWaveformVolume({
diff --git a/packages/timeline-utils/src/test/slice-visible-waveform-peaks.test.ts b/packages/timeline-utils/src/test/slice-visible-waveform-peaks.test.ts
index 5b21132fb9..b75c12aa06 100644
--- a/packages/timeline-utils/src/test/slice-visible-waveform-peaks.test.ts
+++ b/packages/timeline-utils/src/test/slice-visible-waveform-peaks.test.ts
@@ -41,3 +41,28 @@ test('joins waveform portions across a loop boundary', () => {
...Array.from({length: 30}, (_, index) => index),
]);
});
+
+test('terminates with fractional display windows from timeline virtualization', () => {
+ const peaks = Float32Array.from({length: 1000}, (_, index) => index);
+ const visible = sliceVisibleWaveformPeaks({
+ displayDurationInFrames: 33426.30571428571,
+ displayOffsetInFrames: 11256.685714285799,
+ durationInFrames: 100,
+ fps: 10,
+ loopDisplay: {
+ // A fractional loop duration whose residue stalls the previous
+ // `processed += segmentDuration` implementation below 1 ULP
+ durationInFrames: 99.999,
+ numberOfTimes: 600,
+ startOffset: 0,
+ },
+ peaks,
+ playbackRate: 1,
+ startFrom: 0,
+ });
+
+ // 10 peaks per frame at fps 10 and TARGET_SAMPLE_RATE 100,
+ // plus up to 2 peaks of rounding per loop segment
+ expect(visible.length).toBeGreaterThan(334000);
+ expect(visible.length).toBeLessThan(336000);
+});