🎥 Make videos programmatically with React
`@remotion/studio`: Fix out-of-memory crash when tiling looped waveforms (#10588)
Jonny Burger9342fc58159b
diff --git a/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx b/packages/studio/src/components/Timeline/TimelineVideoInfo.tsxindex 40f4200aff..f00aa11c3f 100644--- a/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx+++ b/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx@@ -8,6 +8,7 @@ import {frameDatabase,getAspectRatioFromCache,getFrameDatabaseKeyPrefix,+ getLoopDisplaySegments,getTimestampFromFrameDatabaseKey,makeFrameDatabaseKey,resizeVideoFrame,@@ -504,7 +505,11 @@ const getVisibleSegments = ({readonly displayOffsetInFrames: number;readonly loopDisplay: LoopDisplay | undefined;}) => {- if (!loopDisplay || loopDisplay.numberOfTimes <= 1) {+ if (+ !loopDisplay ||+ loopDisplay.numberOfTimes <= 1 ||+ loopDisplay.durationInFrames <= 0+ ) {return [{key: 'single',@@ -515,37 +520,18 @@ const getVisibleSegments = ({];}- const segments: {- key: string;- displayOffsetInFrames: number;- durationInFrames: number;- sourceOffsetInFrames: number;- }[] = [];- let processed = 0;- while (processed < displayDurationInFrames) {- const absoluteOffset = displayOffsetInFrames + processed;- const sourceOffsetInFrames =- ((absoluteOffset % loopDisplay.durationInFrames) +- loopDisplay.durationInFrames) %- loopDisplay.durationInFrames;- const durationInFrames = Math.min(- displayDurationInFrames - processed,- loopDisplay.durationInFrames - sourceOffsetInFrames,- );- if (durationInFrames <= 0) {- break;- }-- segments.push({- key: `loop-${Math.floor(absoluteOffset / loopDisplay.durationInFrames)}`,- displayOffsetInFrames: absoluteOffset,- durationInFrames,- sourceOffsetInFrames,- });- processed += durationInFrames;- }-- return segments;+ return getLoopDisplaySegments({+ displayDurationInFrames,+ displayOffsetInFrames,+ loopDurationInFrames: loopDisplay.durationInFrames,+ }).map((segment) => {+ return {+ key: `loop-${segment.loopIndex}`,+ displayOffsetInFrames: segment.absoluteOffsetInFrames,+ durationInFrames: segment.durationInFrames,+ sourceOffsetInFrames: segment.loopOffsetInFrames,+ };+ });};const TimelineVideoInfoInner: React.FC<{diff --git a/packages/timeline-utils/src/audio-waveform/get-visible-waveform-volume.ts b/packages/timeline-utils/src/audio-waveform/get-visible-waveform-volume.tsindex 39d6e417a3..97c735b44a 100644--- a/packages/timeline-utils/src/audio-waveform/get-visible-waveform-volume.ts+++ b/packages/timeline-utils/src/audio-waveform/get-visible-waveform-volume.ts@@ -1,4 +1,8 @@-import {shouldTileLoopDisplay, type TimelineLoopDisplay} from '../loop-display';+import {+ getLoopDisplaySegments,+ shouldTileLoopDisplay,+ type TimelineLoopDisplay,+} from '../loop-display';import type {WaveformVolume} from './draw-peaks';export const getVisibleWaveformVolume = ({@@ -35,34 +39,24 @@ export const getVisibleWaveformVolume = ({return volume.slice(start, end);}- const result: number[] = [];- let processed = 0;- while (processed < displayDurationInFrames) {- const absoluteOffset = displayOffsetInFrames + processed;- const loopOffset =- ((absoluteOffset % loopDisplay.durationInFrames) +- loopDisplay.durationInFrames) %- loopDisplay.durationInFrames;- const segmentDuration = Math.min(- displayDurationInFrames - processed,- loopDisplay.durationInFrames - loopOffset,- );- if (segmentDuration <= 0) {- break;- }+ const segments = getLoopDisplaySegments({+ displayDurationInFrames,+ displayOffsetInFrames,+ loopDurationInFrames: loopDisplay.durationInFrames,+ });- const start = Math.max(0, Math.floor(loopOffset));+ const result: number[] = [];+ for (const segment of segments) {+ const start = Math.max(0, Math.floor(segment.loopOffsetInFrames));const end = Math.min(volume.length,- Math.ceil(loopOffset + segmentDuration),+ Math.ceil(segment.loopOffsetInFrames + segment.durationInFrames),);// Do not `push(...slice)`: Chrome throws RangeError: Invalid array length// once a loop segment is longer than ~65k frames.for (let index = start; index < end; index++) {result.push(volume[index]);}-- processed += segmentDuration;}return result;diff --git a/packages/timeline-utils/src/audio-waveform/slice-visible-waveform-peaks.ts b/packages/timeline-utils/src/audio-waveform/slice-visible-waveform-peaks.tsindex 0ccfdd33ba..bc4ec1d493 100644--- a/packages/timeline-utils/src/audio-waveform/slice-visible-waveform-peaks.ts+++ b/packages/timeline-utils/src/audio-waveform/slice-visible-waveform-peaks.ts@@ -1,5 +1,5 @@import type {TimelineLoopDisplay} from '../loop-display';-import {shouldTileLoopDisplay} from '../loop-display';+import {getLoopDisplaySegments, shouldTileLoopDisplay} from '../loop-display';import {sliceWaveformPeaks} from './slice-waveform-peaks';export const sliceVisibleWaveformPeaks = ({@@ -21,7 +21,10 @@ export const sliceVisibleWaveformPeaks = ({readonly playbackRate: number;readonly startFrom: number;}) => {- if (!shouldTileLoopDisplay(loopDisplay)) {+ if (+ !shouldTileLoopDisplay(loopDisplay) ||+ loopDisplay.durationInFrames <= 0+ ) {return sliceWaveformPeaks({durationInFrames: Math.min(displayDurationInFrames,@@ -34,33 +37,24 @@ export const sliceVisibleWaveformPeaks = ({});}+ const segments = getLoopDisplaySegments({+ displayDurationInFrames,+ displayOffsetInFrames,+ loopDurationInFrames: loopDisplay.durationInFrames,+ });+const parts: Float32Array[] = [];- let processed = 0;let totalLength = 0;- while (processed < displayDurationInFrames) {- const absoluteOffset = displayOffsetInFrames + processed;- const loopOffset =- ((absoluteOffset % loopDisplay.durationInFrames) +- loopDisplay.durationInFrames) %- loopDisplay.durationInFrames;- const segmentDuration = Math.min(- displayDurationInFrames - processed,- loopDisplay.durationInFrames - loopOffset,- );- if (segmentDuration <= 0) {- break;- }-+ for (const segment of segments) {const part = sliceWaveformPeaks({- durationInFrames: segmentDuration,+ durationInFrames: segment.durationInFrames,fps,peaks,playbackRate,- startFrom: startFrom + loopOffset * playbackRate,+ startFrom: startFrom + segment.loopOffsetInFrames * playbackRate,});parts.push(part);totalLength += part.length;- processed += segmentDuration;}if (parts.length === 1) {diff --git a/packages/timeline-utils/src/index.ts b/packages/timeline-utils/src/index.tsindex a7223014dd..4726eec0db 100644--- a/packages/timeline-utils/src/index.ts+++ b/packages/timeline-utils/src/index.ts@@ -5,17 +5,15 @@ export type {} from './audio-waveform/audio-waveform-worker-types';export {TARGET_SAMPLE_RATE} from './audio-waveform/constants';export {drawBars, type WaveformVolume} from './audio-waveform/draw-peaks';+export {getVisibleWaveformVolume} from './audio-waveform/get-visible-waveform-volume';export {loadWaveformPeaks} from './audio-waveform/load-waveform-peaks';export {makeAudioWaveformWorker} from './audio-waveform/make-audio-waveform-worker';-export {sliceWaveformPeaks} from './audio-waveform/slice-waveform-peaks';-export {getVisibleWaveformVolume} from './audio-waveform/get-visible-waveform-volume';export {sliceVisibleWaveformPeaks} from './audio-waveform/slice-visible-waveform-peaks';+export {sliceWaveformPeaks} from './audio-waveform/slice-waveform-peaks';export {createWaveformPeakProcessor,emitWaveformProgress,} from './audio-waveform/waveform-peak-processor';-export {drawRepeatingImageThumbnail} from './image-thumbnail/draw-repeating-image-thumbnail';-export {getScaledImageThumbnailDimensions} from './image-thumbnail/get-scaled-image-thumbnail-dimensions';export {clampTimestampsToDuration} from './clamp-timestamps-to-duration';export {extractFrames} from './extract-frames';export type {@@ -32,16 +30,22 @@ export {makeFrameDatabaseKey,} from './frame-database';export type {FrameDatabaseKey} from './frame-database';-export {getLoopDisplayWidth, shouldTileLoopDisplay} from './loop-display';-export type {TimelineLoopDisplay} from './loop-display';+export {drawRepeatingImageThumbnail} from './image-thumbnail/draw-repeating-image-thumbnail';+export {getScaledImageThumbnailDimensions} from './image-thumbnail/get-scaled-image-thumbnail-dimensions';+export {+ getLoopDisplaySegments,+ getLoopDisplayWidth,+ shouldTileLoopDisplay,+} from './loop-display';+export type {LoopDisplaySegment, TimelineLoopDisplay} from './loop-display';export {+ WEBCODECS_TIMESCALE,calculateTimestampSlots,drawSlot,ensureSlots,fillFrameWhereItFits,fillWithCachedFrames,getDurationOfOneFrame,- WEBCODECS_TIMESCALE,} from './render-frame-strip';export {renderFrameStripToCanvas} from './render-frame-strip-to-canvas';export type {RenderFrameStripToCanvasOptions} from './render-frame-strip-to-canvas';diff --git a/packages/timeline-utils/src/loop-display.ts b/packages/timeline-utils/src/loop-display.tsindex bfcdb0066e..b6f4f64fc1 100644--- a/packages/timeline-utils/src/loop-display.ts+++ b/packages/timeline-utils/src/loop-display.ts@@ -10,6 +10,73 @@ export const shouldTileLoopDisplay = (return loopDisplay !== undefined && loopDisplay.numberOfTimes > 1;};+export type LoopDisplaySegment = {+ loopIndex: number;+ // Offset of the segment relative to the start of the tiled display window origin+ absoluteOffsetInFrames: number;+ // Offset within the loop iteration+ loopOffsetInFrames: number;+ durationInFrames: number;+};++// Computes the visible loop segments by loop index instead of accumulating+// floating-point segment durations. With fractional display offsets (e.g. from+// horizontal timeline virtualization), an accumulating `processed += segment`+// loop can stall once the residue is smaller than the ULP of `processed`,+// growing arrays unboundedly until the tab runs out of memory.+export const getLoopDisplaySegments = ({+ displayDurationInFrames,+ displayOffsetInFrames,+ loopDurationInFrames,+}: {+ readonly displayDurationInFrames: number;+ readonly displayOffsetInFrames: number;+ readonly loopDurationInFrames: number;+}): LoopDisplaySegment[] => {+ if (+ !Number.isFinite(displayDurationInFrames) ||+ displayDurationInFrames <= 0 ||+ !Number.isFinite(displayOffsetInFrames) ||+ !Number.isFinite(loopDurationInFrames) ||+ loopDurationInFrames <= 0+ ) {+ return [];+ }++ const displayEndInFrames = displayOffsetInFrames + displayDurationInFrames;+ const firstLoopIndex = Math.floor(+ displayOffsetInFrames / loopDurationInFrames,+ );+ const lastLoopIndex = Math.floor(displayEndInFrames / loopDurationInFrames);++ const segments: LoopDisplaySegment[] = [];+ for (+ let loopIndex = firstLoopIndex;+ loopIndex <= lastLoopIndex;+ loopIndex+++ ) {+ const loopStartInFrames = loopIndex * loopDurationInFrames;+ const segmentStart = Math.max(displayOffsetInFrames, loopStartInFrames);+ const segmentEnd = Math.min(+ displayEndInFrames,+ loopStartInFrames + loopDurationInFrames,+ );+ const durationInFrames = segmentEnd - segmentStart;+ if (durationInFrames <= 0) {+ continue;+ }++ segments.push({+ loopIndex,+ absoluteOffsetInFrames: segmentStart,+ loopOffsetInFrames: segmentStart - loopStartInFrames,+ durationInFrames,+ });+ }++ return segments;+};+export const getLoopDisplayWidth = ({visualizationWidth,loopDisplay,diff --git a/packages/timeline-utils/src/test/get-loop-display-segments.test.ts b/packages/timeline-utils/src/test/get-loop-display-segments.test.tsnew file mode 100644index 0000000000..27e4f4e528--- /dev/null+++ b/packages/timeline-utils/src/test/get-loop-display-segments.test.ts@@ -0,0 +1,135 @@+import {expect, test} from 'bun:test';+import {getLoopDisplaySegments} from '../loop-display';++test('splits an integer display window at loop boundaries', () => {+ expect(+ getLoopDisplaySegments({+ displayDurationInFrames: 5,+ displayOffsetInFrames: 8,+ loopDurationInFrames: 10,+ }),+ ).toEqual([+ {+ loopIndex: 0,+ absoluteOffsetInFrames: 8,+ loopOffsetInFrames: 8,+ durationInFrames: 2,+ },+ {+ loopIndex: 1,+ absoluteOffsetInFrames: 10,+ loopOffsetInFrames: 0,+ durationInFrames: 3,+ },+ ]);+});++test('handles fractional display windows', () => {+ expect(+ getLoopDisplaySegments({+ displayDurationInFrames: 5.25,+ displayOffsetInFrames: 8.5,+ loopDurationInFrames: 10,+ }),+ ).toEqual([+ {+ loopIndex: 0,+ absoluteOffsetInFrames: 8.5,+ loopOffsetInFrames: 8.5,+ durationInFrames: 1.5,+ },+ {+ loopIndex: 1,+ absoluteOffsetInFrames: 10,+ loopOffsetInFrames: 0,+ durationInFrames: 3.75,+ },+ ]);+});++test('does not emit an empty segment when the window ends on a boundary', () => {+ const segments = getLoopDisplaySegments({+ displayDurationInFrames: 10,+ displayOffsetInFrames: 10,+ loopDurationInFrames: 10,+ });+ expect(segments).toHaveLength(1);+ expect(segments[0].loopIndex).toBe(1);+ expect(segments[0].durationInFrames).toBe(10);+});++test('stays bounded for fractional offsets that stalled the accumulating loop', () => {+ // Regression: `processed += segmentDuration` used to no-op once the+ // floating-point residue was smaller than the ULP of `processed`,+ // growing arrays until an out-of-memory crash.+ const displayOffsetInFrames = 11256.685714285799;+ const displayDurationInFrames = 33426.30571428571;+ // Stalls the previous accumulating implementation after 12 iterations+ const loopDurationInFrames = 99.999;+ const segments = getLoopDisplaySegments({+ displayDurationInFrames,+ displayOffsetInFrames,+ loopDurationInFrames,+ });++ expect(segments.length).toBeLessThanOrEqual(+ Math.ceil(displayDurationInFrames / loopDurationInFrames) + 1,+ );+ const total = segments.reduce(+ (sum, segment) => sum + segment.durationInFrames,+ 0,+ );+ expect(total).toBeCloseTo(displayDurationInFrames, 6);+ for (const segment of segments) {+ expect(segment.durationInFrames).toBeGreaterThan(0);+ expect(segment.loopOffsetInFrames).toBeGreaterThanOrEqual(0);+ expect(segment.loopOffsetInFrames).toBeLessThan(loopDurationInFrames);+ }+});++test('handles negative display offsets', () => {+ expect(+ getLoopDisplaySegments({+ displayDurationInFrames: 8,+ displayOffsetInFrames: -3,+ loopDurationInFrames: 10,+ }),+ ).toEqual([+ {+ loopIndex: -1,+ absoluteOffsetInFrames: -3,+ loopOffsetInFrames: 7,+ durationInFrames: 3,+ },+ {+ loopIndex: 0,+ absoluteOffsetInFrames: 0,+ loopOffsetInFrames: 0,+ durationInFrames: 5,+ },+ ]);+});++test('returns no segments for degenerate inputs', () => {+ expect(+ getLoopDisplaySegments({+ displayDurationInFrames: 0,+ displayOffsetInFrames: 0,+ loopDurationInFrames: 10,+ }),+ ).toEqual([]);+ expect(+ getLoopDisplaySegments({+ displayDurationInFrames: 10,+ displayOffsetInFrames: 0,+ loopDurationInFrames: 0,+ }),+ ).toEqual([]);+ expect(+ getLoopDisplaySegments({+ displayDurationInFrames: Infinity,+ displayOffsetInFrames: 0,+ loopDurationInFrames: 10,+ }),+ ).toEqual([]);+});diff --git a/packages/timeline-utils/src/test/get-visible-waveform-volume.test.ts b/packages/timeline-utils/src/test/get-visible-waveform-volume.test.tsindex bdf6663ab6..050654388e 100644--- a/packages/timeline-utils/src/test/get-visible-waveform-volume.test.ts+++ b/packages/timeline-utils/src/test/get-visible-waveform-volume.test.ts@@ -66,6 +66,28 @@ test('tiles a long looped volume curve', () => {expect((visible as number[])[loopDurationInFrames - 1]).toBe(0.5);});+test('terminates with fractional display windows from timeline virtualization', () => {+ const volume = Array.from({length: 100}, (_, index) => index / 100);+ const visible = getVisibleWaveformVolume({+ displayDurationInFrames: 33426.30571428571,+ displayOffsetInFrames: 11256.685714285799,+ loopDisplay: {+ // A fractional loop duration whose residue stalls the previous+ // `processed += segmentDuration` implementation below 1 ULP+ durationInFrames: 99.999,+ numberOfTimes: 600,+ startOffset: 0,+ },+ volume,+ });++ expect(Array.isArray(visible)).toBe(true);+ const values = visible as number[];+ expect(values.length).toBeGreaterThan(33426);+ // Bounded by display duration plus per-segment rounding+ expect(values.length).toBeLessThan(34000);+});+test('stops tiling if a loop segment would not advance', () => {expect(getVisibleWaveformVolume({diff --git a/packages/timeline-utils/src/test/slice-visible-waveform-peaks.test.ts b/packages/timeline-utils/src/test/slice-visible-waveform-peaks.test.tsindex 5b21132fb9..b75c12aa06 100644--- a/packages/timeline-utils/src/test/slice-visible-waveform-peaks.test.ts+++ b/packages/timeline-utils/src/test/slice-visible-waveform-peaks.test.ts@@ -41,3 +41,28 @@ test('joins waveform portions across a loop boundary', () => {...Array.from({length: 30}, (_, index) => index),]);});++test('terminates with fractional display windows from timeline virtualization', () => {+ const peaks = Float32Array.from({length: 1000}, (_, index) => index);+ const visible = sliceVisibleWaveformPeaks({+ displayDurationInFrames: 33426.30571428571,+ displayOffsetInFrames: 11256.685714285799,+ durationInFrames: 100,+ fps: 10,+ loopDisplay: {+ // A fractional loop duration whose residue stalls the previous+ // `processed += segmentDuration` implementation below 1 ULP+ durationInFrames: 99.999,+ numberOfTimes: 600,+ startOffset: 0,+ },+ peaks,+ playbackRate: 1,+ startFrom: 0,+ });++ // 10 peaks per frame at fps 10 and TARGET_SAMPLE_RATE 100,+ // plus up to 2 peaks of rounding per loop segment+ expect(visible.length).toBeGreaterThan(334000);+ expect(visible.length).toBeLessThan(336000);+});