Skip to content

🎥 Make videos programmatically with React

`@remotion/media`: Snap preview playback seeks to the track's time resolution

JonnyBurgered8edb282d07
diff --git a/packages/media/src/prewarm-iterator-for-looping.ts b/packages/media/src/prewarm-iterator-for-looping.ts
index 283fc6b3c7..bc83ca3084 100644
--- a/packages/media/src/prewarm-iterator-for-looping.ts
+++ b/packages/media/src/prewarm-iterator-for-looping.ts
@@ -2,27 +2,38 @@ import type {CanvasSink} from 'mediabunny';
import type {CanvasAheadOfTimeIterator} from './canvas-ahead-of-time';
import {canvasesAheadOfTime} from './canvas-ahead-of-time';
-export const makePrewarmedVideoIteratorCache = (videoSink: CanvasSink) => {
+export const makePrewarmedVideoIteratorCache = (
+ videoSink: CanvasSink,
+ timeResolution: number,
+) => {
const prewarmedVideoIterators: Map<number, CanvasAheadOfTimeIterator> =
new Map();
+ // Snap to the track's own timestamp grid: WebM stores milliseconds, so frame 62
+ // at 30fps (2.0667s) is stored at 2.067s and must not seek into the previous GOP.
+ const snapToTrackTimeResolution = (timeToSeek: number) => {
+ return Math.round(timeToSeek * timeResolution) / timeResolution;
+ };
+
const prewarmIteratorForLooping = ({timeToSeek}: {timeToSeek: number}) => {
- if (!prewarmedVideoIterators.has(timeToSeek)) {
+ const snapped = snapToTrackTimeResolution(timeToSeek);
+ if (!prewarmedVideoIterators.has(snapped)) {
prewarmedVideoIterators.set(
- timeToSeek,
- canvasesAheadOfTime(videoSink, timeToSeek),
+ snapped,
+ canvasesAheadOfTime(videoSink, snapped),
);
}
};
const makeIteratorOrUsePrewarmed = (timeToSeek: number) => {
- const prewarmedIterator = prewarmedVideoIterators.get(timeToSeek);
+ const snapped = snapToTrackTimeResolution(timeToSeek);
+ const prewarmedIterator = prewarmedVideoIterators.get(snapped);
if (prewarmedIterator) {
- prewarmedVideoIterators.delete(timeToSeek);
+ prewarmedVideoIterators.delete(snapped);
return prewarmedIterator;
}
- return canvasesAheadOfTime(videoSink, timeToSeek);
+ return canvasesAheadOfTime(videoSink, snapped);
};
const destroy = () => {
diff --git a/packages/media/src/test/keyframe-bank-seek.test.ts b/packages/media/src/test/keyframe-bank-seek.test.ts
index 3a119ede43..bf6dfe6b6a 100644
--- a/packages/media/src/test/keyframe-bank-seek.test.ts
+++ b/packages/media/src/test/keyframe-bank-seek.test.ts
@@ -1,12 +1,20 @@
-import {ALL_FORMATS, Input, UrlSource, VideoSampleSink} from 'mediabunny';
+import {
+ ALL_FORMATS,
+ CanvasSink,
+ Input,
+ UrlSource,
+ VideoSampleSink,
+} from 'mediabunny';
import {assert, expect, test} from 'vitest';
import {
getMaxVideoCacheSize,
globalMediaCache,
keyframeManager,
} from '../caches';
+import {makePrewarmedVideoIteratorCache} from '../prewarm-iterator-for-looping';
import {extractFrame} from '../video-extraction/extract-frame';
import {makeKeyframeBank} from '../video-extraction/keyframe-bank';
+import {createVideoIterator} from '../video/video-preview-iterator';
const openBankAt = async (src: string, timestamp: number) => {
const input = new Input({
@@ -109,3 +117,38 @@ test('does not skip frames around reordered open-GOP keyframes', async () => {
keyframeManager.clearAll('info');
});
+
+test('preview playback starts decoding at a millisecond-quantized WebM keyframe', async () => {
+ const input = new Input({
+ source: new UrlSource('/millisecond-keyframe.webm'),
+ formats: ALL_FORMATS,
+ });
+ const track = await input.getPrimaryVideoTrack();
+ assert(track);
+
+ const realSink = new CanvasSink(track);
+ const decodedTimestamps: number[] = [];
+ const sink = {
+ async *canvases(start?: number, end?: number) {
+ for await (const canvas of realSink.canvases(start, end)) {
+ decodedTimestamps.push(canvas.timestamp);
+ yield canvas;
+ }
+ },
+ } as CanvasSink;
+
+ const cache = makePrewarmedVideoIteratorCache(
+ sink,
+ await track.getTimeResolution(),
+ );
+ const iterator = await createVideoIterator(62 / 30, cache);
+
+ try {
+ expect(iterator.initialFrame?.timestamp).toBe(2.067);
+ expect(decodedTimestamps[0]).toBe(2.067);
+ } finally {
+ iterator.destroy();
+ cache.destroy();
+ input.dispose();
+ }
+});
diff --git a/packages/media/src/test/variable-fps-issue-8303.test.ts b/packages/media/src/test/variable-fps-issue-8303.test.ts
index e91a47a9c3..d5f6de5282 100644
--- a/packages/media/src/test/variable-fps-issue-8303.test.ts
+++ b/packages/media/src/test/variable-fps-issue-8303.test.ts
@@ -29,7 +29,10 @@ test('variable FPS video can be sought forward one frame at a time', async () =>
alpha: true,
poolSize: 3,
});
- const cache = makePrewarmedVideoIteratorCache(canvasSink);
+ const cache = makePrewarmedVideoIteratorCache(
+ canvasSink,
+ await videoTrack.getTimeResolution(),
+ );
const iterator = await createVideoIterator(0, cache);
try {
diff --git a/packages/media/src/video-iterator-manager.ts b/packages/media/src/video-iterator-manager.ts
index 4b8507eb2f..ca81f1ef86 100644
--- a/packages/media/src/video-iterator-manager.ts
+++ b/packages/media/src/video-iterator-manager.ts
@@ -124,8 +124,10 @@ export const videoIteratorManager = async ({
// Effects expect a source with the size of the output.
let upscaledFrame: OffscreenCanvas | null = null;
- const prewarmedVideoIteratorCache =
- makePrewarmedVideoIteratorCache(canvasSink);
+ const prewarmedVideoIteratorCache = makePrewarmedVideoIteratorCache(
+ canvasSink,
+ await videoTrack.getTimeResolution(),
+ );
const paintFrame = async (frame: WrappedCanvas): Promise<void> => {
if (destroyed) {