🎥 Make videos programmatically with React
`@remotion/media`: Snap preview playback seeks to the track's time resolution
JonnyBurgered8edb282d07
diff --git a/packages/media/src/prewarm-iterator-for-looping.ts b/packages/media/src/prewarm-iterator-for-looping.tsindex 283fc6b3c7..bc83ca3084 100644--- a/packages/media/src/prewarm-iterator-for-looping.ts+++ b/packages/media/src/prewarm-iterator-for-looping.ts@@ -2,27 +2,38 @@ import type {CanvasSink} from 'mediabunny';import type {CanvasAheadOfTimeIterator} from './canvas-ahead-of-time';import {canvasesAheadOfTime} from './canvas-ahead-of-time';-export const makePrewarmedVideoIteratorCache = (videoSink: CanvasSink) => {+export const makePrewarmedVideoIteratorCache = (+ videoSink: CanvasSink,+ timeResolution: number,+) => {const prewarmedVideoIterators: Map<number, CanvasAheadOfTimeIterator> =new Map();+ // Snap to the track's own timestamp grid: WebM stores milliseconds, so frame 62+ // at 30fps (2.0667s) is stored at 2.067s and must not seek into the previous GOP.+ const snapToTrackTimeResolution = (timeToSeek: number) => {+ return Math.round(timeToSeek * timeResolution) / timeResolution;+ };+const prewarmIteratorForLooping = ({timeToSeek}: {timeToSeek: number}) => {- if (!prewarmedVideoIterators.has(timeToSeek)) {+ const snapped = snapToTrackTimeResolution(timeToSeek);+ if (!prewarmedVideoIterators.has(snapped)) {prewarmedVideoIterators.set(- timeToSeek,- canvasesAheadOfTime(videoSink, timeToSeek),+ snapped,+ canvasesAheadOfTime(videoSink, snapped),);}};const makeIteratorOrUsePrewarmed = (timeToSeek: number) => {- const prewarmedIterator = prewarmedVideoIterators.get(timeToSeek);+ const snapped = snapToTrackTimeResolution(timeToSeek);+ const prewarmedIterator = prewarmedVideoIterators.get(snapped);if (prewarmedIterator) {- prewarmedVideoIterators.delete(timeToSeek);+ prewarmedVideoIterators.delete(snapped);return prewarmedIterator;}- return canvasesAheadOfTime(videoSink, timeToSeek);+ return canvasesAheadOfTime(videoSink, snapped);};const destroy = () => {diff --git a/packages/media/src/test/keyframe-bank-seek.test.ts b/packages/media/src/test/keyframe-bank-seek.test.tsindex 3a119ede43..bf6dfe6b6a 100644--- a/packages/media/src/test/keyframe-bank-seek.test.ts+++ b/packages/media/src/test/keyframe-bank-seek.test.ts@@ -1,12 +1,20 @@-import {ALL_FORMATS, Input, UrlSource, VideoSampleSink} from 'mediabunny';+import {+ ALL_FORMATS,+ CanvasSink,+ Input,+ UrlSource,+ VideoSampleSink,+} from 'mediabunny';import {assert, expect, test} from 'vitest';import {getMaxVideoCacheSize,globalMediaCache,keyframeManager,} from '../caches';+import {makePrewarmedVideoIteratorCache} from '../prewarm-iterator-for-looping';import {extractFrame} from '../video-extraction/extract-frame';import {makeKeyframeBank} from '../video-extraction/keyframe-bank';+import {createVideoIterator} from '../video/video-preview-iterator';const openBankAt = async (src: string, timestamp: number) => {const input = new Input({@@ -109,3 +117,38 @@ test('does not skip frames around reordered open-GOP keyframes', async () => {keyframeManager.clearAll('info');});++test('preview playback starts decoding at a millisecond-quantized WebM keyframe', async () => {+ const input = new Input({+ source: new UrlSource('/millisecond-keyframe.webm'),+ formats: ALL_FORMATS,+ });+ const track = await input.getPrimaryVideoTrack();+ assert(track);++ const realSink = new CanvasSink(track);+ const decodedTimestamps: number[] = [];+ const sink = {+ async *canvases(start?: number, end?: number) {+ for await (const canvas of realSink.canvases(start, end)) {+ decodedTimestamps.push(canvas.timestamp);+ yield canvas;+ }+ },+ } as CanvasSink;++ const cache = makePrewarmedVideoIteratorCache(+ sink,+ await track.getTimeResolution(),+ );+ const iterator = await createVideoIterator(62 / 30, cache);++ try {+ expect(iterator.initialFrame?.timestamp).toBe(2.067);+ expect(decodedTimestamps[0]).toBe(2.067);+ } finally {+ iterator.destroy();+ cache.destroy();+ input.dispose();+ }+});diff --git a/packages/media/src/test/variable-fps-issue-8303.test.ts b/packages/media/src/test/variable-fps-issue-8303.test.tsindex e91a47a9c3..d5f6de5282 100644--- a/packages/media/src/test/variable-fps-issue-8303.test.ts+++ b/packages/media/src/test/variable-fps-issue-8303.test.ts@@ -29,7 +29,10 @@ test('variable FPS video can be sought forward one frame at a time', async () =>alpha: true,poolSize: 3,});- const cache = makePrewarmedVideoIteratorCache(canvasSink);+ const cache = makePrewarmedVideoIteratorCache(+ canvasSink,+ await videoTrack.getTimeResolution(),+ );const iterator = await createVideoIterator(0, cache);try {diff --git a/packages/media/src/video-iterator-manager.ts b/packages/media/src/video-iterator-manager.tsindex 4b8507eb2f..ca81f1ef86 100644--- a/packages/media/src/video-iterator-manager.ts+++ b/packages/media/src/video-iterator-manager.ts@@ -124,8 +124,10 @@ export const videoIteratorManager = async ({// Effects expect a source with the size of the output.let upscaledFrame: OffscreenCanvas | null = null;- const prewarmedVideoIteratorCache =- makePrewarmedVideoIteratorCache(canvasSink);+ const prewarmedVideoIteratorCache = makePrewarmedVideoIteratorCache(+ canvasSink,+ await videoTrack.getTimeResolution(),+ );const paintFrame = async (frame: WrappedCanvas): Promise<void> => {if (destroyed) {