🎥 Make videos programmatically with React
`@remotion/media`: Reduce decoder recreation during forward playback (#11544)
Igor Samokhovetsb95cafffe9c4
diff --git a/packages/core/src/TimelineContext.tsx b/packages/core/src/TimelineContext.tsxindex dcb4e83d55..f385f111ab 100644--- a/packages/core/src/TimelineContext.tsx+++ b/packages/core/src/TimelineContext.tsx@@ -13,6 +13,8 @@ import {type PlayableMediaTag,} from './timeline-position-state';import {useDelayRender} from './use-delay-render';+import {useRemotionEnvironment} from './use-remotion-environment';+import {useTimelineSeek, type TimelineSeek} from './use-timeline-seek';export type TimelineContextValue = {frame: Record<string, number>;@@ -35,6 +37,8 @@ export type BufferingState = Readonly<{}>;export type SetTimelineContextValue = {+ // Null in rendering and thumbnails, which do not publish preview seek intent.+ seek: TimelineSeek | null;setFrame: (u: React.SetStateAction<Record<string, number>>) => void;setPlaying: (u: React.SetStateAction<boolean>) => void;setBuffering: (buffering: boolean) => void;@@ -53,6 +57,7 @@ const missingSetTimelineContext = (): never => {};export const SetTimelineContext = createContext<SetTimelineContextValue>({+ seek: null,setFrame: missingSetTimelineContext,setPlaying: missingSetTimelineContext,setBuffering: missingSetTimelineContext,@@ -92,6 +97,9 @@ export const TimelineContextProvider: React.FC<{getInitialFrameState(),);+ const timelineSeek = useTimelineSeek(setFrame);+ const {isStudio} = useRemotionEnvironment();+ const seek = isStudio ? timelineSeek : null;const frame = frameState ?? _frame;const frameRef = useRef(frame);frameRef.current = frame;@@ -161,6 +169,7 @@ export const TimelineContextProvider: React.FC<{const setTimelineContextValue = useMemo((): SetTimelineContextValue => {return {setFrame,+ seek,setPlaying: (updater) => {const current = playingStore.store.getSnapshot().playing;const next = typeof updater === 'function' ? updater(current) : updater;@@ -180,7 +189,7 @@ export const TimelineContextProvider: React.FC<{frameRef,audioAndVideoTags,};- }, [bufferingStore, playingStore, readIsBuffering, readIsPlaying]);+ }, [bufferingStore, playingStore, readIsBuffering, readIsPlaying, seek]);return (<AbsoluteTimeContext.Provider value={timelineContextValue}>diff --git a/packages/core/src/internals.ts b/packages/core/src/internals.tsindex 1e9a23555b..3c0085d66b 100644--- a/packages/core/src/internals.ts+++ b/packages/core/src/internals.ts@@ -248,6 +248,7 @@ import {type PropStatuses,} from './use-schema.js';import {useSyncExternalStore} from './use-sync-external-store.js';+import {useTimelineSeek} from './use-timeline-seek.js';import {useUnsafeVideoConfig} from './use-unsafe-video-config.js';import {useVideo} from './use-video.js';import {validateMediaProps} from './validate-media-props.js';@@ -438,6 +439,7 @@ export const Internals = {useAudioEnabled,useBuffering,TimelinePosition,+ useTimelineSeek,DelayRenderContextType,TimelineContext,usePlaying,diff --git a/packages/core/src/test/wrap-sequence-context.tsx b/packages/core/src/test/wrap-sequence-context.tsxindex 2ff4e3c0a8..4eb50dc673 100644--- a/packages/core/src/test/wrap-sequence-context.tsx+++ b/packages/core/src/test/wrap-sequence-context.tsx@@ -131,6 +131,7 @@ export const WrapSequenceContext: React.FC<{const setTimelineContext = useMemo<SetTimelineContextValue>(() => ({setFrame: () => undefined,+ seek: null,setPlaying: () => undefined,setBuffering: (buffering) => {if (bufferingStore.store.getSnapshot().buffering !== buffering) {diff --git a/packages/core/src/timeline-position-state.ts b/packages/core/src/timeline-position-state.tsindex 8a4f1ad567..d4efebad71 100644--- a/packages/core/src/timeline-position-state.ts+++ b/packages/core/src/timeline-position-state.ts@@ -121,5 +121,11 @@ export const useTimelineSetFrame = (): ((return setFrame;};+// Explicit navigation only. Playback and frame persistence must use setFrame.+export const useTimelineSeekFrame = () => {+ const {seek, setFrame} = useContext(SetTimelineContext);+ return seek?.setFrame ?? setFrame;+};+export {usePlaying} from './use-playing.js';export {useBuffering} from './use-buffering.js';diff --git a/packages/core/src/use-timeline-seek.ts b/packages/core/src/use-timeline-seek.tsnew file mode 100644index 0000000000..fa7f572e91--- /dev/null+++ b/packages/core/src/use-timeline-seek.ts@@ -0,0 +1,26 @@+import type {RefObject, SetStateAction} from 'react';+import {useMemo, useRef} from 'react';++export type TimelineSeek = {+ readonly revision: RefObject<number>;+ readonly setFrame: (frame: SetStateAction<Record<string, number>>) => void;+};++// One instance per timeline, shared by Studio and Player. Keep the revision+// outside the React updater: updaters may be deferred or run more than once.+export const useTimelineSeek = (+ setFrame: (frame: SetStateAction<Record<string, number>>) => void,+): TimelineSeek => {+ const revision = useRef(0);+ return useMemo(+ () => ({+ revision,+ setFrame: (frame) => {+ // Even a same-frame seek is a boundary for the next media update.+ revision.current++;+ setFrame(frame);+ },+ }),+ [setFrame],+ );+};diff --git a/packages/media/src/media-player.ts b/packages/media/src/media-player.tsindex 3cbb20f8fa..1b3084899c 100644--- a/packages/media/src/media-player.ts+++ b/packages/media/src/media-player.ts@@ -40,6 +40,14 @@ export type MediaPlayerInitResult =| {type: 'no-tracks'}| {type: 'disposed'};+export type MediaSeekIntent =+ | {readonly revision: number; readonly playing: boolean}+ // An internal resync after media configuration changes, not a timeline seek.+ // Invalidate the satisfied revision so the next video request cannot be+ // mistaken for continuous playback. This does not always restart the iterator.+ | 'discontinuity'+ | null;+export class MediaPlayer {private tagType: 'audio' | 'video';private canvas: HTMLCanvasElement | OffscreenCanvas | null;@@ -61,6 +69,8 @@ export class MediaPlayer {videoIteratorManager: VideoIteratorManager | null = null;private playing = false;+ private lastSeekIntent: MediaSeekIntent = null;+ private satisfiedSeekRevision: number | null = null;private loop = false;private fps: number;@@ -488,6 +498,7 @@ export class MediaPlayer {private seekToWithQueue = async (newTime: number,unloopedNewTime: number,+ intent: MediaSeekIntent,) => {const nonce = this.nonceManager.createAsyncOperation();await this.seekPromiseChain;@@ -496,11 +507,12 @@ export class MediaPlayer {newTime,unloopedNewTime,nonce,+ intent,);await this.seekPromiseChain;};- public async seekTo(time: number): Promise<void> {+ public async seekTo(time: number, intent: MediaSeekIntent): Promise<void> {if (this.terminalError || this.isDisposalError()) {return;}@@ -511,13 +523,20 @@ export class MediaPlayer {throw new Error(`should have asserted that the time is not null`);}- await this.seekToWithQueue(newTime, time);+ this.lastSeekIntent = intent;+ if (intent === 'discontinuity') {+ // A mapping change may supersede a queued timeline request.+ this.satisfiedSeekRevision = null;+ }++ await this.seekToWithQueue(newTime, time, intent);}private async seekToDoNotCallDirectly(newTime: number,unloopedNewTime: number,nonce: Nonce,+ intent: MediaSeekIntent,): Promise<void> {if (nonce.isStale() || this.terminalError || this.isDisposalError()) {return;@@ -531,6 +550,12 @@ export class MediaPlayer {fps: this.fps,playbackRate: this.playbackRate,isPlaying: this.playing,+ continuousPlayback:+ intent === null+ ? null+ : intent !== 'discontinuity' &&+ intent.playing &&+ intent.revision === this.satisfiedSeekRevision,}),this.audioIteratorManager?.seek({newTime,@@ -551,6 +576,10 @@ export class MediaPlayer {this.sharedAudioContext!.audioContext.currentTime,}),]);+ // Superseded work must not consume an explicit seek boundary.+ if (!nonce.isStale() && intent !== null && intent !== 'discontinuity') {+ this.satisfiedSeekRevision = intent.revision;+ }} catch (error) {if (this.isDisposalError()) {return;@@ -619,7 +648,7 @@ export class MediaPlayer {if (this.trimBefore !== trimBefore) {this.trimBefore = trimBefore;this.audioIteratorManager?.destroyIterator();- await this.seekTo(unloopedTimeInSeconds);+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');}}@@ -630,7 +659,7 @@ export class MediaPlayer {if (this.trimAfter !== trimAfter) {this.trimAfter = trimAfter;this.audioIteratorManager?.destroyIterator();- await this.seekTo(unloopedTimeInSeconds);+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');}}@@ -647,7 +676,7 @@ export class MediaPlayer {if (previousRate !== rate) {this.playbackRate = rate;this.audioIteratorManager?.destroyIterator();- await this.seekTo(unloopedTimeInSeconds);+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');}}@@ -663,7 +692,7 @@ export class MediaPlayer {this.audioIteratorManager.setToneFrequency(toneFrequency);this.audioIteratorManager.destroyIterator();- await this.seekTo(unloopedTimeInSeconds);+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');}}@@ -675,7 +704,7 @@ export class MediaPlayer {if (previousRate !== rate) {this.globalPlaybackRate = rate;this.audioIteratorManager?.destroyIterator();- await this.seekTo(unloopedTimeInSeconds);+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');}}@@ -687,7 +716,7 @@ export class MediaPlayer {if (previousFps !== fps) {this.fps = fps;this.audioIteratorManager?.destroyIterator();- await this.seekTo(unloopedTimeInSeconds);+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');}}@@ -709,7 +738,7 @@ export class MediaPlayer {if (previousLoop !== loop) {this.loop = loop;this.audioIteratorManager?.destroyIterator();- await this.seekTo(unloopedTimeInSeconds);+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');}}@@ -721,7 +750,7 @@ export class MediaPlayer {if (previousOffset !== offset) {this.sequenceOffset = offset;this.audioIteratorManager?.destroyIterator();- await this.seekTo(unloopedTimeInSeconds);+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');}}@@ -732,7 +761,7 @@ export class MediaPlayer {const previousDuration = this.sequenceDurationInFrames;if (previousDuration !== sequenceDurationInFrames) {this.sequenceDurationInFrames = sequenceDurationInFrames;- await this.seekTo(unloopedTimeInSeconds);+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');}}@@ -904,6 +933,15 @@ export class MediaPlayer {this.audioIteratorManager.destroyIterator();// An anchor can change while paused, when no frame update will restart// scheduling. Refill the queue against the new anchor immediately.- await this.seekTo(unloopedTimeInSeconds);+ // Re-anchoring is not navigation. Preserve the captured revision so an+ // outstanding explicit seek still takes the restart path, but don't let+ // an audio-clock pause turn ordinary video catch-up into a scrub.+ const intent = this.lastSeekIntent;+ await this.seekTo(+ unloopedTimeInSeconds,+ intent !== null && intent !== 'discontinuity'+ ? {...intent, playing: true}+ : intent,+ );};}diff --git a/packages/media/src/test/media-player.test.ts b/packages/media/src/test/media-player.test.tsindex a084ef3da4..1060f54d85 100644--- a/packages/media/src/test/media-player.test.ts+++ b/packages/media/src/test/media-player.test.ts@@ -79,6 +79,32 @@ test('audio-only file should initialize without `audioStreamIndex` (regression fawait player.dispose();});+test('queued playback cannot consume a superseded seek boundary', async () => {+ const player = makeAudioPlayer(null, null, 'video');+ try {+ expect((await player.initialize(0, true, 1)).type).toBe('success');+ const manager = player.videoIteratorManager!;+ await player.seekTo(0, {revision: 0, playing: true});+ await player.seekTo(2, {revision: 0, playing: true});+ expect(manager.getVideoIteratorsCreated()).toBe(1);+ expect(manager.getFramesRendered()).toBeGreaterThan(1);++ // Both requests are enqueued before either executes. The first request+ // goes stale, but the second still needs to establish revision 1.+ await Promise.all([+ player.seekTo(8, {revision: 1, playing: true}),+ player.seekTo(8.1, {revision: 1, playing: true}),+ ]);+ expect(manager.getVideoIteratorsCreated()).toBe(2);+ await player.seekTo(8.5, {revision: 1, playing: true});+ expect(manager.getVideoIteratorsCreated()).toBe(2);+ await player.seekTo(0, {revision: 1, playing: true});+ expect(manager.getVideoIteratorsCreated()).toBe(3);+ } finally {+ await player.dispose();+ }+});+test('uses the initial volume before audio playback is ready', async () => {const sharedAudioContext = makeSharedAudioContext();const mediaGainNodes: GainNode[] = [];@@ -111,7 +137,7 @@ test('uses the initial volume before audio playback is ready', async () => {}});-test('dispose should immediately unblock playback delays', async () => {+test('dispose should immediately unblock a pending playback catch-up', async () => {let activeBlocks = 0;let delayPlaybackCalled: () => void = () => {};@@ -173,12 +199,13 @@ test('dispose should immediately unblock playback delays', async () => {});await player.initialize(0, false, 1);+ await player.seekTo(0, {revision: 0, playing: true});const seekDelayPromise = new Promise<void>((resolve) => {delayPlaybackCalled = resolve;});- const seekPromise = player.seekTo(9);+ const seekPromise = player.seekTo(9, {revision: 0, playing: true});await seekDelayPromise;@@ -201,9 +228,9 @@ test.each(['video'] as const)(player[`${tagType}IteratorManager`] = {seek, destroy: vi.fn()} as never;player.play();- await player.seekTo(1);+ await player.seekTo(1, null);player.play();- await player.seekTo(2);+ await player.seekTo(2, null);expect(onError).toHaveBeenCalledOnce();expect(onError).toHaveBeenCalledWith(error);@@ -252,13 +279,13 @@ test.each(['audio', 'video'] as const)(},);try {- await player.seekTo(1);+ await player.seekTo(1, null);await vi.waitFor(() => expect(rejectRead).toBeDefined());rejectRead(error);await vi.waitFor(() => expect(onError).toHaveBeenCalledWith(error));player.play();- await player.seekTo(2);+ await player.seekTo(2, null);expect(onError).toHaveBeenCalledOnce();expect(unblocks).toHaveBeenCalled();expect(buffers).toHaveBeenCalledOnce();@@ -283,14 +310,14 @@ test('ignores pending seek failures after disposal and skips new seeks', async ();player.videoIteratorManager = {seek} as never;- const pending = player.seekTo(1);+ const pending = player.seekTo(1, null);await vi.waitFor(() => expect(seek).toHaveBeenCalledOnce());player.videoIteratorManager = null;await player.dispose();rejectSeek(new TypeError('Failed to fetch'));await pending;player.videoIteratorManager = {seek} as never;- await player.seekTo(2);+ await player.seekTo(2, null);expect(onError).not.toHaveBeenCalled();expect(seek).toHaveBeenCalledOnce();diff --git a/packages/media/src/test/player-seek-intent.test.tsx b/packages/media/src/test/player-seek-intent.test.tsxnew file mode 100644index 0000000000..507393f8dc--- /dev/null+++ b/packages/media/src/test/player-seek-intent.test.tsx@@ -0,0 +1,110 @@+import {Player, type PlayerRef} from '@remotion/player';+import React, {useContext} from 'react';+import {createRoot} from 'react-dom/client';+import {Internals, useBufferState} from 'remotion';+import {expect, test, vi} from 'vitest';+import {Video} from '../video/video';+import {findLargestCanvas, waitFor} from './player-frame-accuracy-utils';++test('Player distinguishes playback catch-up while buffering from explicit seeks', async () => {+ let timeline: React.ContextType<typeof Internals.SetTimelineContext> | null =+ null;+ let bufferState: ReturnType<typeof useBufferState> | null = null;+ let draws = 0;+ const errors: Error[] = [];+ const Composition = () => {+ timeline = useContext(Internals.SetTimelineContext);+ bufferState = useBufferState();+ return (+ <Video+ src="/bigbuckbunny.mp4"+ muted+ onVideoFrame={() => {+ draws++;+ }}+ onError={(error) => {+ errors.push(error);+ }}+ />+ );+ };++ const container = document.createElement('div');+ document.body.appendChild(container);+ const root = createRoot(container);+ const ref = React.createRef<PlayerRef>();+ const configurations = vi.spyOn(VideoDecoder.prototype, 'configure');+ let foreignBuffer: {unblock: () => void} | null = null;+ root.render(+ <Player+ ref={ref}+ component={Composition}+ compositionWidth={320}+ compositionHeight={180}+ durationInFrames={300}+ fps={30}+ numberOfSharedAudioTags={0}+ />,+ );+ try {+ await waitFor(() => draws > 0 && !timeline!.isBuffering());+ // Establish the correct output through a public paused seek.+ let before = draws;+ ref.current!.seekTo(60);+ await waitFor(() => draws > before && !timeline!.isBuffering());+ const target = findLargestCanvas(container).toDataURL();+ before = draws;+ ref.current!.seekTo(0);+ await waitFor(() => draws > before && !timeline!.isBuffering());+ expect(findLargestCanvas(container).toDataURL()).not.toBe(target);++ // Another layer blocks the clock. A missed playback update still must+ // consume the existing iterator, not guess intent from the time gap.+ foreignBuffer = bufferState!.delayPlayback();+ ref.current!.play();+ await waitFor(() => ref.current!.isPlaying() && timeline!.isBuffering());+ const configuredBeforeCatchup = configurations.mock.calls.length;+ const revision = timeline!.seek!.revision.current;+ before = draws;+ timeline!.setFrame((frames) => ({...frames, [Object.keys(frames)[0]]: 60}));+ await waitFor(+ () =>+ draws > before && findLargestCanvas(container).toDataURL() === target,+ );+ expect(ref.current!.getCurrentFrame()).toBe(60);+ expect(timeline!.seek!.revision.current).toBe(revision);+ expect(configurations.mock.calls.length).toBe(configuredBeforeCatchup);++ // The same playing/buffering state, but now a deliberate distant seek.+ before = draws;+ ref.current!.seekTo(240);+ await waitFor(+ () => draws > before && ref.current!.getCurrentFrame() === 240,+ );+ expect(configurations.mock.calls.length).toBeGreaterThan(+ configuredBeforeCatchup,+ );+ expect(findLargestCanvas(container).toDataURL()).not.toBe(target);++ before = draws;+ ref.current!.seekTo(120);+ ref.current!.seekTo(15);+ ref.current!.seekTo(60);+ ref.current!.pause();+ foreignBuffer.unblock();+ foreignBuffer = null;+ await waitFor(+ () =>+ draws > before &&+ !timeline!.isBuffering() &&+ findLargestCanvas(container).toDataURL() === target,+ );+ expect(ref.current!.getCurrentFrame()).toBe(60);+ expect(errors).toEqual([]);+ } finally {+ foreignBuffer?.unblock();+ root.unmount();+ container.remove();+ configurations.mockRestore();+ }+}, 30000);diff --git a/packages/media/src/test/variable-fps-issue-8303.test.ts b/packages/media/src/test/variable-fps-issue-8303.test.tsindex cdf67ea3e7..e91a47a9c3 100644--- a/packages/media/src/test/variable-fps-issue-8303.test.ts+++ b/packages/media/src/test/variable-fps-issue-8303.test.ts@@ -38,6 +38,7 @@ test('variable FPS video can be sought forward one frame at a time', async () =>for (let frame = 0; frame <= 210; frame++) {const time = frame / FPS;const result = await iterator.tryToSatisfySeek(time, {+ onWait: () => undefined,pendingFrameBehavior: 'wait',shouldContinue: () => true,});diff --git a/packages/media/src/test/video-iterator-manager.test.ts b/packages/media/src/test/video-iterator-manager.test.tsindex b1ba294da6..a333c792c1 100644--- a/packages/media/src/test/video-iterator-manager.test.ts+++ b/packages/media/src/test/video-iterator-manager.test.ts@@ -115,6 +115,7 @@ test('plays at a high playback rate without restarting the iterator', async () =fps: 30,playbackRate: 3.75,isPlaying: true,+ continuousPlayback: null,});}@@ -147,6 +148,7 @@ test('paused forward scrubs do not wait for pending frames', async () => {};await manager.seek({+ continuousPlayback: null,newTime: 0.1,nonce: nonceManager.createAsyncOperation(),fps: 30,@@ -204,6 +206,7 @@ test('seek should not cause overlapping block/unblock cycles', async () => {// Perform seeks that will trigger 'not-satisfied' (jumping around)// These should NOT cause overlapping blocksawait manager.seek({+ continuousPlayback: null,newTime: 5,nonce: nonceManager.createAsyncOperation(),fps: 30,@@ -211,6 +214,7 @@ test('seek should not cause overlapping block/unblock cycles', async () => {isPlaying: false,});await manager.seek({+ continuousPlayback: null,newTime: 0,nonce: nonceManager.createAsyncOperation(),fps: 30,@@ -218,6 +222,7 @@ test('seek should not cause overlapping block/unblock cycles', async () => {isPlaying: false,});await manager.seek({+ continuousPlayback: null,newTime: 8,nonce: nonceManager.createAsyncOperation(),fps: 30,@@ -277,6 +282,7 @@ test('rapid sequential seeks should not cause overlapping blocks', async () => {// Alternate between distant positions to force iterator recreationconst time = i % 2 === 0 ? i * 0.5 : 9 - i * 0.5;await manager.seek({+ continuousPlayback: null,newTime: time,nonce: nonceManager.createAsyncOperation(),fps: 30,diff --git a/packages/media/src/test/video-preview-iterator.test.ts b/packages/media/src/test/video-preview-iterator.test.tsindex 37392e9b8b..bfe681dce0 100644--- a/packages/media/src/test/video-preview-iterator.test.ts+++ b/packages/media/src/test/video-preview-iterator.test.ts@@ -44,6 +44,7 @@ test('preview iterator uses next frame timestamp instead of reported duration',try {const result = await iterator.tryToSatisfySeek(1.5, {+ onWait: () => undefined,pendingFrameBehavior: 'wait',shouldContinue: () => true,});@@ -95,6 +96,7 @@ test('preview iterator does not trust reported duration without a next timestamptry {const result = await iterator.tryToSatisfySeek(1.5, {+ onWait: () => undefined,pendingFrameBehavior: 'restart-iterator',shouldContinue: () => true,});@@ -143,6 +145,7 @@ test('preview iterator waits through high-FPS frames for a sequential timeline stry {const result = await iterator.tryToSatisfySeek(1 / timelineFps, {+ onWait: () => undefined,pendingFrameBehavior: 'wait',shouldContinue: () => true,});@@ -192,6 +195,7 @@ test('preview iterator supports frames longer than one timeline step', async ()try {const result = await iterator.tryToSatisfySeek(0.075, {+ onWait: () => undefined,pendingFrameBehavior: 'wait',shouldContinue: () => true,});@@ -245,6 +249,7 @@ test('preview iterator stops decoding when a seek is superseded', async () => {try {const result = await iterator.tryToSatisfySeek(1, {+ onWait: () => undefined,pendingFrameBehavior: 'wait',shouldContinue: () => shouldContinue,});diff --git a/packages/media/src/test/video-with-no-frames-in-beginning.test.ts b/packages/media/src/test/video-with-no-frames-in-beginning.test.tsindex cadb9c495e..fad14b6788 100644--- a/packages/media/src/test/video-with-no-frames-in-beginning.test.ts+++ b/packages/media/src/test/video-with-no-frames-in-beginning.test.ts@@ -53,6 +53,7 @@ test('in preview, should properly buffer and draw frames', async (t) => {await manager.startVideoIterator(0, nonceManager.createAsyncOperation());await manager.seek({newTime: 0.03,+ continuousPlayback: null,nonce: nonceManager.createAsyncOperation(),fps: 30,playbackRate: 1,@@ -60,6 +61,7 @@ test('in preview, should properly buffer and draw frames', async (t) => {});await manager.seek({newTime: 1,+ continuousPlayback: null,nonce: nonceManager.createAsyncOperation(),fps: 30,playbackRate: 1,@@ -67,6 +69,7 @@ test('in preview, should properly buffer and draw frames', async (t) => {});await manager.seek({newTime: 2,+ continuousPlayback: null,nonce: nonceManager.createAsyncOperation(),fps: 30,playbackRate: 1,@@ -78,6 +81,7 @@ test('in preview, should properly buffer and draw frames', async (t) => {await manager.seek({newTime: 4.5,+ continuousPlayback: null,nonce: nonceManager.createAsyncOperation(),fps: 30,playbackRate: 1,diff --git a/packages/media/src/use-common-effects.ts b/packages/media/src/use-common-effects.tsindex d74cc56bc6..b581d50258 100644--- a/packages/media/src/use-common-effects.ts+++ b/packages/media/src/use-common-effects.ts@@ -1,5 +1,5 @@import type React from 'react';-import {useContext, useLayoutEffect} from 'react';+import {useContext, useLayoutEffect, useRef} from 'react';import type {LogLevel} from 'remotion';import {Internals} from 'remotion';import type {MediaPlayer} from './media-player';@@ -212,16 +212,46 @@ export const useCommonEffects = ({mediaPlayer.setSequenceOffset(sequenceOffset, currentTimeRef.current);}, [sequenceOffset, mediaPlayerReady, mediaPlayerRef, currentTimeRef]);+ const {seek} = useContext(Internals.SetTimelineContext);+ const revision = seek?.revision.current ?? null;+ const lastRequest = useRef<{+ player: MediaPlayer;+ time: number;+ revision: number | null;+ } | null>(null);+useLayoutEffect(() => {const mediaPlayer = mediaPlayerRef.current;if (!mediaPlayer || !mediaPlayerReady) return;- mediaPlayer.seekTo(currentTime).catch(() => {- // Might be disposed- });+ // Pausing or buffering at the same target must not supersede an in-flight+ // playback request with a scrub. A same-frame explicit seek still counts.+ const previous = lastRequest.current;+ if (+ previous?.player === mediaPlayer &&+ previous.time === currentTime &&+ previous.revision === revision+ ) {+ return;+ }++ lastRequest.current = {player: mediaPlayer, time: currentTime, revision};+ mediaPlayer+ .seekTo(currentTime, revision === null ? null : {revision, playing})+ .catch(() => {+ // Might be disposed+ });Internals.Log.trace({logLevel, tag: '@remotion/media'},`[${label}] Updating target time to ${currentTime.toFixed(3)}s`,);- }, [currentTime, logLevel, mediaPlayerReady, label, mediaPlayerRef]);+ }, [+ currentTime,+ logLevel,+ mediaPlayerReady,+ label,+ mediaPlayerRef,+ playing,+ revision,+ ]);};diff --git a/packages/media/src/video-iterator-manager.ts b/packages/media/src/video-iterator-manager.tsindex dd6a681bb0..d84c378407 100644--- a/packages/media/src/video-iterator-manager.ts+++ b/packages/media/src/video-iterator-manager.ts@@ -34,6 +34,7 @@ export const isSequentialMediaTimeAdvance = ({return false;}+ // Legacy fallback for hosts without explicit preview seek intent.const maximumSequentialAdvance = Math.abs(playbackRate) / fps;return (roundTo4Digits(newTime - previousTime) <=@@ -214,12 +215,14 @@ export const videoIteratorManager = async ({fps,playbackRate,isPlaying,+ continuousPlayback,}: {newTime: number;nonce: Nonce;fps: number;playbackRate: number;isPlaying: boolean;+ continuousPlayback: boolean | null;}) => {if (!videoFrameIterator) {return;@@ -233,7 +236,6 @@ export const videoIteratorManager = async ({}const previousTime = currentSeek;- currentSeek = newTime;if (getIsLooping()) {// If less than 1 second from the end away, we pre-warm a new iterator@@ -246,36 +248,52 @@ export const videoIteratorManager = async ({const pendingFrameBehavior =previousTime !== null &&- isSequentialMediaTimeAdvance({- previousTime,- newTime,- fps,- playbackRate,- isPlaying,- })+ newTime >= previousTime &&+ (continuousPlayback ??+ isSequentialMediaTimeAdvance({+ previousTime,+ newTime,+ fps,+ playbackRate,+ isPlaying,+ }))? 'wait': 'restart-iterator';- const videoSatisfyResult = await videoFrameIterator.tryToSatisfySeek(- newTime,- {+ const iterator = videoFrameIterator;+ const pending: {handle: DelayPlaybackIfNotPremounting | null} = {+ handle: null,+ };+ try {+ const result = await iterator.tryToSatisfySeek(newTime, {pendingFrameBehavior,- shouldContinue: () => !nonce.isStale(),- },- );+ onWait: () => {+ pending.handle ??= delayPlaybackHandleIfNotPremounting();+ currentDelayHandle = pending.handle;+ },+ shouldContinue: () => !nonce.isStale() && !iterator.isDestroyed(),+ });++ if (nonce.isStale() || iterator.isDestroyed()) {+ return;+ }- // Doing this before the staleness check, because- // frame might be better than what we currently have- // TODO: check if this is actually true- if (videoSatisfyResult.type === 'satisfied') {- await drawFrame(videoSatisfyResult.frame);- return;- }+ if (result.type === 'satisfied') {+ await drawFrame(result.frame);+ currentSeek = newTime;+ return;+ }- if (nonce.isStale()) {- return;+ // The replacement iterator owns its own buffering handle.+ pending.handle?.unblock();+ pending.handle = null;+ currentDelayHandle = null;+ await startVideoIterator(newTime, nonce);+ } finally {+ pending.handle?.unblock();+ if (currentDelayHandle === pending.handle) {+ currentDelayHandle = null;+ }}-- await startVideoIterator(newTime, nonce);};return {diff --git a/packages/media/src/video/video-preview-iterator.ts b/packages/media/src/video/video-preview-iterator.tsindex 0ac96840ad..e736c2f824 100644--- a/packages/media/src/video/video-preview-iterator.ts+++ b/packages/media/src/video/video-preview-iterator.ts@@ -21,6 +21,11 @@ export const createVideoIterator = async (let peekedFrame: WrappedCanvas | null = null;const setLastReturnedFrame = (frame: WrappedCanvas) => {+ if (destroyed) {+ releaseStableFrame(frame);+ return;+ }+if (lastReturnedFrame !== frame) {releaseStableFrame(lastReturnedFrame);}@@ -29,6 +34,11 @@ export const createVideoIterator = async (};const setPeekedFrame = (frame: WrappedCanvas | null) => {+ if (destroyed) {+ releaseStableFrame(frame);+ return null;+ }+peekedFrame = frame;if (peekedFrame === null) {iteratorEnded = true;@@ -63,9 +73,11 @@ export const createVideoIterator = async (const getFrameEndTimestamp = async ({pendingFrameBehavior,+ onWait,shouldContinue,}: {pendingFrameBehavior: 'wait' | 'restart-iterator';+ onWait: () => void;shouldContinue: () => boolean;}) => {const peeked = peekIfReady();@@ -84,6 +96,7 @@ export const createVideoIterator = async (return {type: 'cancelled' as const};}+ onWait();const awaitedPeeked = setPeekedFrame(await peeked.wait());if (!shouldContinue()) {return {type: 'cancelled' as const};@@ -153,6 +166,7 @@ export const createVideoIterator = async (time: number,options: {pendingFrameBehavior: 'wait' | 'restart-iterator';+ onWait: () => void;shouldContinue: () => boolean;},): Promise<@@ -196,6 +210,7 @@ export const createVideoIterator = async (}const frameEndTimestamp = await getFrameEndTimestamp({+ onWait: options.onWait,pendingFrameBehavior: options.pendingFrameBehavior,shouldContinue: options.shouldContinue,});@@ -246,13 +261,26 @@ export const createVideoIterator = async (};}- const frame = getNextOrNullIfNotAvailable();+ let frame = getNextOrNullIfNotAvailable();if (frame.type === 'need-to-wait-for-it') {- return {- type: 'not-satisfied' as const,- reason: 'iterator did not have frame ready',- };+ if (options.pendingFrameBehavior === 'restart-iterator') {+ return {+ type: 'not-satisfied' as const,+ reason: 'iterator did not have frame ready',+ };+ }++ options.onWait();+ const awaitedFrame = await frame.waitPromise();+ if (!options.shouldContinue()) {+ return {+ type: 'not-satisfied',+ reason: 'seek was superseded',+ };+ }++ frame = {type: 'got-frame-or-end', frame: awaitedFrame};}if (frame.type === 'got-frame-or-end') {@@ -273,6 +301,7 @@ export const createVideoIterator = async (const frameTimestamp = roundTo4Digits(frame.frame.timestamp);const frameEndTimestamp = await getFrameEndTimestamp({+ onWait: options.onWait,pendingFrameBehavior: options.pendingFrameBehavior,shouldContinue: options.shouldContinue,});diff --git a/packages/media/vitest.config.ts b/packages/media/vitest.config.tsindex 2fd8e5434f..6c2bc3531f 100644--- a/packages/media/vitest.config.ts+++ b/packages/media/vitest.config.ts@@ -3,6 +3,18 @@ import {webdriverio} from '@vitest/browser-webdriverio';import {defineConfig} from 'vitest/config';export default defineConfig({+ resolve: {dedupe: ['react', 'react-dom', 'remotion']},+ // Avoid discovering another React runtime midway through browser tests.+ optimizeDeps: {+ include: [+ 'react',+ 'react/jsx-runtime',+ 'react/jsx-dev-runtime',+ 'react-dom/client',+ 'remotion',+ '@remotion/player',+ ],+ },test: {browser: {provider: webdriverio(),diff --git a/packages/player/src/Player.tsx b/packages/player/src/Player.tsxindex 4c70214194..68c9d1b13a 100644--- a/packages/player/src/Player.tsx+++ b/packages/player/src/Player.tsx@@ -246,6 +246,7 @@ const PlayerFn = <const [frame, setFrame] = useState<Record<string, number>>(() => ({[PLAYER_COMP_ID]: initialFrame ?? 0,}));+ const seek = Internals.useTimelineSeek(setFrame);const frameRef = useRef(frame);frameRef.current = frame;const rootRef = useRef<PlayerRef>(null);@@ -446,6 +447,7 @@ const PlayerFn = <const setTimelineContextValue = useMemo((): SetTimelineContextValue => {return {setFrame,+ seek,setPlaying: (updater) => {const current = playingStore.store.getSnapshot().playing;const next = typeof updater === 'function' ? updater(current) : updater;@@ -467,6 +469,7 @@ const PlayerFn = <};}, [bufferingStore,+ seek,setFrame,frameRef,playingStore,diff --git a/packages/player/src/Thumbnail.tsx b/packages/player/src/Thumbnail.tsxindex cd18f99633..01b8e1ba48 100644--- a/packages/player/src/Thumbnail.tsx+++ b/packages/player/src/Thumbnail.tsx@@ -116,6 +116,7 @@ const ThumbnailFn = <const setTimelineContext: SetTimelineContextValue = useMemo(() => {return {setFrame: () => undefined,+ seek: null,setPlaying: () => undefined,setBuffering: (buffering) => {if (bufferingStore.store.getSnapshot().buffering !== buffering) {diff --git a/packages/player/src/test/seek-intent.test.tsx b/packages/player/src/test/seek-intent.test.tsxnew file mode 100644index 0000000000..8eed7fdea9--- /dev/null+++ b/packages/player/src/test/seek-intent.test.tsx@@ -0,0 +1,73 @@+import {afterEach, expect, test} from 'bun:test';+import React, {createRef, useContext} from 'react';+import {Internals, useCurrentFrame} from 'remotion';+import type {PlayerRef} from '../player-methods.js';+import {Player} from '../Player.js';+import {act, cleanup, render} from './test-utils.js';++afterEach(cleanup);++test('Player seeks publish boundaries before frame updates and stay local to each Player', () => {+ const players = [createRef<PlayerRef>(), createRef<PlayerRef>()];+ const ids = ['first', 'second'];+ const timelines: React.ContextType<typeof Internals.SetTimelineContext>[] =+ [];+ const Composition: React.FC<{readonly index: number}> = ({index}) => {+ timelines[index] = useContext(Internals.SetTimelineContext);+ return <output data-testid={`frame-${index}`}>{useCurrentFrame()}</output>;+ };++ const view = render(+ <>+ {players.map((ref, index) => (+ <Player+ key={ids[index]}+ ref={ref}+ component={Composition}+ inputProps={{index}}+ durationInFrames={300}+ compositionWidth={320}+ compositionHeight={180}+ fps={30}+ numberOfSharedAudioTags={0}+ />+ ))}+ </>,+ );+ const {revision} = timelines[0].seek!;+ const events: number[] = [];+ players[0].current!.addEventListener('seeked', () => {+ events.push(revision.current);+ });++ act(() => {+ players[0].current!.seekTo(30);+ expect(revision.current).toBe(1);+ });+ expect(view.getByTestId('frame-0').textContent).toBe('30');+ act(() => players[0].current!.play());+ expect(revision.current).toBe(1);+ act(() => {+ players[0].current!.seekTo(31);+ players[0].current!.seekTo(180);+ players[0].current!.seekTo(15);+ });+ expect(view.getByTestId('frame-0').textContent).toBe('15');+ expect(revision.current).toBe(4);+ act(() => players[0].current!.pause());+ act(() => players[0].current!.seekTo(15));+ expect(revision.current).toBe(5);+ expect(events).toEqual([1, 2, 3, 4, 5]);+ expect(view.getByTestId('frame-1').textContent).toBe('0');+ expect(timelines[1].seek!.revision.current).toBe(0);++ // The animation path updates frames without publishing navigation intent.+ act(() =>+ timelines[0].setFrame((frames) => ({+ ...frames,+ [Object.keys(frames)[0]]: 90,+ })),+ );+ expect(view.getByTestId('frame-0').textContent).toBe('90');+ expect(revision.current).toBe(5);+});diff --git a/packages/player/src/use-player-methods.ts b/packages/player/src/use-player-methods.tsindex eb70afa3f4..4b9d3aea0d 100644--- a/packages/player/src/use-player-methods.ts+++ b/packages/player/src/use-player-methods.ts@@ -20,8 +20,8 @@ export type UsePlayerMethods = {};export const usePlayerMethods = (): UsePlayerMethods => {- const setFrame = Internals.Timeline.useTimelineSetFrame();- const setTimelinePosition = Internals.Timeline.useTimelineSetFrame();+ const setFrame = Internals.Timeline.useTimelineSeekFrame();+ const setTimelinePosition = setFrame;const {setPlaying,frameRef,@@ -77,18 +77,17 @@ export const usePlayerMethods = (): UsePlayerMethods => {fallbackFrame.current = frameToSeekTo;if (video?.id) {+ setTimelinePosition((currentFrames) =>+ currentFrames[video.id] === frameToSeekTo+ ? currentFrames+ : {...currentFrames, [video.id]: frameToSeekTo},+ );if (frameRef.current[video.id] !== frameToSeekTo) {frameRef.current = {...frameRef.current,[video.id]: frameToSeekTo,};}-- setTimelinePosition((currentFrames) =>- currentFrames[video.id] === frameToSeekTo- ? currentFrames- : {...currentFrames, [video.id]: frameToSeekTo},- );}emitter.dispatchSeek(frameToSeekTo);@@ -165,14 +164,14 @@ export const usePlayerMethods = (): UsePlayerMethods => {setPlaying(false);fallbackFrame.current = playStart.current;if (config) {- frameRef.current = {- ...frameRef.current,- [config.id]: playStart.current,- };setTimelinePosition((currentFrames) => ({...currentFrames,[config.id]: playStart.current,}));+ frameRef.current = {+ ...frameRef.current,+ [config.id]: playStart.current,+ };emitter.dispatchPause();}}@@ -205,15 +204,15 @@ export const usePlayerMethods = (): UsePlayerMethods => {return;}- frameRef.current = {- ...frameRef.current,- [videoId]: newFrame,- };setFrame((currentFrames) =>currentFrames[videoId] === newFrame? currentFrames: {...currentFrames, [videoId]: newFrame},);+ frameRef.current = {+ ...frameRef.current,+ [videoId]: newFrame,+ };},[frameRef, readIsPlaying, setFrame, videoId],);@@ -235,15 +234,15 @@ export const usePlayerMethods = (): UsePlayerMethods => {return;}- frameRef.current = {- ...frameRef.current,- [videoId]: newFrame,- };setFrame((currentFrames) =>currentFrames[videoId] === newFrame? currentFrames: {...currentFrames, [videoId]: newFrame},);+ frameRef.current = {+ ...frameRef.current,+ [videoId]: newFrame,+ };},[frameRef, lastFrame, readIsPlaying, setFrame, videoId],);diff --git a/packages/studio/src/components/InitialCompositionLoader.tsx b/packages/studio/src/components/InitialCompositionLoader.tsxindex 64398ea2a1..d0ff40497c 100644--- a/packages/studio/src/components/InitialCompositionLoader.tsx+++ b/packages/studio/src/components/InitialCompositionLoader.tsx@@ -19,7 +19,9 @@ import {useStaticFiles} from './use-static-files';export const useSelectComposition = () => {const {setCompositionFoldersExpanded} = useContext(FolderContext);const {setCanvasContent} = useContext(Internals.CompositionSetters);- const setFrame = Internals.useTimelineSetFrame();+ // A supplied frame is explicit navigation (for example, opening a nested+ // composition at its sequence's frame). Plain selection does not seek.+ const setFrame = Internals.Timeline.useTimelineSeekFrame();return useCallback((diff --git a/packages/studio/src/components/InspectorPanel/KeyframeEasingNavigator.tsx b/packages/studio/src/components/InspectorPanel/KeyframeEasingNavigator.tsxindex 79547b90e6..ef7bf5cfd0 100644--- a/packages/studio/src/components/InspectorPanel/KeyframeEasingNavigator.tsx+++ b/packages/studio/src/components/InspectorPanel/KeyframeEasingNavigator.tsx@@ -181,7 +181,7 @@ export const KeyframeEasingNavigator: React.FC<{readonly nodePathInfo: SequenceNodePathInfo;}> = ({currentSelection, includeEasings, keyframes, nodePathInfo}) => {const {isSelected, selectItems} = useTimelineSelection();- const setFrame = Internals.useTimelineSetFrame();+ const setFrame = Internals.Timeline.useTimelineSeekFrame();const videoConfig = useVideoConfig();const items = useMemo(() =>diff --git a/packages/studio/src/components/Timeline/TimelineDragHandler.tsx b/packages/studio/src/components/Timeline/TimelineDragHandler.tsxindex dd7c3efeda..21e79626ae 100644--- a/packages/studio/src/components/Timeline/TimelineDragHandler.tsx+++ b/packages/studio/src/components/Timeline/TimelineDragHandler.tsx@@ -135,7 +135,7 @@ const TimelineDragHandlerInner: React.FC = () => {shouldApplyCssTransforms: true,});const {isHighestContext} = useZIndex();- const setFrame = Internals.useTimelineSetFrame();+ const setFrame = Internals.Timeline.useTimelineSeekFrame();const width = getTimelineContentWidth();const left = size?.left ?? 0;diff --git a/packages/studio/src/components/Timeline/TimelineKeyframeControls.tsx b/packages/studio/src/components/Timeline/TimelineKeyframeControls.tsxindex 91543ce990..6f0db9a675 100644--- a/packages/studio/src/components/Timeline/TimelineKeyframeControls.tsx+++ b/packages/studio/src/components/Timeline/TimelineKeyframeControls.tsx@@ -550,7 +550,7 @@ export const TimelineKeyframeControls: React.FC<{}) => {const videoConfig = useVideoConfig();const timelinePosition = Internals.Timeline.useTimelinePosition();- const setFrame = Internals.useTimelineSetFrame();+ const setFrame = Internals.Timeline.useTimelineSeekFrame();const {setPropStatuses} = useContext(Internals.VisualModeSettersContext);const {propStatuses} = useContext(Internals.VisualModePropStatusesContext);const {getDragOverrides, getEffectDragOverrides} = useContext(