Skip to content

🎥 Make videos programmatically with React

`@remotion/media`: Reduce decoder recreation during forward playback (#11544)

Igor Samokhovetsb95cafffe9c4
diff --git a/packages/core/src/TimelineContext.tsx b/packages/core/src/TimelineContext.tsx
index dcb4e83d55..f385f111ab 100644
--- a/packages/core/src/TimelineContext.tsx
+++ b/packages/core/src/TimelineContext.tsx
@@ -13,6 +13,8 @@ import {
type PlayableMediaTag,
} from './timeline-position-state';
import {useDelayRender} from './use-delay-render';
+import {useRemotionEnvironment} from './use-remotion-environment';
+import {useTimelineSeek, type TimelineSeek} from './use-timeline-seek';
export type TimelineContextValue = {
frame: Record<string, number>;
@@ -35,6 +37,8 @@ export type BufferingState = Readonly<{
}>;
export type SetTimelineContextValue = {
+ // Null in rendering and thumbnails, which do not publish preview seek intent.
+ seek: TimelineSeek | null;
setFrame: (u: React.SetStateAction<Record<string, number>>) => void;
setPlaying: (u: React.SetStateAction<boolean>) => void;
setBuffering: (buffering: boolean) => void;
@@ -53,6 +57,7 @@ const missingSetTimelineContext = (): never => {
};
export const SetTimelineContext = createContext<SetTimelineContextValue>({
+ seek: null,
setFrame: missingSetTimelineContext,
setPlaying: missingSetTimelineContext,
setBuffering: missingSetTimelineContext,
@@ -92,6 +97,9 @@ export const TimelineContextProvider: React.FC<{
getInitialFrameState(),
);
+ const timelineSeek = useTimelineSeek(setFrame);
+ const {isStudio} = useRemotionEnvironment();
+ const seek = isStudio ? timelineSeek : null;
const frame = frameState ?? _frame;
const frameRef = useRef(frame);
frameRef.current = frame;
@@ -161,6 +169,7 @@ export const TimelineContextProvider: React.FC<{
const setTimelineContextValue = useMemo((): SetTimelineContextValue => {
return {
setFrame,
+ seek,
setPlaying: (updater) => {
const current = playingStore.store.getSnapshot().playing;
const next = typeof updater === 'function' ? updater(current) : updater;
@@ -180,7 +189,7 @@ export const TimelineContextProvider: React.FC<{
frameRef,
audioAndVideoTags,
};
- }, [bufferingStore, playingStore, readIsBuffering, readIsPlaying]);
+ }, [bufferingStore, playingStore, readIsBuffering, readIsPlaying, seek]);
return (
<AbsoluteTimeContext.Provider value={timelineContextValue}>
diff --git a/packages/core/src/internals.ts b/packages/core/src/internals.ts
index 1e9a23555b..3c0085d66b 100644
--- a/packages/core/src/internals.ts
+++ b/packages/core/src/internals.ts
@@ -248,6 +248,7 @@ import {
type PropStatuses,
} from './use-schema.js';
import {useSyncExternalStore} from './use-sync-external-store.js';
+import {useTimelineSeek} from './use-timeline-seek.js';
import {useUnsafeVideoConfig} from './use-unsafe-video-config.js';
import {useVideo} from './use-video.js';
import {validateMediaProps} from './validate-media-props.js';
@@ -438,6 +439,7 @@ export const Internals = {
useAudioEnabled,
useBuffering,
TimelinePosition,
+ useTimelineSeek,
DelayRenderContextType,
TimelineContext,
usePlaying,
diff --git a/packages/core/src/test/wrap-sequence-context.tsx b/packages/core/src/test/wrap-sequence-context.tsx
index 2ff4e3c0a8..4eb50dc673 100644
--- a/packages/core/src/test/wrap-sequence-context.tsx
+++ b/packages/core/src/test/wrap-sequence-context.tsx
@@ -131,6 +131,7 @@ export const WrapSequenceContext: React.FC<{
const setTimelineContext = useMemo<SetTimelineContextValue>(
() => ({
setFrame: () => undefined,
+ seek: null,
setPlaying: () => undefined,
setBuffering: (buffering) => {
if (bufferingStore.store.getSnapshot().buffering !== buffering) {
diff --git a/packages/core/src/timeline-position-state.ts b/packages/core/src/timeline-position-state.ts
index 8a4f1ad567..d4efebad71 100644
--- a/packages/core/src/timeline-position-state.ts
+++ b/packages/core/src/timeline-position-state.ts
@@ -121,5 +121,11 @@ export const useTimelineSetFrame = (): ((
return setFrame;
};
+// Explicit navigation only. Playback and frame persistence must use setFrame.
+export const useTimelineSeekFrame = () => {
+ const {seek, setFrame} = useContext(SetTimelineContext);
+ return seek?.setFrame ?? setFrame;
+};
+
export {usePlaying} from './use-playing.js';
export {useBuffering} from './use-buffering.js';
diff --git a/packages/core/src/use-timeline-seek.ts b/packages/core/src/use-timeline-seek.ts
new file mode 100644
index 0000000000..fa7f572e91
--- /dev/null
+++ b/packages/core/src/use-timeline-seek.ts
@@ -0,0 +1,26 @@
+import type {RefObject, SetStateAction} from 'react';
+import {useMemo, useRef} from 'react';
+
+export type TimelineSeek = {
+ readonly revision: RefObject<number>;
+ readonly setFrame: (frame: SetStateAction<Record<string, number>>) => void;
+};
+
+// One instance per timeline, shared by Studio and Player. Keep the revision
+// outside the React updater: updaters may be deferred or run more than once.
+export const useTimelineSeek = (
+ setFrame: (frame: SetStateAction<Record<string, number>>) => void,
+): TimelineSeek => {
+ const revision = useRef(0);
+ return useMemo(
+ () => ({
+ revision,
+ setFrame: (frame) => {
+ // Even a same-frame seek is a boundary for the next media update.
+ revision.current++;
+ setFrame(frame);
+ },
+ }),
+ [setFrame],
+ );
+};
diff --git a/packages/media/src/media-player.ts b/packages/media/src/media-player.ts
index 3cbb20f8fa..1b3084899c 100644
--- a/packages/media/src/media-player.ts
+++ b/packages/media/src/media-player.ts
@@ -40,6 +40,14 @@ export type MediaPlayerInitResult =
| {type: 'no-tracks'}
| {type: 'disposed'};
+export type MediaSeekIntent =
+ | {readonly revision: number; readonly playing: boolean}
+ // An internal resync after media configuration changes, not a timeline seek.
+ // Invalidate the satisfied revision so the next video request cannot be
+ // mistaken for continuous playback. This does not always restart the iterator.
+ | 'discontinuity'
+ | null;
+
export class MediaPlayer {
private tagType: 'audio' | 'video';
private canvas: HTMLCanvasElement | OffscreenCanvas | null;
@@ -61,6 +69,8 @@ export class MediaPlayer {
videoIteratorManager: VideoIteratorManager | null = null;
private playing = false;
+ private lastSeekIntent: MediaSeekIntent = null;
+ private satisfiedSeekRevision: number | null = null;
private loop = false;
private fps: number;
@@ -488,6 +498,7 @@ export class MediaPlayer {
private seekToWithQueue = async (
newTime: number,
unloopedNewTime: number,
+ intent: MediaSeekIntent,
) => {
const nonce = this.nonceManager.createAsyncOperation();
await this.seekPromiseChain;
@@ -496,11 +507,12 @@ export class MediaPlayer {
newTime,
unloopedNewTime,
nonce,
+ intent,
);
await this.seekPromiseChain;
};
- public async seekTo(time: number): Promise<void> {
+ public async seekTo(time: number, intent: MediaSeekIntent): Promise<void> {
if (this.terminalError || this.isDisposalError()) {
return;
}
@@ -511,13 +523,20 @@ export class MediaPlayer {
throw new Error(`should have asserted that the time is not null`);
}
- await this.seekToWithQueue(newTime, time);
+ this.lastSeekIntent = intent;
+ if (intent === 'discontinuity') {
+ // A mapping change may supersede a queued timeline request.
+ this.satisfiedSeekRevision = null;
+ }
+
+ await this.seekToWithQueue(newTime, time, intent);
}
private async seekToDoNotCallDirectly(
newTime: number,
unloopedNewTime: number,
nonce: Nonce,
+ intent: MediaSeekIntent,
): Promise<void> {
if (nonce.isStale() || this.terminalError || this.isDisposalError()) {
return;
@@ -531,6 +550,12 @@ export class MediaPlayer {
fps: this.fps,
playbackRate: this.playbackRate,
isPlaying: this.playing,
+ continuousPlayback:
+ intent === null
+ ? null
+ : intent !== 'discontinuity' &&
+ intent.playing &&
+ intent.revision === this.satisfiedSeekRevision,
}),
this.audioIteratorManager?.seek({
newTime,
@@ -551,6 +576,10 @@ export class MediaPlayer {
this.sharedAudioContext!.audioContext.currentTime,
}),
]);
+ // Superseded work must not consume an explicit seek boundary.
+ if (!nonce.isStale() && intent !== null && intent !== 'discontinuity') {
+ this.satisfiedSeekRevision = intent.revision;
+ }
} catch (error) {
if (this.isDisposalError()) {
return;
@@ -619,7 +648,7 @@ export class MediaPlayer {
if (this.trimBefore !== trimBefore) {
this.trimBefore = trimBefore;
this.audioIteratorManager?.destroyIterator();
- await this.seekTo(unloopedTimeInSeconds);
+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');
}
}
@@ -630,7 +659,7 @@ export class MediaPlayer {
if (this.trimAfter !== trimAfter) {
this.trimAfter = trimAfter;
this.audioIteratorManager?.destroyIterator();
- await this.seekTo(unloopedTimeInSeconds);
+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');
}
}
@@ -647,7 +676,7 @@ export class MediaPlayer {
if (previousRate !== rate) {
this.playbackRate = rate;
this.audioIteratorManager?.destroyIterator();
- await this.seekTo(unloopedTimeInSeconds);
+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');
}
}
@@ -663,7 +692,7 @@ export class MediaPlayer {
this.audioIteratorManager.setToneFrequency(toneFrequency);
this.audioIteratorManager.destroyIterator();
- await this.seekTo(unloopedTimeInSeconds);
+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');
}
}
@@ -675,7 +704,7 @@ export class MediaPlayer {
if (previousRate !== rate) {
this.globalPlaybackRate = rate;
this.audioIteratorManager?.destroyIterator();
- await this.seekTo(unloopedTimeInSeconds);
+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');
}
}
@@ -687,7 +716,7 @@ export class MediaPlayer {
if (previousFps !== fps) {
this.fps = fps;
this.audioIteratorManager?.destroyIterator();
- await this.seekTo(unloopedTimeInSeconds);
+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');
}
}
@@ -709,7 +738,7 @@ export class MediaPlayer {
if (previousLoop !== loop) {
this.loop = loop;
this.audioIteratorManager?.destroyIterator();
- await this.seekTo(unloopedTimeInSeconds);
+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');
}
}
@@ -721,7 +750,7 @@ export class MediaPlayer {
if (previousOffset !== offset) {
this.sequenceOffset = offset;
this.audioIteratorManager?.destroyIterator();
- await this.seekTo(unloopedTimeInSeconds);
+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');
}
}
@@ -732,7 +761,7 @@ export class MediaPlayer {
const previousDuration = this.sequenceDurationInFrames;
if (previousDuration !== sequenceDurationInFrames) {
this.sequenceDurationInFrames = sequenceDurationInFrames;
- await this.seekTo(unloopedTimeInSeconds);
+ await this.seekTo(unloopedTimeInSeconds, 'discontinuity');
}
}
@@ -904,6 +933,15 @@ export class MediaPlayer {
this.audioIteratorManager.destroyIterator();
// An anchor can change while paused, when no frame update will restart
// scheduling. Refill the queue against the new anchor immediately.
- await this.seekTo(unloopedTimeInSeconds);
+ // Re-anchoring is not navigation. Preserve the captured revision so an
+ // outstanding explicit seek still takes the restart path, but don't let
+ // an audio-clock pause turn ordinary video catch-up into a scrub.
+ const intent = this.lastSeekIntent;
+ await this.seekTo(
+ unloopedTimeInSeconds,
+ intent !== null && intent !== 'discontinuity'
+ ? {...intent, playing: true}
+ : intent,
+ );
};
}
diff --git a/packages/media/src/test/media-player.test.ts b/packages/media/src/test/media-player.test.ts
index a084ef3da4..1060f54d85 100644
--- a/packages/media/src/test/media-player.test.ts
+++ b/packages/media/src/test/media-player.test.ts
@@ -79,6 +79,32 @@ test('audio-only file should initialize without `audioStreamIndex` (regression f
await player.dispose();
});
+test('queued playback cannot consume a superseded seek boundary', async () => {
+ const player = makeAudioPlayer(null, null, 'video');
+ try {
+ expect((await player.initialize(0, true, 1)).type).toBe('success');
+ const manager = player.videoIteratorManager!;
+ await player.seekTo(0, {revision: 0, playing: true});
+ await player.seekTo(2, {revision: 0, playing: true});
+ expect(manager.getVideoIteratorsCreated()).toBe(1);
+ expect(manager.getFramesRendered()).toBeGreaterThan(1);
+
+ // Both requests are enqueued before either executes. The first request
+ // goes stale, but the second still needs to establish revision 1.
+ await Promise.all([
+ player.seekTo(8, {revision: 1, playing: true}),
+ player.seekTo(8.1, {revision: 1, playing: true}),
+ ]);
+ expect(manager.getVideoIteratorsCreated()).toBe(2);
+ await player.seekTo(8.5, {revision: 1, playing: true});
+ expect(manager.getVideoIteratorsCreated()).toBe(2);
+ await player.seekTo(0, {revision: 1, playing: true});
+ expect(manager.getVideoIteratorsCreated()).toBe(3);
+ } finally {
+ await player.dispose();
+ }
+});
+
test('uses the initial volume before audio playback is ready', async () => {
const sharedAudioContext = makeSharedAudioContext();
const mediaGainNodes: GainNode[] = [];
@@ -111,7 +137,7 @@ test('uses the initial volume before audio playback is ready', async () => {
}
});
-test('dispose should immediately unblock playback delays', async () => {
+test('dispose should immediately unblock a pending playback catch-up', async () => {
let activeBlocks = 0;
let delayPlaybackCalled: () => void = () => {};
@@ -173,12 +199,13 @@ test('dispose should immediately unblock playback delays', async () => {
});
await player.initialize(0, false, 1);
+ await player.seekTo(0, {revision: 0, playing: true});
const seekDelayPromise = new Promise<void>((resolve) => {
delayPlaybackCalled = resolve;
});
- const seekPromise = player.seekTo(9);
+ const seekPromise = player.seekTo(9, {revision: 0, playing: true});
await seekDelayPromise;
@@ -201,9 +228,9 @@ test.each(['video'] as const)(
player[`${tagType}IteratorManager`] = {seek, destroy: vi.fn()} as never;
player.play();
- await player.seekTo(1);
+ await player.seekTo(1, null);
player.play();
- await player.seekTo(2);
+ await player.seekTo(2, null);
expect(onError).toHaveBeenCalledOnce();
expect(onError).toHaveBeenCalledWith(error);
@@ -252,13 +279,13 @@ test.each(['audio', 'video'] as const)(
},
);
try {
- await player.seekTo(1);
+ await player.seekTo(1, null);
await vi.waitFor(() => expect(rejectRead).toBeDefined());
rejectRead(error);
await vi.waitFor(() => expect(onError).toHaveBeenCalledWith(error));
player.play();
- await player.seekTo(2);
+ await player.seekTo(2, null);
expect(onError).toHaveBeenCalledOnce();
expect(unblocks).toHaveBeenCalled();
expect(buffers).toHaveBeenCalledOnce();
@@ -283,14 +310,14 @@ test('ignores pending seek failures after disposal and skips new seeks', async (
);
player.videoIteratorManager = {seek} as never;
- const pending = player.seekTo(1);
+ const pending = player.seekTo(1, null);
await vi.waitFor(() => expect(seek).toHaveBeenCalledOnce());
player.videoIteratorManager = null;
await player.dispose();
rejectSeek(new TypeError('Failed to fetch'));
await pending;
player.videoIteratorManager = {seek} as never;
- await player.seekTo(2);
+ await player.seekTo(2, null);
expect(onError).not.toHaveBeenCalled();
expect(seek).toHaveBeenCalledOnce();
diff --git a/packages/media/src/test/player-seek-intent.test.tsx b/packages/media/src/test/player-seek-intent.test.tsx
new file mode 100644
index 0000000000..507393f8dc
--- /dev/null
+++ b/packages/media/src/test/player-seek-intent.test.tsx
@@ -0,0 +1,110 @@
+import {Player, type PlayerRef} from '@remotion/player';
+import React, {useContext} from 'react';
+import {createRoot} from 'react-dom/client';
+import {Internals, useBufferState} from 'remotion';
+import {expect, test, vi} from 'vitest';
+import {Video} from '../video/video';
+import {findLargestCanvas, waitFor} from './player-frame-accuracy-utils';
+
+test('Player distinguishes playback catch-up while buffering from explicit seeks', async () => {
+ let timeline: React.ContextType<typeof Internals.SetTimelineContext> | null =
+ null;
+ let bufferState: ReturnType<typeof useBufferState> | null = null;
+ let draws = 0;
+ const errors: Error[] = [];
+ const Composition = () => {
+ timeline = useContext(Internals.SetTimelineContext);
+ bufferState = useBufferState();
+ return (
+ <Video
+ src="/bigbuckbunny.mp4"
+ muted
+ onVideoFrame={() => {
+ draws++;
+ }}
+ onError={(error) => {
+ errors.push(error);
+ }}
+ />
+ );
+ };
+
+ const container = document.createElement('div');
+ document.body.appendChild(container);
+ const root = createRoot(container);
+ const ref = React.createRef<PlayerRef>();
+ const configurations = vi.spyOn(VideoDecoder.prototype, 'configure');
+ let foreignBuffer: {unblock: () => void} | null = null;
+ root.render(
+ <Player
+ ref={ref}
+ component={Composition}
+ compositionWidth={320}
+ compositionHeight={180}
+ durationInFrames={300}
+ fps={30}
+ numberOfSharedAudioTags={0}
+ />,
+ );
+ try {
+ await waitFor(() => draws > 0 && !timeline!.isBuffering());
+ // Establish the correct output through a public paused seek.
+ let before = draws;
+ ref.current!.seekTo(60);
+ await waitFor(() => draws > before && !timeline!.isBuffering());
+ const target = findLargestCanvas(container).toDataURL();
+ before = draws;
+ ref.current!.seekTo(0);
+ await waitFor(() => draws > before && !timeline!.isBuffering());
+ expect(findLargestCanvas(container).toDataURL()).not.toBe(target);
+
+ // Another layer blocks the clock. A missed playback update still must
+ // consume the existing iterator, not guess intent from the time gap.
+ foreignBuffer = bufferState!.delayPlayback();
+ ref.current!.play();
+ await waitFor(() => ref.current!.isPlaying() && timeline!.isBuffering());
+ const configuredBeforeCatchup = configurations.mock.calls.length;
+ const revision = timeline!.seek!.revision.current;
+ before = draws;
+ timeline!.setFrame((frames) => ({...frames, [Object.keys(frames)[0]]: 60}));
+ await waitFor(
+ () =>
+ draws > before && findLargestCanvas(container).toDataURL() === target,
+ );
+ expect(ref.current!.getCurrentFrame()).toBe(60);
+ expect(timeline!.seek!.revision.current).toBe(revision);
+ expect(configurations.mock.calls.length).toBe(configuredBeforeCatchup);
+
+ // The same playing/buffering state, but now a deliberate distant seek.
+ before = draws;
+ ref.current!.seekTo(240);
+ await waitFor(
+ () => draws > before && ref.current!.getCurrentFrame() === 240,
+ );
+ expect(configurations.mock.calls.length).toBeGreaterThan(
+ configuredBeforeCatchup,
+ );
+ expect(findLargestCanvas(container).toDataURL()).not.toBe(target);
+
+ before = draws;
+ ref.current!.seekTo(120);
+ ref.current!.seekTo(15);
+ ref.current!.seekTo(60);
+ ref.current!.pause();
+ foreignBuffer.unblock();
+ foreignBuffer = null;
+ await waitFor(
+ () =>
+ draws > before &&
+ !timeline!.isBuffering() &&
+ findLargestCanvas(container).toDataURL() === target,
+ );
+ expect(ref.current!.getCurrentFrame()).toBe(60);
+ expect(errors).toEqual([]);
+ } finally {
+ foreignBuffer?.unblock();
+ root.unmount();
+ container.remove();
+ configurations.mockRestore();
+ }
+}, 30000);
diff --git a/packages/media/src/test/variable-fps-issue-8303.test.ts b/packages/media/src/test/variable-fps-issue-8303.test.ts
index cdf67ea3e7..e91a47a9c3 100644
--- a/packages/media/src/test/variable-fps-issue-8303.test.ts
+++ b/packages/media/src/test/variable-fps-issue-8303.test.ts
@@ -38,6 +38,7 @@ test('variable FPS video can be sought forward one frame at a time', async () =>
for (let frame = 0; frame <= 210; frame++) {
const time = frame / FPS;
const result = await iterator.tryToSatisfySeek(time, {
+ onWait: () => undefined,
pendingFrameBehavior: 'wait',
shouldContinue: () => true,
});
diff --git a/packages/media/src/test/video-iterator-manager.test.ts b/packages/media/src/test/video-iterator-manager.test.ts
index b1ba294da6..a333c792c1 100644
--- a/packages/media/src/test/video-iterator-manager.test.ts
+++ b/packages/media/src/test/video-iterator-manager.test.ts
@@ -115,6 +115,7 @@ test('plays at a high playback rate without restarting the iterator', async () =
fps: 30,
playbackRate: 3.75,
isPlaying: true,
+ continuousPlayback: null,
});
}
@@ -147,6 +148,7 @@ test('paused forward scrubs do not wait for pending frames', async () => {
};
await manager.seek({
+ continuousPlayback: null,
newTime: 0.1,
nonce: nonceManager.createAsyncOperation(),
fps: 30,
@@ -204,6 +206,7 @@ test('seek should not cause overlapping block/unblock cycles', async () => {
// Perform seeks that will trigger 'not-satisfied' (jumping around)
// These should NOT cause overlapping blocks
await manager.seek({
+ continuousPlayback: null,
newTime: 5,
nonce: nonceManager.createAsyncOperation(),
fps: 30,
@@ -211,6 +214,7 @@ test('seek should not cause overlapping block/unblock cycles', async () => {
isPlaying: false,
});
await manager.seek({
+ continuousPlayback: null,
newTime: 0,
nonce: nonceManager.createAsyncOperation(),
fps: 30,
@@ -218,6 +222,7 @@ test('seek should not cause overlapping block/unblock cycles', async () => {
isPlaying: false,
});
await manager.seek({
+ continuousPlayback: null,
newTime: 8,
nonce: nonceManager.createAsyncOperation(),
fps: 30,
@@ -277,6 +282,7 @@ test('rapid sequential seeks should not cause overlapping blocks', async () => {
// Alternate between distant positions to force iterator recreation
const time = i % 2 === 0 ? i * 0.5 : 9 - i * 0.5;
await manager.seek({
+ continuousPlayback: null,
newTime: time,
nonce: nonceManager.createAsyncOperation(),
fps: 30,
diff --git a/packages/media/src/test/video-preview-iterator.test.ts b/packages/media/src/test/video-preview-iterator.test.ts
index 37392e9b8b..bfe681dce0 100644
--- a/packages/media/src/test/video-preview-iterator.test.ts
+++ b/packages/media/src/test/video-preview-iterator.test.ts
@@ -44,6 +44,7 @@ test('preview iterator uses next frame timestamp instead of reported duration',
try {
const result = await iterator.tryToSatisfySeek(1.5, {
+ onWait: () => undefined,
pendingFrameBehavior: 'wait',
shouldContinue: () => true,
});
@@ -95,6 +96,7 @@ test('preview iterator does not trust reported duration without a next timestamp
try {
const result = await iterator.tryToSatisfySeek(1.5, {
+ onWait: () => undefined,
pendingFrameBehavior: 'restart-iterator',
shouldContinue: () => true,
});
@@ -143,6 +145,7 @@ test('preview iterator waits through high-FPS frames for a sequential timeline s
try {
const result = await iterator.tryToSatisfySeek(1 / timelineFps, {
+ onWait: () => undefined,
pendingFrameBehavior: 'wait',
shouldContinue: () => true,
});
@@ -192,6 +195,7 @@ test('preview iterator supports frames longer than one timeline step', async ()
try {
const result = await iterator.tryToSatisfySeek(0.075, {
+ onWait: () => undefined,
pendingFrameBehavior: 'wait',
shouldContinue: () => true,
});
@@ -245,6 +249,7 @@ test('preview iterator stops decoding when a seek is superseded', async () => {
try {
const result = await iterator.tryToSatisfySeek(1, {
+ onWait: () => undefined,
pendingFrameBehavior: 'wait',
shouldContinue: () => shouldContinue,
});
diff --git a/packages/media/src/test/video-with-no-frames-in-beginning.test.ts b/packages/media/src/test/video-with-no-frames-in-beginning.test.ts
index cadb9c495e..fad14b6788 100644
--- a/packages/media/src/test/video-with-no-frames-in-beginning.test.ts
+++ b/packages/media/src/test/video-with-no-frames-in-beginning.test.ts
@@ -53,6 +53,7 @@ test('in preview, should properly buffer and draw frames', async (t) => {
await manager.startVideoIterator(0, nonceManager.createAsyncOperation());
await manager.seek({
newTime: 0.03,
+ continuousPlayback: null,
nonce: nonceManager.createAsyncOperation(),
fps: 30,
playbackRate: 1,
@@ -60,6 +61,7 @@ test('in preview, should properly buffer and draw frames', async (t) => {
});
await manager.seek({
newTime: 1,
+ continuousPlayback: null,
nonce: nonceManager.createAsyncOperation(),
fps: 30,
playbackRate: 1,
@@ -67,6 +69,7 @@ test('in preview, should properly buffer and draw frames', async (t) => {
});
await manager.seek({
newTime: 2,
+ continuousPlayback: null,
nonce: nonceManager.createAsyncOperation(),
fps: 30,
playbackRate: 1,
@@ -78,6 +81,7 @@ test('in preview, should properly buffer and draw frames', async (t) => {
await manager.seek({
newTime: 4.5,
+ continuousPlayback: null,
nonce: nonceManager.createAsyncOperation(),
fps: 30,
playbackRate: 1,
diff --git a/packages/media/src/use-common-effects.ts b/packages/media/src/use-common-effects.ts
index d74cc56bc6..b581d50258 100644
--- a/packages/media/src/use-common-effects.ts
+++ b/packages/media/src/use-common-effects.ts
@@ -1,5 +1,5 @@
import type React from 'react';
-import {useContext, useLayoutEffect} from 'react';
+import {useContext, useLayoutEffect, useRef} from 'react';
import type {LogLevel} from 'remotion';
import {Internals} from 'remotion';
import type {MediaPlayer} from './media-player';
@@ -212,16 +212,46 @@ export const useCommonEffects = ({
mediaPlayer.setSequenceOffset(sequenceOffset, currentTimeRef.current);
}, [sequenceOffset, mediaPlayerReady, mediaPlayerRef, currentTimeRef]);
+ const {seek} = useContext(Internals.SetTimelineContext);
+ const revision = seek?.revision.current ?? null;
+ const lastRequest = useRef<{
+ player: MediaPlayer;
+ time: number;
+ revision: number | null;
+ } | null>(null);
+
useLayoutEffect(() => {
const mediaPlayer = mediaPlayerRef.current;
if (!mediaPlayer || !mediaPlayerReady) return;
- mediaPlayer.seekTo(currentTime).catch(() => {
- // Might be disposed
- });
+ // Pausing or buffering at the same target must not supersede an in-flight
+ // playback request with a scrub. A same-frame explicit seek still counts.
+ const previous = lastRequest.current;
+ if (
+ previous?.player === mediaPlayer &&
+ previous.time === currentTime &&
+ previous.revision === revision
+ ) {
+ return;
+ }
+
+ lastRequest.current = {player: mediaPlayer, time: currentTime, revision};
+ mediaPlayer
+ .seekTo(currentTime, revision === null ? null : {revision, playing})
+ .catch(() => {
+ // Might be disposed
+ });
Internals.Log.trace(
{logLevel, tag: '@remotion/media'},
`[${label}] Updating target time to ${currentTime.toFixed(3)}s`,
);
- }, [currentTime, logLevel, mediaPlayerReady, label, mediaPlayerRef]);
+ }, [
+ currentTime,
+ logLevel,
+ mediaPlayerReady,
+ label,
+ mediaPlayerRef,
+ playing,
+ revision,
+ ]);
};
diff --git a/packages/media/src/video-iterator-manager.ts b/packages/media/src/video-iterator-manager.ts
index dd6a681bb0..d84c378407 100644
--- a/packages/media/src/video-iterator-manager.ts
+++ b/packages/media/src/video-iterator-manager.ts
@@ -34,6 +34,7 @@ export const isSequentialMediaTimeAdvance = ({
return false;
}
+ // Legacy fallback for hosts without explicit preview seek intent.
const maximumSequentialAdvance = Math.abs(playbackRate) / fps;
return (
roundTo4Digits(newTime - previousTime) <=
@@ -214,12 +215,14 @@ export const videoIteratorManager = async ({
fps,
playbackRate,
isPlaying,
+ continuousPlayback,
}: {
newTime: number;
nonce: Nonce;
fps: number;
playbackRate: number;
isPlaying: boolean;
+ continuousPlayback: boolean | null;
}) => {
if (!videoFrameIterator) {
return;
@@ -233,7 +236,6 @@ export const videoIteratorManager = async ({
}
const previousTime = currentSeek;
- currentSeek = newTime;
if (getIsLooping()) {
// If less than 1 second from the end away, we pre-warm a new iterator
@@ -246,36 +248,52 @@ export const videoIteratorManager = async ({
const pendingFrameBehavior =
previousTime !== null &&
- isSequentialMediaTimeAdvance({
- previousTime,
- newTime,
- fps,
- playbackRate,
- isPlaying,
- })
+ newTime >= previousTime &&
+ (continuousPlayback ??
+ isSequentialMediaTimeAdvance({
+ previousTime,
+ newTime,
+ fps,
+ playbackRate,
+ isPlaying,
+ }))
? 'wait'
: 'restart-iterator';
- const videoSatisfyResult = await videoFrameIterator.tryToSatisfySeek(
- newTime,
- {
+ const iterator = videoFrameIterator;
+ const pending: {handle: DelayPlaybackIfNotPremounting | null} = {
+ handle: null,
+ };
+ try {
+ const result = await iterator.tryToSatisfySeek(newTime, {
pendingFrameBehavior,
- shouldContinue: () => !nonce.isStale(),
- },
- );
+ onWait: () => {
+ pending.handle ??= delayPlaybackHandleIfNotPremounting();
+ currentDelayHandle = pending.handle;
+ },
+ shouldContinue: () => !nonce.isStale() && !iterator.isDestroyed(),
+ });
+
+ if (nonce.isStale() || iterator.isDestroyed()) {
+ return;
+ }
- // Doing this before the staleness check, because
- // frame might be better than what we currently have
- // TODO: check if this is actually true
- if (videoSatisfyResult.type === 'satisfied') {
- await drawFrame(videoSatisfyResult.frame);
- return;
- }
+ if (result.type === 'satisfied') {
+ await drawFrame(result.frame);
+ currentSeek = newTime;
+ return;
+ }
- if (nonce.isStale()) {
- return;
+ // The replacement iterator owns its own buffering handle.
+ pending.handle?.unblock();
+ pending.handle = null;
+ currentDelayHandle = null;
+ await startVideoIterator(newTime, nonce);
+ } finally {
+ pending.handle?.unblock();
+ if (currentDelayHandle === pending.handle) {
+ currentDelayHandle = null;
+ }
}
-
- await startVideoIterator(newTime, nonce);
};
return {
diff --git a/packages/media/src/video/video-preview-iterator.ts b/packages/media/src/video/video-preview-iterator.ts
index 0ac96840ad..e736c2f824 100644
--- a/packages/media/src/video/video-preview-iterator.ts
+++ b/packages/media/src/video/video-preview-iterator.ts
@@ -21,6 +21,11 @@ export const createVideoIterator = async (
let peekedFrame: WrappedCanvas | null = null;
const setLastReturnedFrame = (frame: WrappedCanvas) => {
+ if (destroyed) {
+ releaseStableFrame(frame);
+ return;
+ }
+
if (lastReturnedFrame !== frame) {
releaseStableFrame(lastReturnedFrame);
}
@@ -29,6 +34,11 @@ export const createVideoIterator = async (
};
const setPeekedFrame = (frame: WrappedCanvas | null) => {
+ if (destroyed) {
+ releaseStableFrame(frame);
+ return null;
+ }
+
peekedFrame = frame;
if (peekedFrame === null) {
iteratorEnded = true;
@@ -63,9 +73,11 @@ export const createVideoIterator = async (
const getFrameEndTimestamp = async ({
pendingFrameBehavior,
+ onWait,
shouldContinue,
}: {
pendingFrameBehavior: 'wait' | 'restart-iterator';
+ onWait: () => void;
shouldContinue: () => boolean;
}) => {
const peeked = peekIfReady();
@@ -84,6 +96,7 @@ export const createVideoIterator = async (
return {type: 'cancelled' as const};
}
+ onWait();
const awaitedPeeked = setPeekedFrame(await peeked.wait());
if (!shouldContinue()) {
return {type: 'cancelled' as const};
@@ -153,6 +166,7 @@ export const createVideoIterator = async (
time: number,
options: {
pendingFrameBehavior: 'wait' | 'restart-iterator';
+ onWait: () => void;
shouldContinue: () => boolean;
},
): Promise<
@@ -196,6 +210,7 @@ export const createVideoIterator = async (
}
const frameEndTimestamp = await getFrameEndTimestamp({
+ onWait: options.onWait,
pendingFrameBehavior: options.pendingFrameBehavior,
shouldContinue: options.shouldContinue,
});
@@ -246,13 +261,26 @@ export const createVideoIterator = async (
};
}
- const frame = getNextOrNullIfNotAvailable();
+ let frame = getNextOrNullIfNotAvailable();
if (frame.type === 'need-to-wait-for-it') {
- return {
- type: 'not-satisfied' as const,
- reason: 'iterator did not have frame ready',
- };
+ if (options.pendingFrameBehavior === 'restart-iterator') {
+ return {
+ type: 'not-satisfied' as const,
+ reason: 'iterator did not have frame ready',
+ };
+ }
+
+ options.onWait();
+ const awaitedFrame = await frame.waitPromise();
+ if (!options.shouldContinue()) {
+ return {
+ type: 'not-satisfied',
+ reason: 'seek was superseded',
+ };
+ }
+
+ frame = {type: 'got-frame-or-end', frame: awaitedFrame};
}
if (frame.type === 'got-frame-or-end') {
@@ -273,6 +301,7 @@ export const createVideoIterator = async (
const frameTimestamp = roundTo4Digits(frame.frame.timestamp);
const frameEndTimestamp = await getFrameEndTimestamp({
+ onWait: options.onWait,
pendingFrameBehavior: options.pendingFrameBehavior,
shouldContinue: options.shouldContinue,
});
diff --git a/packages/media/vitest.config.ts b/packages/media/vitest.config.ts
index 2fd8e5434f..6c2bc3531f 100644
--- a/packages/media/vitest.config.ts
+++ b/packages/media/vitest.config.ts
@@ -3,6 +3,18 @@ import {webdriverio} from '@vitest/browser-webdriverio';
import {defineConfig} from 'vitest/config';
export default defineConfig({
+ resolve: {dedupe: ['react', 'react-dom', 'remotion']},
+ // Avoid discovering another React runtime midway through browser tests.
+ optimizeDeps: {
+ include: [
+ 'react',
+ 'react/jsx-runtime',
+ 'react/jsx-dev-runtime',
+ 'react-dom/client',
+ 'remotion',
+ '@remotion/player',
+ ],
+ },
test: {
browser: {
provider: webdriverio(),
diff --git a/packages/player/src/Player.tsx b/packages/player/src/Player.tsx
index 4c70214194..68c9d1b13a 100644
--- a/packages/player/src/Player.tsx
+++ b/packages/player/src/Player.tsx
@@ -246,6 +246,7 @@ const PlayerFn = <
const [frame, setFrame] = useState<Record<string, number>>(() => ({
[PLAYER_COMP_ID]: initialFrame ?? 0,
}));
+ const seek = Internals.useTimelineSeek(setFrame);
const frameRef = useRef(frame);
frameRef.current = frame;
const rootRef = useRef<PlayerRef>(null);
@@ -446,6 +447,7 @@ const PlayerFn = <
const setTimelineContextValue = useMemo((): SetTimelineContextValue => {
return {
setFrame,
+ seek,
setPlaying: (updater) => {
const current = playingStore.store.getSnapshot().playing;
const next = typeof updater === 'function' ? updater(current) : updater;
@@ -467,6 +469,7 @@ const PlayerFn = <
};
}, [
bufferingStore,
+ seek,
setFrame,
frameRef,
playingStore,
diff --git a/packages/player/src/Thumbnail.tsx b/packages/player/src/Thumbnail.tsx
index cd18f99633..01b8e1ba48 100644
--- a/packages/player/src/Thumbnail.tsx
+++ b/packages/player/src/Thumbnail.tsx
@@ -116,6 +116,7 @@ const ThumbnailFn = <
const setTimelineContext: SetTimelineContextValue = useMemo(() => {
return {
setFrame: () => undefined,
+ seek: null,
setPlaying: () => undefined,
setBuffering: (buffering) => {
if (bufferingStore.store.getSnapshot().buffering !== buffering) {
diff --git a/packages/player/src/test/seek-intent.test.tsx b/packages/player/src/test/seek-intent.test.tsx
new file mode 100644
index 0000000000..8eed7fdea9
--- /dev/null
+++ b/packages/player/src/test/seek-intent.test.tsx
@@ -0,0 +1,73 @@
+import {afterEach, expect, test} from 'bun:test';
+import React, {createRef, useContext} from 'react';
+import {Internals, useCurrentFrame} from 'remotion';
+import type {PlayerRef} from '../player-methods.js';
+import {Player} from '../Player.js';
+import {act, cleanup, render} from './test-utils.js';
+
+afterEach(cleanup);
+
+test('Player seeks publish boundaries before frame updates and stay local to each Player', () => {
+ const players = [createRef<PlayerRef>(), createRef<PlayerRef>()];
+ const ids = ['first', 'second'];
+ const timelines: React.ContextType<typeof Internals.SetTimelineContext>[] =
+ [];
+ const Composition: React.FC<{readonly index: number}> = ({index}) => {
+ timelines[index] = useContext(Internals.SetTimelineContext);
+ return <output data-testid={`frame-${index}`}>{useCurrentFrame()}</output>;
+ };
+
+ const view = render(
+ <>
+ {players.map((ref, index) => (
+ <Player
+ key={ids[index]}
+ ref={ref}
+ component={Composition}
+ inputProps={{index}}
+ durationInFrames={300}
+ compositionWidth={320}
+ compositionHeight={180}
+ fps={30}
+ numberOfSharedAudioTags={0}
+ />
+ ))}
+ </>,
+ );
+ const {revision} = timelines[0].seek!;
+ const events: number[] = [];
+ players[0].current!.addEventListener('seeked', () => {
+ events.push(revision.current);
+ });
+
+ act(() => {
+ players[0].current!.seekTo(30);
+ expect(revision.current).toBe(1);
+ });
+ expect(view.getByTestId('frame-0').textContent).toBe('30');
+ act(() => players[0].current!.play());
+ expect(revision.current).toBe(1);
+ act(() => {
+ players[0].current!.seekTo(31);
+ players[0].current!.seekTo(180);
+ players[0].current!.seekTo(15);
+ });
+ expect(view.getByTestId('frame-0').textContent).toBe('15');
+ expect(revision.current).toBe(4);
+ act(() => players[0].current!.pause());
+ act(() => players[0].current!.seekTo(15));
+ expect(revision.current).toBe(5);
+ expect(events).toEqual([1, 2, 3, 4, 5]);
+ expect(view.getByTestId('frame-1').textContent).toBe('0');
+ expect(timelines[1].seek!.revision.current).toBe(0);
+
+ // The animation path updates frames without publishing navigation intent.
+ act(() =>
+ timelines[0].setFrame((frames) => ({
+ ...frames,
+ [Object.keys(frames)[0]]: 90,
+ })),
+ );
+ expect(view.getByTestId('frame-0').textContent).toBe('90');
+ expect(revision.current).toBe(5);
+});
diff --git a/packages/player/src/use-player-methods.ts b/packages/player/src/use-player-methods.ts
index eb70afa3f4..4b9d3aea0d 100644
--- a/packages/player/src/use-player-methods.ts
+++ b/packages/player/src/use-player-methods.ts
@@ -20,8 +20,8 @@ export type UsePlayerMethods = {
};
export const usePlayerMethods = (): UsePlayerMethods => {
- const setFrame = Internals.Timeline.useTimelineSetFrame();
- const setTimelinePosition = Internals.Timeline.useTimelineSetFrame();
+ const setFrame = Internals.Timeline.useTimelineSeekFrame();
+ const setTimelinePosition = setFrame;
const {
setPlaying,
frameRef,
@@ -77,18 +77,17 @@ export const usePlayerMethods = (): UsePlayerMethods => {
fallbackFrame.current = frameToSeekTo;
if (video?.id) {
+ setTimelinePosition((currentFrames) =>
+ currentFrames[video.id] === frameToSeekTo
+ ? currentFrames
+ : {...currentFrames, [video.id]: frameToSeekTo},
+ );
if (frameRef.current[video.id] !== frameToSeekTo) {
frameRef.current = {
...frameRef.current,
[video.id]: frameToSeekTo,
};
}
-
- setTimelinePosition((currentFrames) =>
- currentFrames[video.id] === frameToSeekTo
- ? currentFrames
- : {...currentFrames, [video.id]: frameToSeekTo},
- );
}
emitter.dispatchSeek(frameToSeekTo);
@@ -165,14 +164,14 @@ export const usePlayerMethods = (): UsePlayerMethods => {
setPlaying(false);
fallbackFrame.current = playStart.current;
if (config) {
- frameRef.current = {
- ...frameRef.current,
- [config.id]: playStart.current,
- };
setTimelinePosition((currentFrames) => ({
...currentFrames,
[config.id]: playStart.current,
}));
+ frameRef.current = {
+ ...frameRef.current,
+ [config.id]: playStart.current,
+ };
emitter.dispatchPause();
}
}
@@ -205,15 +204,15 @@ export const usePlayerMethods = (): UsePlayerMethods => {
return;
}
- frameRef.current = {
- ...frameRef.current,
- [videoId]: newFrame,
- };
setFrame((currentFrames) =>
currentFrames[videoId] === newFrame
? currentFrames
: {...currentFrames, [videoId]: newFrame},
);
+ frameRef.current = {
+ ...frameRef.current,
+ [videoId]: newFrame,
+ };
},
[frameRef, readIsPlaying, setFrame, videoId],
);
@@ -235,15 +234,15 @@ export const usePlayerMethods = (): UsePlayerMethods => {
return;
}
- frameRef.current = {
- ...frameRef.current,
- [videoId]: newFrame,
- };
setFrame((currentFrames) =>
currentFrames[videoId] === newFrame
? currentFrames
: {...currentFrames, [videoId]: newFrame},
);
+ frameRef.current = {
+ ...frameRef.current,
+ [videoId]: newFrame,
+ };
},
[frameRef, lastFrame, readIsPlaying, setFrame, videoId],
);
diff --git a/packages/studio/src/components/InitialCompositionLoader.tsx b/packages/studio/src/components/InitialCompositionLoader.tsx
index 64398ea2a1..d0ff40497c 100644
--- a/packages/studio/src/components/InitialCompositionLoader.tsx
+++ b/packages/studio/src/components/InitialCompositionLoader.tsx
@@ -19,7 +19,9 @@ import {useStaticFiles} from './use-static-files';
export const useSelectComposition = () => {
const {setCompositionFoldersExpanded} = useContext(FolderContext);
const {setCanvasContent} = useContext(Internals.CompositionSetters);
- const setFrame = Internals.useTimelineSetFrame();
+ // A supplied frame is explicit navigation (for example, opening a nested
+ // composition at its sequence's frame). Plain selection does not seek.
+ const setFrame = Internals.Timeline.useTimelineSeekFrame();
return useCallback(
(
diff --git a/packages/studio/src/components/InspectorPanel/KeyframeEasingNavigator.tsx b/packages/studio/src/components/InspectorPanel/KeyframeEasingNavigator.tsx
index 79547b90e6..ef7bf5cfd0 100644
--- a/packages/studio/src/components/InspectorPanel/KeyframeEasingNavigator.tsx
+++ b/packages/studio/src/components/InspectorPanel/KeyframeEasingNavigator.tsx
@@ -181,7 +181,7 @@ export const KeyframeEasingNavigator: React.FC<{
readonly nodePathInfo: SequenceNodePathInfo;
}> = ({currentSelection, includeEasings, keyframes, nodePathInfo}) => {
const {isSelected, selectItems} = useTimelineSelection();
- const setFrame = Internals.useTimelineSetFrame();
+ const setFrame = Internals.Timeline.useTimelineSeekFrame();
const videoConfig = useVideoConfig();
const items = useMemo(
() =>
diff --git a/packages/studio/src/components/Timeline/TimelineDragHandler.tsx b/packages/studio/src/components/Timeline/TimelineDragHandler.tsx
index dd7c3efeda..21e79626ae 100644
--- a/packages/studio/src/components/Timeline/TimelineDragHandler.tsx
+++ b/packages/studio/src/components/Timeline/TimelineDragHandler.tsx
@@ -135,7 +135,7 @@ const TimelineDragHandlerInner: React.FC = () => {
shouldApplyCssTransforms: true,
});
const {isHighestContext} = useZIndex();
- const setFrame = Internals.useTimelineSetFrame();
+ const setFrame = Internals.Timeline.useTimelineSeekFrame();
const width = getTimelineContentWidth();
const left = size?.left ?? 0;
diff --git a/packages/studio/src/components/Timeline/TimelineKeyframeControls.tsx b/packages/studio/src/components/Timeline/TimelineKeyframeControls.tsx
index 91543ce990..6f0db9a675 100644
--- a/packages/studio/src/components/Timeline/TimelineKeyframeControls.tsx
+++ b/packages/studio/src/components/Timeline/TimelineKeyframeControls.tsx
@@ -550,7 +550,7 @@ export const TimelineKeyframeControls: React.FC<{
}) => {
const videoConfig = useVideoConfig();
const timelinePosition = Internals.Timeline.useTimelinePosition();
- const setFrame = Internals.useTimelineSetFrame();
+ const setFrame = Internals.Timeline.useTimelineSeekFrame();
const {setPropStatuses} = useContext(Internals.VisualModeSettersContext);
const {propStatuses} = useContext(Internals.VisualModePropStatusesContext);
const {getDragOverrides, getEffectDragOverrides} = useContext(