Skip to content

🎥 Make videos programmatically with React

`@remotion/player`: Don't drop frames when a frame update commits after the next tick

auiphca172b520d3c6
diff --git a/packages/player/src/pending-playback-frame.ts b/packages/player/src/pending-playback-frame.ts
new file mode 100644
index 0000000000..8878154bb4
--- /dev/null
+++ b/packages/player/src/pending-playback-frame.ts
@@ -0,0 +1,35 @@
+/**
+ * The playback loop requests frames through a React state update, which commits
+ * asynchronously. When every animation frame advances the timeline (e.g. 2x
+ * playback of a 30fps composition on a 60Hz display), the next tick can run
+ * before the previous update has committed. Advancing from the stale committed
+ * frame then drops a frame permanently, and playback falls behind real time.
+ *
+ * Tracks the frame the loop last requested and the frames it advanced from.
+ * Reading one of those back means the request is still pending; any other
+ * frame came from elsewhere (e.g. a seek) and is adopted.
+ */
+export const createPendingPlaybackFrame = () => {
+ let requestedFrame: number | null = null;
+ const supersededFrames = new Set<number>();
+
+ return {
+ resolve(committedFrame: number): number {
+ if (
+ requestedFrame !== null &&
+ committedFrame !== requestedFrame &&
+ supersededFrames.has(committedFrame)
+ ) {
+ return requestedFrame;
+ }
+
+ requestedFrame = null;
+ supersededFrames.clear();
+ return committedFrame;
+ },
+ request(fromFrame: number, nextFrame: number) {
+ supersededFrames.add(fromFrame);
+ requestedFrame = nextFrame;
+ },
+ };
+};
diff --git a/packages/player/src/test/pending-playback-frame.test.ts b/packages/player/src/test/pending-playback-frame.test.ts
new file mode 100644
index 0000000000..c52201718e
--- /dev/null
+++ b/packages/player/src/test/pending-playback-frame.test.ts
@@ -0,0 +1,115 @@
+import {expect, test} from 'bun:test';
+import {calculateNextFrame} from '../calculate-next-frame.js';
+import {createPendingPlaybackFrame} from '../pending-playback-frame.js';
+
+// Mirrors the playback loop: one tick per animation frame on a 60Hz display.
+// `commitLate(tick)` decides whether React commits a requested frame only after
+// the next tick has already read the current frame.
+const play = ({
+ ticks,
+ playbackSpeed,
+ commitLate,
+ trackPending = true,
+ startFrame = 0,
+ lastFrame = 100_000,
+ shouldLoop = false,
+ seek,
+}: {
+ ticks: number;
+ playbackSpeed: number;
+ commitLate: (tick: number) => boolean;
+ trackPending?: boolean;
+ startFrame?: number;
+ lastFrame?: number;
+ shouldLoop?: boolean;
+ seek?: {atTick: number; frame: number};
+}) => {
+ let committedFrame = startFrame;
+ let queuedFrame: number | null = null;
+ let framesAdvanced = 0;
+ const pendingFrame = createPendingPlaybackFrame();
+
+ for (let tick = 1; tick <= ticks; tick++) {
+ if (seek?.atTick === tick) {
+ // Seeks write the frame ref synchronously.
+ committedFrame = seek.frame;
+ queuedFrame = null;
+ }
+
+ const currentFrame = trackPending
+ ? pendingFrame.resolve(committedFrame)
+ : committedFrame;
+ const {nextFrame, framesToAdvance} = calculateNextFrame({
+ time: (tick * 1000) / 60,
+ currentFrame,
+ playbackSpeed,
+ fps: 30,
+ actualFirstFrame: 0,
+ actualLastFrame: lastFrame,
+ framesAdvanced,
+ shouldLoop,
+ });
+ framesAdvanced += framesToAdvance;
+ if (nextFrame !== currentFrame) {
+ pendingFrame.request(currentFrame, nextFrame);
+ queuedFrame = nextFrame;
+ }
+
+ if (queuedFrame !== null && !commitLate(tick)) {
+ committedFrame = queuedFrame;
+ queuedFrame = null;
+ }
+ }
+
+ return queuedFrame ?? committedFrame;
+};
+
+const every = (n: number) => (tick: number) => tick % n === 0;
+
+test('Late commits drop frames at 2x without tracking the pending frame', () => {
+ expect(
+ play({
+ ticks: 300,
+ playbackSpeed: 2,
+ commitLate: every(23),
+ trackPending: false,
+ }),
+ ).toBeLessThan(300);
+});
+
+test('Playback keeps real time at 2x when commits land after the next tick', () => {
+ expect(play({ticks: 300, playbackSpeed: 2, commitLate: every(23)})).toBe(300);
+ expect(play({ticks: 300, playbackSpeed: 2, commitLate: every(2)})).toBe(300);
+ expect(play({ticks: 300, playbackSpeed: 2, commitLate: () => true})).toBe(
+ 300,
+ );
+});
+
+test('Playback keeps real time at 1x and 4x with late commits', () => {
+ expect(play({ticks: 300, playbackSpeed: 1, commitLate: every(3)})).toBe(150);
+ expect(play({ticks: 300, playbackSpeed: 4, commitLate: every(3)})).toBe(600);
+});
+
+test('A seek during playback is adopted while a request is pending', () => {
+ expect(
+ play({
+ ticks: 300,
+ playbackSpeed: 2,
+ commitLate: every(2),
+ startFrame: 600,
+ seek: {atTick: 151, frame: 100},
+ }),
+ ).toBe(100 + 150);
+});
+
+test('Looping playback wraps instead of sticking to the first frame', () => {
+ expect(
+ play({
+ ticks: 300,
+ playbackSpeed: 2,
+ commitLate: every(2),
+ lastFrame: 99,
+ shouldLoop: true,
+ }),
+ ).toBe(300 % 100);
+});
diff --git a/packages/player/src/use-playback.ts b/packages/player/src/use-playback.ts
index 8c600cae8b..37250dedb5 100644
--- a/packages/player/src/use-playback.ts
+++ b/packages/player/src/use-playback.ts
@@ -7,6 +7,7 @@ import type {BrowserMediaControlsBehavior} from './browser-mediasession.js';
import {useBrowserMediaSession} from './browser-mediasession.js';
import {calculateNextFrame} from './calculate-next-frame.js';
import {useIsBackgrounded} from './is-backgrounded.js';
+import {createPendingPlaybackFrame} from './pending-playback-frame.js';
import {setGlobalTimeAnchor} from './set-global-time-anchor.js';
import {type UsePlayerMethods, usePlayerMethods} from './use-player-methods.js';
@@ -202,6 +203,7 @@ export const usePlayback = ({
| null = null;
let startedTime = performance.now();
let framesAdvanced = 0;
+ const pendingFrame = createPendingPlaybackFrame();
const cancelQueuedFrame = () => {
if (reqAnimFrameCall !== null) {
@@ -236,7 +238,7 @@ export const usePlayback = ({
const actualLastFrame = outFrame ?? config.durationInFrames - 1;
const actualFirstFrame = inFrame ?? 0;
- const currentFrame = getCurrentFrame();
+ const currentFrame = pendingFrame.resolve(getCurrentFrame());
const {nextFrame, framesToAdvance, hasEnded} = calculateNextFrame({
time,
currentFrame,
@@ -251,10 +253,11 @@ export const usePlayback = ({
framesAdvanced += framesToAdvance;
if (
- nextFrame !== getCurrentFrame() &&
+ nextFrame !== currentFrame &&
(!hasEnded || moveToBeginningWhenEnded) &&
!isBuffering()
) {
+ pendingFrame.request(currentFrame, nextFrame);
setFrameFromPlayback((c) => ({...c, [config.id]: nextFrame}));
}