Skip to content

🎥 Make videos programmatically with React

`@remotion/studio`: Fix filmstrip repeating frames after zooming in

JonnyBurger62fbd53bdc44
diff --git a/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx b/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx
index 63c00c1e63..a9e8512495 100644
--- a/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx
+++ b/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx
@@ -9,9 +9,11 @@ import {
getAspectRatioFromCache,
getFrameDatabaseKeyPrefix,
getLoopDisplaySegments,
+ getSlotsToExtract,
getTimestampFromFrameDatabaseKey,
hydrateThumbnailsFromPersistentCache,
makeFrameDatabaseKey,
+ mediaDurationCache,
resizeVideoFrame,
snapCanvasPositionToDevicePixel,
WEBCODECS_TIMESCALE,
@@ -119,7 +121,6 @@ const TimelineVideoInfoSegment: React.FC<{
}
const controller = new AbortController();
- let mediaDurationInSeconds: number | null = null;
const pixelRatio = getStudioPixelRatio();
const {horizontalOffset} = alignCanvasToDevicePixels({
canvas,
@@ -226,7 +227,9 @@ const TimelineVideoInfoSegment: React.FC<{
}) => {
aspectRatio.current = track.width / track.height;
aspectRatioCache.set(src, aspectRatio.current);
- mediaDurationInSeconds = durationInSeconds;
+ if (durationInSeconds !== null) {
+ mediaDurationCache.set(src, durationInSeconds);
+ }
return [times.timestampInSeconds];
},
@@ -254,14 +257,7 @@ const TimelineVideoInfoSegment: React.FC<{
);
addFrameToCache(databaseKey, transformed);
- if (aspectRatio.current !== null) {
- writeThumbnailToPersistentCache({
- src,
- frame: transformed,
- aspectRatio: aspectRatio.current,
- durationInSeconds: mediaDurationInSeconds,
- });
- }
+ writeThumbnailToPersistentCache({src, frame: transformed});
drawRepeatedFrame(transformed);
} catch (e) {
@@ -368,12 +364,10 @@ const TimelineVideoInfoSegment: React.FC<{
});
repeatTarget();
- return Array.from(filledSlots.values()).every(
- (filled) => filled !== undefined,
- );
+ return getSlotsToExtract({filledSlots, src}).length === 0;
};
- // Don't extract frames if all slots are filled
+ // Don't extract frames if every slot has a fitting frame
if (fillWithFramesInMemory()) {
return;
}
@@ -422,7 +416,9 @@ const TimelineVideoInfoSegment: React.FC<{
}) => {
aspectRatio.current = track.width / track.height;
aspectRatioCache.set(src, aspectRatio.current);
- mediaDurationInSeconds = durationInSeconds;
+ if (durationInSeconds !== null) {
+ mediaDurationCache.set(src, durationInSeconds);
+ }
ensureSlots({
filledSlots,
@@ -445,9 +441,9 @@ const TimelineVideoInfoSegment: React.FC<{
});
repeatTarget();
- return Array.from(filledSlots.keys())
- .filter((timestamp) => filledSlots.get(timestamp) === undefined)
- .map((timestamp) => timestamp / WEBCODECS_TIMESCALE);
+ return getSlotsToExtract({filledSlots, src}).map(
+ (timestamp) => timestamp / WEBCODECS_TIMESCALE,
+ );
},
src: resolvedSrc,
onVideoSample: (sample) => {
@@ -477,12 +473,7 @@ const TimelineVideoInfoSegment: React.FC<{
throw new Error('Aspect ratio is not set');
}
- writeThumbnailToPersistentCache({
- src,
- frame: transformed,
- aspectRatio: aspectRatio.current,
- durationInSeconds: mediaDurationInSeconds,
- });
+ writeThumbnailToPersistentCache({src, frame: transformed});
ensureSlots({
filledSlots,
@@ -494,6 +485,7 @@ const TimelineVideoInfoSegment: React.FC<{
});
fillFrameWhereItFits({
ctx: targetCtx,
+ src,
filledSlots,
visualizationWidth: targetWidth,
frame: transformed,
diff --git a/packages/timeline-utils/src/frame-database.ts b/packages/timeline-utils/src/frame-database.ts
index 6a6569efbb..b0f88bebeb 100644
--- a/packages/timeline-utils/src/frame-database.ts
+++ b/packages/timeline-utils/src/frame-database.ts
@@ -24,6 +24,7 @@ let totalCacheSize = 0;
export const frameDatabase: Map<FrameDatabaseKey, VideoFrameAndLastUsed> =
new Map();
export const aspectRatioCache: Map<string, number> = new Map();
+export const mediaDurationCache: Map<string, number> = new Map();
export const getTimestampFromFrameDatabaseKey = (key: FrameDatabaseKey) => {
const split = key.split(KEY_SEPARATOR);
diff --git a/packages/timeline-utils/src/index.ts b/packages/timeline-utils/src/index.ts
index 4e445f9df5..9a58a8318d 100644
--- a/packages/timeline-utils/src/index.ts
+++ b/packages/timeline-utils/src/index.ts
@@ -35,6 +35,7 @@ export {
getFrameDatabaseKeyPrefix,
getTimestampFromFrameDatabaseKey,
makeFrameDatabaseKey,
+ mediaDurationCache,
} from './frame-database';
export type {FrameDatabaseKey} from './frame-database';
export {
@@ -56,6 +57,7 @@ export {
fillFrameWhereItFits,
fillWithCachedFrames,
getDurationOfOneFrame,
+ getSlotsToExtract,
} from './render-frame-strip';
export {renderFrameStripToCanvas} from './render-frame-strip-to-canvas';
export type {RenderFrameStripToCanvasOptions} from './render-frame-strip-to-canvas';
diff --git a/packages/timeline-utils/src/persistent-cache/persistent-cache.ts b/packages/timeline-utils/src/persistent-cache/persistent-cache.ts
index 8074627d06..3a25c67dd4 100644
--- a/packages/timeline-utils/src/persistent-cache/persistent-cache.ts
+++ b/packages/timeline-utils/src/persistent-cache/persistent-cache.ts
@@ -5,8 +5,9 @@ import {
aspectRatioCache,
frameDatabase,
makeFrameDatabaseKey,
+ mediaDurationCache,
} from '../frame-database';
-import {MAX_TIME_DEVIATION, WEBCODECS_TIMESCALE} from '../render-frame-strip';
+import {getFrameDistance, getSlotTolerance} from '../render-frame-strip';
import {
dequantizeWaveformPeaks,
quantizeWaveformPeaks,
@@ -59,7 +60,7 @@ type Fingerprint = {
type PendingThumbnail = {
readonly src: string;
- readonly aspectRatio: number;
+ readonly aspectRatio: number | null;
readonly durationInSeconds: number | null;
readonly height: number;
readonly timestamp: number;
@@ -363,6 +364,10 @@ const getDatabaseForSource = async (
aspectRatioCache.set(src, record.aspectRatio);
}
+ if (record.durationInSeconds !== null && !mediaDurationCache.has(src)) {
+ mediaDurationCache.set(src, record.durationInSeconds);
+ }
+
scheduleEviction();
return database;
})().catch(() => null);
@@ -462,11 +467,9 @@ export const writeWaveformToPersistentCache = ({
})().catch(onWriteError);
};
-// Loads persisted thumbnails into the in-memory frame database, so the
-// filmstrip can be drawn without decoding the video.
-// Any frame in memory fills every slot of the filmstrip, so a partial set
-// would prevent the missing frames from being decoded. Thumbnails are therefore
-// only loaded if they cover all timestamps.
+// Loads the persisted thumbnails that fit the slots of a filmstrip into the
+// in-memory frame database, so they can be drawn without decoding the video.
+// Slots without a fitting thumbnail are decoded afterwards.
export const hydrateThumbnailsFromPersistentCache = async ({
src,
height,
@@ -483,29 +486,20 @@ export const hydrateThumbnailsFromPersistentCache = async ({
return;
}
- const transaction = database.transaction(['sources', 'thumbnails']);
- const source: SourceRecord | undefined = await toPromise(
- transaction.objectStore('sources').get(src),
- );
- // Like extractFrames(), show the last frame for timestamps after the end.
- const durationInSeconds = source?.durationInSeconds ?? null;
- const lastTimestamp =
- durationInSeconds === null
- ? Infinity
- : durationInSeconds * WEBCODECS_TIMESCALE - 1;
// Segments without a width produce slots that are not finite.
- const timestamps = getTimestamps()
- .filter((timestamp) => Number.isFinite(timestamp))
- .map((timestamp) => Math.min(timestamp, lastTimestamp));
+ const timestamps = getTimestamps().filter((timestamp) =>
+ Number.isFinite(timestamp),
+ );
if (timestamps.length === 0) {
return;
}
- const from = Math.min(...timestamps) - MAX_TIME_DEVIATION;
- const to = Math.max(...timestamps) + MAX_TIME_DEVIATION;
- const store = transaction.objectStore('thumbnails');
- // A frame of a variable frame rate video can be shown long after its
- // timestamp, so the last frame before the range may cover it as well.
+ const tolerance = getSlotTolerance(timestamps);
+ const from = Math.min(...timestamps) - tolerance;
+ const to = Math.max(...timestamps) + tolerance;
+ const store = database.transaction('thumbnails').objectStore('thumbnails');
+ // A frame of a low or variable frame rate video, or the last frame of a
+ // video, can be shown long after its timestamp.
const previousRequest = store.openCursor(
IDBKeyRange.bound([src, height], [src, height, from], false, true),
'prev',
@@ -521,23 +515,30 @@ export const hydrateThumbnailsFromPersistentCache = async ({
? [previous.value, ...records]
: records;
- const covering = new Set<ThumbnailRecord>();
- for (const timestamp of timestamps) {
- const record = candidates.find(
- (candidate) =>
- Math.abs(candidate.timestamp - timestamp) <= MAX_TIME_DEVIATION ||
- (candidate.timestamp <= timestamp &&
- timestamp < candidate.timestamp + (candidate.duration ?? 0)),
- );
- if (!record) {
- return;
+ const fitting = new Set<ThumbnailRecord>();
+ for (const slot of timestamps) {
+ let best: ThumbnailRecord | undefined;
+ let bestDistance = Infinity;
+ for (const candidate of candidates) {
+ const distance = getFrameDistance({
+ src,
+ frameTimestamp: candidate.timestamp,
+ frameDuration: candidate.duration,
+ slot,
+ });
+ if (distance < bestDistance) {
+ bestDistance = distance;
+ best = candidate;
+ }
}
- covering.add(record);
+ if (best && bestDistance <= tolerance) {
+ fitting.add(best);
+ }
}
await Promise.all(
- Array.from(covering).map(async (record) => {
+ Array.from(fitting).map(async (record) => {
const key = makeFrameDatabaseKey(src, record.timestamp);
if (frameDatabase.has(key)) {
return;
@@ -620,8 +621,11 @@ const flushThumbnails = async () => {
);
sources.put({
...source,
- aspectRatio: thumbnails[thumbnails.length - 1].aspectRatio,
- durationInSeconds: thumbnails[thumbnails.length - 1].durationInSeconds,
+ aspectRatio:
+ thumbnails[thumbnails.length - 1].aspectRatio ?? source.aspectRatio,
+ durationInSeconds:
+ thumbnails[thumbnails.length - 1].durationInSeconds ??
+ source.durationInSeconds,
thumbnailBytes,
} satisfies SourceRecord);
await transactionDone(transaction);
@@ -637,13 +641,9 @@ const flushThumbnails = async () => {
export const writeThumbnailToPersistentCache = ({
src,
frame,
- aspectRatio,
- durationInSeconds,
}: {
readonly src: string;
readonly frame: VideoFrame;
- readonly aspectRatio: number;
- readonly durationInSeconds: number | null;
}) => {
if (typeof OffscreenCanvas === 'undefined') {
return;
@@ -662,8 +662,8 @@ export const writeThumbnailToPersistentCache = ({
image.catch(() => undefined);
pendingThumbnails.push({
src,
- aspectRatio,
- durationInSeconds,
+ aspectRatio: aspectRatioCache.get(src) ?? null,
+ durationInSeconds: mediaDurationCache.get(src) ?? null,
height: frame.displayHeight,
timestamp: frame.timestamp,
duration: frame.duration,
diff --git a/packages/timeline-utils/src/render-frame-strip-to-canvas.ts b/packages/timeline-utils/src/render-frame-strip-to-canvas.ts
index d3e2f262da..fb65e9a6a2 100644
--- a/packages/timeline-utils/src/render-frame-strip-to-canvas.ts
+++ b/packages/timeline-utils/src/render-frame-strip-to-canvas.ts
@@ -4,11 +4,13 @@ import {
aspectRatioCache,
getAspectRatioFromCache,
makeFrameDatabaseKey,
+ mediaDurationCache,
} from './frame-database';
import {
ensureSlots,
fillFrameWhereItFits,
fillWithCachedFrames,
+ getSlotsToExtract,
WEBCODECS_TIMESCALE,
} from './render-frame-strip';
import {resizeVideoFrame} from './resize-video-frame';
@@ -89,10 +91,7 @@ export const renderFrameStripToCanvas = async ({
horizontalOffset,
});
- const hasUnfilledSlots = Array.from(filledSlots.values()).some(
- (value) => value === undefined,
- );
- if (!hasUnfilledSlots) {
+ if (getSlotsToExtract({filledSlots, src}).length === 0) {
return;
}
}
@@ -100,9 +99,12 @@ export const renderFrameStripToCanvas = async ({
await extractFrames({
src,
signal,
- timestampsInSeconds: ({track}) => {
+ timestampsInSeconds: ({track, durationInSeconds}) => {
aspectRatio = track.width / track.height;
aspectRatioCache.set(src, aspectRatio);
+ if (durationInSeconds !== null) {
+ mediaDurationCache.set(src, durationInSeconds);
+ }
ensureSlots({
filledSlots,
@@ -113,7 +115,7 @@ export const renderFrameStripToCanvas = async ({
frameHeight,
});
- return Array.from(filledSlots.keys()).map(
+ return getSlotsToExtract({filledSlots, src}).map(
(timestamp) => timestamp / WEBCODECS_TIMESCALE,
);
},
@@ -150,6 +152,7 @@ export const renderFrameStripToCanvas = async ({
fillFrameWhereItFits({
ctx,
+ src,
filledSlots,
visualizationWidth: drawingWidth,
frame: transformed,
diff --git a/packages/timeline-utils/src/render-frame-strip.ts b/packages/timeline-utils/src/render-frame-strip.ts
index fd218c4f28..e9e3f9c3ac 100644
--- a/packages/timeline-utils/src/render-frame-strip.ts
+++ b/packages/timeline-utils/src/render-frame-strip.ts
@@ -1,14 +1,95 @@
import {snapCanvasPositionToDevicePixel} from './device-pixel-alignment';
-import type {FrameDatabaseKey} from './frame-database';
import {
frameDatabase,
getFrameDatabaseKeyPrefix,
- getTimestampFromFrameDatabaseKey,
+ makeFrameDatabaseKey,
+ mediaDurationCache,
} from './frame-database';
export const WEBCODECS_TIMESCALE = 1_000_000;
export const MAX_TIME_DEVIATION = WEBCODECS_TIMESCALE * 0.05;
+// How far a frame is from the frame that is shown at a slot, 0 if it is shown.
+// The duration makes this work for low and variable frame rate videos.
+// Like extractFrames(), the last frame is shown after the end of the media.
+export const getFrameDistance = ({
+ src,
+ frameTimestamp,
+ frameDuration,
+ slot,
+}: {
+ src: string;
+ frameTimestamp: number;
+ frameDuration: number | null;
+ slot: number;
+}) => {
+ const mediaDuration = mediaDurationCache.get(src);
+ const timestamp =
+ mediaDuration === undefined
+ ? slot
+ : Math.min(slot, mediaDuration * WEBCODECS_TIMESCALE - 1);
+ if (
+ frameTimestamp <= timestamp &&
+ timestamp < frameTimestamp + (frameDuration ?? 0)
+ ) {
+ return 0;
+ }
+
+ return Math.abs(frameTimestamp - timestamp);
+};
+
+const getFilledDistance = ({
+ src,
+ filled,
+ slot,
+}: {
+ src: string;
+ filled: number | undefined;
+ slot: number;
+}) => {
+ if (filled === undefined) {
+ return Infinity;
+ }
+
+ return getFrameDistance({
+ src,
+ frameTimestamp: filled,
+ frameDuration:
+ frameDatabase.get(makeFrameDatabaseKey(src, filled))?.frame.duration ??
+ null,
+ slot,
+ });
+};
+
+// A frame that is closer to a slot than to the neighboring slots is a good
+// enough thumbnail for it, so zooming out reuses the frames of a closer zoom.
+export const getSlotTolerance = (slots: number[]) => {
+ if (slots.length < 2) {
+ return MAX_TIME_DEVIATION;
+ }
+
+ const [first, second] = [...slots].sort((a, b) => a - b);
+ return Math.max(MAX_TIME_DEVIATION, (second - first) / 2);
+};
+
+// Slots that are filled with a frame that is too far away, for example after
+// zooming in, or not filled at all.
+export const getSlotsToExtract = ({
+ filledSlots,
+ src,
+}: {
+ filledSlots: Map<number, number | undefined>;
+ src: string;
+}) => {
+ const slots = Array.from(filledSlots.keys());
+ const tolerance = getSlotTolerance(slots);
+
+ return slots.filter(
+ (slot) =>
+ getFilledDistance({src, filled: filledSlots.get(slot), slot}) > tolerance,
+ );
+};
+
export const getDurationOfOneFrame = ({
visualizationWidth,
aspectRatio,
@@ -180,47 +261,46 @@ export const fillWithCachedFrames = ({
horizontalOffset: number;
}) => {
const prefix = getFrameDatabaseKeyPrefix(src);
- const keys = Array.from(frameDatabase.keys()).filter((k) =>
- k.startsWith(prefix),
- );
+ const candidates = Array.from(frameDatabase.entries())
+ .filter(([key]) => key.startsWith(prefix))
+ .map(([, candidate]) => candidate);
const targets = Array.from(filledSlots.keys());
for (const timestamp of targets) {
- let bestKey: FrameDatabaseKey | undefined;
+ let best: (typeof candidates)[number] | undefined;
let bestDistance = Infinity;
- for (const key of keys) {
- const distance = Math.abs(
- getTimestampFromFrameDatabaseKey(key) - timestamp,
- );
+ for (const candidate of candidates) {
+ const distance = getFrameDistance({
+ src,
+ frameTimestamp: candidate.frame.timestamp,
+ frameDuration: candidate.frame.duration,
+ slot: timestamp,
+ });
if (distance < bestDistance) {
bestDistance = distance;
- bestKey = key;
+ best = candidate;
}
}
- if (!bestKey) {
+ if (!best) {
continue;
}
- const frame = frameDatabase.get(bestKey);
- if (!frame) {
- continue;
- }
-
- const alreadyFilled = filledSlots.get(timestamp);
if (
- alreadyFilled &&
- Math.abs(alreadyFilled - timestamp) <=
- Math.abs(frame.frame.timestamp - timestamp)
+ getFilledDistance({
+ src,
+ filled: filledSlots.get(timestamp),
+ slot: timestamp,
+ }) <= bestDistance
) {
continue;
}
- frame.lastUsed = Date.now();
+ best.lastUsed = Date.now();
drawSlot({
ctx,
- frame: frame.frame,
+ frame: best.frame,
filledSlots,
visualizationWidth: naturalWidth,
timestamp,
@@ -235,6 +315,7 @@ export const fillWithCachedFrames = ({
export const fillFrameWhereItFits = ({
frame,
+ src,
filledSlots,
ctx,
visualizationWidth,
@@ -245,6 +326,7 @@ export const fillFrameWhereItFits = ({
horizontalOffset,
}: {
frame: VideoFrame;
+ src: string;
filledSlots: Map<number, number | undefined>;
ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
visualizationWidth: number;
@@ -256,16 +338,19 @@ export const fillFrameWhereItFits = ({
}) => {
const slots = Array.from(filledSlots.keys());
- for (let i = 0; i < slots.length; i++) {
- const slot = slots[i];
- if (Math.abs(slot - frame.timestamp) > MAX_TIME_DEVIATION) {
+ for (const slot of slots) {
+ const distance = getFrameDistance({
+ src,
+ frameTimestamp: frame.timestamp,
+ frameDuration: frame.duration,
+ slot,
+ });
+ if (distance > MAX_TIME_DEVIATION) {
continue;
}
- const filled = filledSlots.get(slot);
if (
- filled &&
- Math.abs(filled - slot) <= Math.abs(filled - frame.timestamp)
+ getFilledDistance({src, filled: filledSlots.get(slot), slot}) <= distance
) {
continue;
}