🎥 Make videos programmatically with React
`@remotion/studio`: Fix filmstrip repeating frames after zooming in
JonnyBurger62fbd53bdc44
diff --git a/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx b/packages/studio/src/components/Timeline/TimelineVideoInfo.tsxindex 63c00c1e63..a9e8512495 100644--- a/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx+++ b/packages/studio/src/components/Timeline/TimelineVideoInfo.tsx@@ -9,9 +9,11 @@ import {getAspectRatioFromCache,getFrameDatabaseKeyPrefix,getLoopDisplaySegments,+ getSlotsToExtract,getTimestampFromFrameDatabaseKey,hydrateThumbnailsFromPersistentCache,makeFrameDatabaseKey,+ mediaDurationCache,resizeVideoFrame,snapCanvasPositionToDevicePixel,WEBCODECS_TIMESCALE,@@ -119,7 +121,6 @@ const TimelineVideoInfoSegment: React.FC<{}const controller = new AbortController();- let mediaDurationInSeconds: number | null = null;const pixelRatio = getStudioPixelRatio();const {horizontalOffset} = alignCanvasToDevicePixels({canvas,@@ -226,7 +227,9 @@ const TimelineVideoInfoSegment: React.FC<{}) => {aspectRatio.current = track.width / track.height;aspectRatioCache.set(src, aspectRatio.current);- mediaDurationInSeconds = durationInSeconds;+ if (durationInSeconds !== null) {+ mediaDurationCache.set(src, durationInSeconds);+ }return [times.timestampInSeconds];},@@ -254,14 +257,7 @@ const TimelineVideoInfoSegment: React.FC<{);addFrameToCache(databaseKey, transformed);- if (aspectRatio.current !== null) {- writeThumbnailToPersistentCache({- src,- frame: transformed,- aspectRatio: aspectRatio.current,- durationInSeconds: mediaDurationInSeconds,- });- }+ writeThumbnailToPersistentCache({src, frame: transformed});drawRepeatedFrame(transformed);} catch (e) {@@ -368,12 +364,10 @@ const TimelineVideoInfoSegment: React.FC<{});repeatTarget();- return Array.from(filledSlots.values()).every(- (filled) => filled !== undefined,- );+ return getSlotsToExtract({filledSlots, src}).length === 0;};- // Don't extract frames if all slots are filled+ // Don't extract frames if every slot has a fitting frameif (fillWithFramesInMemory()) {return;}@@ -422,7 +416,9 @@ const TimelineVideoInfoSegment: React.FC<{}) => {aspectRatio.current = track.width / track.height;aspectRatioCache.set(src, aspectRatio.current);- mediaDurationInSeconds = durationInSeconds;+ if (durationInSeconds !== null) {+ mediaDurationCache.set(src, durationInSeconds);+ }ensureSlots({filledSlots,@@ -445,9 +441,9 @@ const TimelineVideoInfoSegment: React.FC<{});repeatTarget();- return Array.from(filledSlots.keys())- .filter((timestamp) => filledSlots.get(timestamp) === undefined)- .map((timestamp) => timestamp / WEBCODECS_TIMESCALE);+ return getSlotsToExtract({filledSlots, src}).map(+ (timestamp) => timestamp / WEBCODECS_TIMESCALE,+ );},src: resolvedSrc,onVideoSample: (sample) => {@@ -477,12 +473,7 @@ const TimelineVideoInfoSegment: React.FC<{throw new Error('Aspect ratio is not set');}- writeThumbnailToPersistentCache({- src,- frame: transformed,- aspectRatio: aspectRatio.current,- durationInSeconds: mediaDurationInSeconds,- });+ writeThumbnailToPersistentCache({src, frame: transformed});ensureSlots({filledSlots,@@ -494,6 +485,7 @@ const TimelineVideoInfoSegment: React.FC<{});fillFrameWhereItFits({ctx: targetCtx,+ src,filledSlots,visualizationWidth: targetWidth,frame: transformed,diff --git a/packages/timeline-utils/src/frame-database.ts b/packages/timeline-utils/src/frame-database.tsindex 6a6569efbb..b0f88bebeb 100644--- a/packages/timeline-utils/src/frame-database.ts+++ b/packages/timeline-utils/src/frame-database.ts@@ -24,6 +24,7 @@ let totalCacheSize = 0;export const frameDatabase: Map<FrameDatabaseKey, VideoFrameAndLastUsed> =new Map();export const aspectRatioCache: Map<string, number> = new Map();+export const mediaDurationCache: Map<string, number> = new Map();export const getTimestampFromFrameDatabaseKey = (key: FrameDatabaseKey) => {const split = key.split(KEY_SEPARATOR);diff --git a/packages/timeline-utils/src/index.ts b/packages/timeline-utils/src/index.tsindex 4e445f9df5..9a58a8318d 100644--- a/packages/timeline-utils/src/index.ts+++ b/packages/timeline-utils/src/index.ts@@ -35,6 +35,7 @@ export {getFrameDatabaseKeyPrefix,getTimestampFromFrameDatabaseKey,makeFrameDatabaseKey,+ mediaDurationCache,} from './frame-database';export type {FrameDatabaseKey} from './frame-database';export {@@ -56,6 +57,7 @@ export {fillFrameWhereItFits,fillWithCachedFrames,getDurationOfOneFrame,+ getSlotsToExtract,} from './render-frame-strip';export {renderFrameStripToCanvas} from './render-frame-strip-to-canvas';export type {RenderFrameStripToCanvasOptions} from './render-frame-strip-to-canvas';diff --git a/packages/timeline-utils/src/persistent-cache/persistent-cache.ts b/packages/timeline-utils/src/persistent-cache/persistent-cache.tsindex 8074627d06..3a25c67dd4 100644--- a/packages/timeline-utils/src/persistent-cache/persistent-cache.ts+++ b/packages/timeline-utils/src/persistent-cache/persistent-cache.ts@@ -5,8 +5,9 @@ import {aspectRatioCache,frameDatabase,makeFrameDatabaseKey,+ mediaDurationCache,} from '../frame-database';-import {MAX_TIME_DEVIATION, WEBCODECS_TIMESCALE} from '../render-frame-strip';+import {getFrameDistance, getSlotTolerance} from '../render-frame-strip';import {dequantizeWaveformPeaks,quantizeWaveformPeaks,@@ -59,7 +60,7 @@ type Fingerprint = {type PendingThumbnail = {readonly src: string;- readonly aspectRatio: number;+ readonly aspectRatio: number | null;readonly durationInSeconds: number | null;readonly height: number;readonly timestamp: number;@@ -363,6 +364,10 @@ const getDatabaseForSource = async (aspectRatioCache.set(src, record.aspectRatio);}+ if (record.durationInSeconds !== null && !mediaDurationCache.has(src)) {+ mediaDurationCache.set(src, record.durationInSeconds);+ }+scheduleEviction();return database;})().catch(() => null);@@ -462,11 +467,9 @@ export const writeWaveformToPersistentCache = ({})().catch(onWriteError);};-// Loads persisted thumbnails into the in-memory frame database, so the-// filmstrip can be drawn without decoding the video.-// Any frame in memory fills every slot of the filmstrip, so a partial set-// would prevent the missing frames from being decoded. Thumbnails are therefore-// only loaded if they cover all timestamps.+// Loads the persisted thumbnails that fit the slots of a filmstrip into the+// in-memory frame database, so they can be drawn without decoding the video.+// Slots without a fitting thumbnail are decoded afterwards.export const hydrateThumbnailsFromPersistentCache = async ({src,height,@@ -483,29 +486,20 @@ export const hydrateThumbnailsFromPersistentCache = async ({return;}- const transaction = database.transaction(['sources', 'thumbnails']);- const source: SourceRecord | undefined = await toPromise(- transaction.objectStore('sources').get(src),- );- // Like extractFrames(), show the last frame for timestamps after the end.- const durationInSeconds = source?.durationInSeconds ?? null;- const lastTimestamp =- durationInSeconds === null- ? Infinity- : durationInSeconds * WEBCODECS_TIMESCALE - 1;// Segments without a width produce slots that are not finite.- const timestamps = getTimestamps()- .filter((timestamp) => Number.isFinite(timestamp))- .map((timestamp) => Math.min(timestamp, lastTimestamp));+ const timestamps = getTimestamps().filter((timestamp) =>+ Number.isFinite(timestamp),+ );if (timestamps.length === 0) {return;}- const from = Math.min(...timestamps) - MAX_TIME_DEVIATION;- const to = Math.max(...timestamps) + MAX_TIME_DEVIATION;- const store = transaction.objectStore('thumbnails');- // A frame of a variable frame rate video can be shown long after its- // timestamp, so the last frame before the range may cover it as well.+ const tolerance = getSlotTolerance(timestamps);+ const from = Math.min(...timestamps) - tolerance;+ const to = Math.max(...timestamps) + tolerance;+ const store = database.transaction('thumbnails').objectStore('thumbnails');+ // A frame of a low or variable frame rate video, or the last frame of a+ // video, can be shown long after its timestamp.const previousRequest = store.openCursor(IDBKeyRange.bound([src, height], [src, height, from], false, true),'prev',@@ -521,23 +515,30 @@ export const hydrateThumbnailsFromPersistentCache = async ({? [previous.value, ...records]: records;- const covering = new Set<ThumbnailRecord>();- for (const timestamp of timestamps) {- const record = candidates.find(- (candidate) =>- Math.abs(candidate.timestamp - timestamp) <= MAX_TIME_DEVIATION ||- (candidate.timestamp <= timestamp &&- timestamp < candidate.timestamp + (candidate.duration ?? 0)),- );- if (!record) {- return;+ const fitting = new Set<ThumbnailRecord>();+ for (const slot of timestamps) {+ let best: ThumbnailRecord | undefined;+ let bestDistance = Infinity;+ for (const candidate of candidates) {+ const distance = getFrameDistance({+ src,+ frameTimestamp: candidate.timestamp,+ frameDuration: candidate.duration,+ slot,+ });+ if (distance < bestDistance) {+ bestDistance = distance;+ best = candidate;+ }}- covering.add(record);+ if (best && bestDistance <= tolerance) {+ fitting.add(best);+ }}await Promise.all(- Array.from(covering).map(async (record) => {+ Array.from(fitting).map(async (record) => {const key = makeFrameDatabaseKey(src, record.timestamp);if (frameDatabase.has(key)) {return;@@ -620,8 +621,11 @@ const flushThumbnails = async () => {);sources.put({...source,- aspectRatio: thumbnails[thumbnails.length - 1].aspectRatio,- durationInSeconds: thumbnails[thumbnails.length - 1].durationInSeconds,+ aspectRatio:+ thumbnails[thumbnails.length - 1].aspectRatio ?? source.aspectRatio,+ durationInSeconds:+ thumbnails[thumbnails.length - 1].durationInSeconds ??+ source.durationInSeconds,thumbnailBytes,} satisfies SourceRecord);await transactionDone(transaction);@@ -637,13 +641,9 @@ const flushThumbnails = async () => {export const writeThumbnailToPersistentCache = ({src,frame,- aspectRatio,- durationInSeconds,}: {readonly src: string;readonly frame: VideoFrame;- readonly aspectRatio: number;- readonly durationInSeconds: number | null;}) => {if (typeof OffscreenCanvas === 'undefined') {return;@@ -662,8 +662,8 @@ export const writeThumbnailToPersistentCache = ({image.catch(() => undefined);pendingThumbnails.push({src,- aspectRatio,- durationInSeconds,+ aspectRatio: aspectRatioCache.get(src) ?? null,+ durationInSeconds: mediaDurationCache.get(src) ?? null,height: frame.displayHeight,timestamp: frame.timestamp,duration: frame.duration,diff --git a/packages/timeline-utils/src/render-frame-strip-to-canvas.ts b/packages/timeline-utils/src/render-frame-strip-to-canvas.tsindex d3e2f262da..fb65e9a6a2 100644--- a/packages/timeline-utils/src/render-frame-strip-to-canvas.ts+++ b/packages/timeline-utils/src/render-frame-strip-to-canvas.ts@@ -4,11 +4,13 @@ import {aspectRatioCache,getAspectRatioFromCache,makeFrameDatabaseKey,+ mediaDurationCache,} from './frame-database';import {ensureSlots,fillFrameWhereItFits,fillWithCachedFrames,+ getSlotsToExtract,WEBCODECS_TIMESCALE,} from './render-frame-strip';import {resizeVideoFrame} from './resize-video-frame';@@ -89,10 +91,7 @@ export const renderFrameStripToCanvas = async ({horizontalOffset,});- const hasUnfilledSlots = Array.from(filledSlots.values()).some(- (value) => value === undefined,- );- if (!hasUnfilledSlots) {+ if (getSlotsToExtract({filledSlots, src}).length === 0) {return;}}@@ -100,9 +99,12 @@ export const renderFrameStripToCanvas = async ({await extractFrames({src,signal,- timestampsInSeconds: ({track}) => {+ timestampsInSeconds: ({track, durationInSeconds}) => {aspectRatio = track.width / track.height;aspectRatioCache.set(src, aspectRatio);+ if (durationInSeconds !== null) {+ mediaDurationCache.set(src, durationInSeconds);+ }ensureSlots({filledSlots,@@ -113,7 +115,7 @@ export const renderFrameStripToCanvas = async ({frameHeight,});- return Array.from(filledSlots.keys()).map(+ return getSlotsToExtract({filledSlots, src}).map((timestamp) => timestamp / WEBCODECS_TIMESCALE,);},@@ -150,6 +152,7 @@ export const renderFrameStripToCanvas = async ({fillFrameWhereItFits({ctx,+ src,filledSlots,visualizationWidth: drawingWidth,frame: transformed,diff --git a/packages/timeline-utils/src/render-frame-strip.ts b/packages/timeline-utils/src/render-frame-strip.tsindex fd218c4f28..e9e3f9c3ac 100644--- a/packages/timeline-utils/src/render-frame-strip.ts+++ b/packages/timeline-utils/src/render-frame-strip.ts@@ -1,14 +1,95 @@import {snapCanvasPositionToDevicePixel} from './device-pixel-alignment';-import type {FrameDatabaseKey} from './frame-database';import {frameDatabase,getFrameDatabaseKeyPrefix,- getTimestampFromFrameDatabaseKey,+ makeFrameDatabaseKey,+ mediaDurationCache,} from './frame-database';export const WEBCODECS_TIMESCALE = 1_000_000;export const MAX_TIME_DEVIATION = WEBCODECS_TIMESCALE * 0.05;+// How far a frame is from the frame that is shown at a slot, 0 if it is shown.+// The duration makes this work for low and variable frame rate videos.+// Like extractFrames(), the last frame is shown after the end of the media.+export const getFrameDistance = ({+ src,+ frameTimestamp,+ frameDuration,+ slot,+}: {+ src: string;+ frameTimestamp: number;+ frameDuration: number | null;+ slot: number;+}) => {+ const mediaDuration = mediaDurationCache.get(src);+ const timestamp =+ mediaDuration === undefined+ ? slot+ : Math.min(slot, mediaDuration * WEBCODECS_TIMESCALE - 1);+ if (+ frameTimestamp <= timestamp &&+ timestamp < frameTimestamp + (frameDuration ?? 0)+ ) {+ return 0;+ }++ return Math.abs(frameTimestamp - timestamp);+};++const getFilledDistance = ({+ src,+ filled,+ slot,+}: {+ src: string;+ filled: number | undefined;+ slot: number;+}) => {+ if (filled === undefined) {+ return Infinity;+ }++ return getFrameDistance({+ src,+ frameTimestamp: filled,+ frameDuration:+ frameDatabase.get(makeFrameDatabaseKey(src, filled))?.frame.duration ??+ null,+ slot,+ });+};++// A frame that is closer to a slot than to the neighboring slots is a good+// enough thumbnail for it, so zooming out reuses the frames of a closer zoom.+export const getSlotTolerance = (slots: number[]) => {+ if (slots.length < 2) {+ return MAX_TIME_DEVIATION;+ }++ const [first, second] = [...slots].sort((a, b) => a - b);+ return Math.max(MAX_TIME_DEVIATION, (second - first) / 2);+};++// Slots that are filled with a frame that is too far away, for example after+// zooming in, or not filled at all.+export const getSlotsToExtract = ({+ filledSlots,+ src,+}: {+ filledSlots: Map<number, number | undefined>;+ src: string;+}) => {+ const slots = Array.from(filledSlots.keys());+ const tolerance = getSlotTolerance(slots);++ return slots.filter(+ (slot) =>+ getFilledDistance({src, filled: filledSlots.get(slot), slot}) > tolerance,+ );+};+export const getDurationOfOneFrame = ({visualizationWidth,aspectRatio,@@ -180,47 +261,46 @@ export const fillWithCachedFrames = ({horizontalOffset: number;}) => {const prefix = getFrameDatabaseKeyPrefix(src);- const keys = Array.from(frameDatabase.keys()).filter((k) =>- k.startsWith(prefix),- );+ const candidates = Array.from(frameDatabase.entries())+ .filter(([key]) => key.startsWith(prefix))+ .map(([, candidate]) => candidate);const targets = Array.from(filledSlots.keys());for (const timestamp of targets) {- let bestKey: FrameDatabaseKey | undefined;+ let best: (typeof candidates)[number] | undefined;let bestDistance = Infinity;- for (const key of keys) {- const distance = Math.abs(- getTimestampFromFrameDatabaseKey(key) - timestamp,- );+ for (const candidate of candidates) {+ const distance = getFrameDistance({+ src,+ frameTimestamp: candidate.frame.timestamp,+ frameDuration: candidate.frame.duration,+ slot: timestamp,+ });if (distance < bestDistance) {bestDistance = distance;- bestKey = key;+ best = candidate;}}- if (!bestKey) {+ if (!best) {continue;}- const frame = frameDatabase.get(bestKey);- if (!frame) {- continue;- }-- const alreadyFilled = filledSlots.get(timestamp);if (- alreadyFilled &&- Math.abs(alreadyFilled - timestamp) <=- Math.abs(frame.frame.timestamp - timestamp)+ getFilledDistance({+ src,+ filled: filledSlots.get(timestamp),+ slot: timestamp,+ }) <= bestDistance) {continue;}- frame.lastUsed = Date.now();+ best.lastUsed = Date.now();drawSlot({ctx,- frame: frame.frame,+ frame: best.frame,filledSlots,visualizationWidth: naturalWidth,timestamp,@@ -235,6 +315,7 @@ export const fillWithCachedFrames = ({export const fillFrameWhereItFits = ({frame,+ src,filledSlots,ctx,visualizationWidth,@@ -245,6 +326,7 @@ export const fillFrameWhereItFits = ({horizontalOffset,}: {frame: VideoFrame;+ src: string;filledSlots: Map<number, number | undefined>;ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;visualizationWidth: number;@@ -256,16 +338,19 @@ export const fillFrameWhereItFits = ({}) => {const slots = Array.from(filledSlots.keys());- for (let i = 0; i < slots.length; i++) {- const slot = slots[i];- if (Math.abs(slot - frame.timestamp) > MAX_TIME_DEVIATION) {+ for (const slot of slots) {+ const distance = getFrameDistance({+ src,+ frameTimestamp: frame.timestamp,+ frameDuration: frame.duration,+ slot,+ });+ if (distance > MAX_TIME_DEVIATION) {continue;}- const filled = filledSlots.get(slot);if (- filled &&- Math.abs(filled - slot) <= Math.abs(filled - frame.timestamp)+ getFilledDistance({src, filled: filledSlots.get(slot), slot}) <= distance) {continue;}