Skip to content

Write HTML. Render video. Built for agents.

perf(studio): zooming a long timeline re-decodes and redraws less (#5142)

Miguel Ángel9f09861d2e78
diff --git a/packages/core/src/beats/beatDetection.test.ts b/packages/core/src/beats/beatDetection.test.ts
index 31bd167535..f0a6ec72fc 100644
--- a/packages/core/src/beats/beatDetection.test.ts
+++ b/packages/core/src/beats/beatDetection.test.ts
@@ -1,6 +1,6 @@
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
-import { loadBpmDetective } from "./beatDetection";
+import { analyzeMusicFromBuffer, loadBpmDetective } from "./beatDetection";
describe("loadBpmDetective", () => {
it("resolves to null when the import is unavailable", async () => {
@@ -39,3 +39,72 @@ describe("loadBpmDetective", () => {
expect(detect).toBe(fn);
});
});
+
+describe("analyzeMusicFromBuffer", () => {
+ it("waits on the caller's pause before each long stage", async () => {
+ const sampleRate = 8000;
+ const channel = new Float32Array(sampleRate * 2);
+ for (let beat = 0; beat < 4; beat++) channel.fill(0.9, beat * 4000, beat * 4000 + 80);
+ const audio = {
+ getChannelData: () => channel,
+ sampleRate,
+ duration: 2,
+ } as unknown as AudioBuffer;
+ const held: (() => void)[] = [];
+ const pause = () => new Promise<void>((resolve) => held.push(resolve));
+ let done = false;
+ const analysis = analyzeMusicFromBuffer(audio, { pause }).then((result) => {
+ done = true;
+ return result;
+ });
+ for (let stage = 1; stage <= 3; stage++) {
+ await vi.waitFor(() => expect(held).toHaveLength(stage));
+ expect(done).toBe(false);
+ held[stage - 1]!();
+ }
+ await expect(analysis).resolves.toMatchObject({ sampleRate });
+ });
+
+ it("rechecks pause after a slow tempo-detector import", async () => {
+ let releaseImport!: () => void;
+ const importGate = new Promise<void>((resolve) => (releaseImport = resolve));
+ let importStarted = false;
+ let moving = false;
+ const movingAtDetect: boolean[] = [];
+ const detect = vi.fn(() => {
+ movingAtDetect.push(moving);
+ return 120;
+ });
+ vi.resetModules();
+ vi.doMock("bpm-detective", async () => {
+ importStarted = true;
+ await importGate;
+ return { default: detect };
+ });
+ try {
+ const { analyzeMusicFromBuffer: analyze } = await import("./beatDetection");
+ const sampleRate = 8000;
+ const audio = {
+ getChannelData: () => new Float32Array(sampleRate * 2),
+ sampleRate,
+ duration: 2,
+ } as unknown as AudioBuffer;
+ const waiters: (() => void)[] = [];
+ const pause = () =>
+ moving ? new Promise<void>((resolve) => waiters.push(resolve)) : Promise.resolve();
+ const analysis = analyze(audio, { pause });
+ await vi.waitFor(() => expect(importStarted).toBe(true));
+ moving = true;
+ releaseImport();
+ await vi.waitFor(() => expect(waiters).toHaveLength(1));
+ expect(detect).not.toHaveBeenCalled();
+ moving = false;
+ waiters.shift()!();
+ await analysis;
+ expect(movingAtDetect).toEqual([false]);
+ } finally {
+ vi.doUnmock("bpm-detective");
+ vi.resetModules();
+ }
+ });
+});
diff --git a/packages/core/src/beats/beatDetection.ts b/packages/core/src/beats/beatDetection.ts
index 59bc109f6b..db2eb2399a 100644
--- a/packages/core/src/beats/beatDetection.ts
+++ b/packages/core/src/beats/beatDetection.ts
@@ -212,18 +212,27 @@ function gateBeatsBySilence(
return { times, strengths, peak };
}
+export interface MusicAnalysisOptions {
+ pause?: () => Promise<void>;
+}
+
// fallow-ignore-next-line complexity
-export async function analyzeMusicFromBuffer(audioBuffer: AudioBuffer): Promise<MusicBeatAnalysis> {
+export async function analyzeMusicFromBuffer(
+ audioBuffer: AudioBuffer,
+ { pause }: MusicAnalysisOptions = {},
+): Promise<MusicBeatAnalysis> {
const channelData = audioBuffer.getChannelData(0);
const sampleRate = audioBuffer.sampleRate;
const duration = audioBuffer.duration;
+ await pause?.();
const rawBeats = await detectBeats(audioBuffer);
const onsetBpm = computeBpmFromBeats(rawBeats);
let detectiveBpm: number | null = null;
+ const detect = await loadBpmDetective();
+ await pause?.();
try {
- const detect = await loadBpmDetective();
if (detect) detectiveBpm = detect(audioBuffer);
} catch {
// Not enough peaks or browser context unavailable
@@ -261,6 +270,7 @@ export async function analyzeMusicFromBuffer(audioBuffer: AudioBuffer): Promise<
regularizeBpm = detectiveBpm;
}
+ await pause?.();
const gridBeats =
regularizeBpm !== null ? regularizeBeats(rawBeats, regularizeBpm, duration) : rawBeats;
const gated = gateBeatsBySilence(gridBeats, channelData, sampleRate);
@@ -288,13 +298,16 @@ export async function detectBeatsFromUrl(url: string): Promise<number[]> {
}
}
-export async function analyzeMusicFromUrl(url: string): Promise<MusicBeatAnalysis> {
+export async function analyzeMusicFromUrl(
+ url: string,
+ options: MusicAnalysisOptions = {},
+): Promise<MusicBeatAnalysis> {
const audioContext = new AudioContext();
try {
const response = await fetch(url);
const arrayBuffer = await response.arrayBuffer();
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
- return analyzeMusicFromBuffer(audioBuffer);
+ return analyzeMusicFromBuffer(audioBuffer, options);
} finally {
await audioContext.close();
}
diff --git a/packages/studio/src/hooks/useMusicBeatAnalysis.ts b/packages/studio/src/hooks/useMusicBeatAnalysis.ts
index 8d0f2dc585..e71339ba99 100644
--- a/packages/studio/src/hooks/useMusicBeatAnalysis.ts
+++ b/packages/studio/src/hooks/useMusicBeatAnalysis.ts
@@ -2,6 +2,7 @@ import { useEffect, useMemo, useRef } from "react";
import { usePlayerStore } from "../player/store/playerStore";
import { resolveBeatSourceTrack } from "../utils/timelineInspector";
import { analyzeMusicFromUrl } from "@hyperframes/core/beats";
+import { whenTimelineIdle } from "../player/components/timelineMotion";
import { useFileManagerContextOptional } from "../contexts/FileManagerContext";
import { mergeUserBeats } from "../utils/beatEditing";
import { getTimelineElementIndexes } from "../player/lib/timelineElementIndexes";
@@ -79,7 +80,7 @@ async function loadBeatAnalysis(
): Promise<{ analysis: MusicAnalysis; times: number[]; strengths: number[] } | null> {
let promise = analysisCache.get(musicSrc);
if (!promise) {
- promise = analyzeMusicFromUrl(musicSrc);
+ promise = analyzeMusicFromUrl(musicSrc, { pause: whenTimelineIdle });
cacheAnalysis(musicSrc, promise);
}
try {
diff --git a/packages/studio/src/player/components/AudioWaveform.test.tsx b/packages/studio/src/player/components/AudioWaveform.test.tsx
index 1652e04b9a..20819a8c97 100644
--- a/packages/studio/src/player/components/AudioWaveform.test.tsx
+++ b/packages/studio/src/player/components/AudioWaveform.test.tsx
@@ -11,6 +11,29 @@ vi.mock("../../hooks/useThumbnailLease", () => ({
useThumbnailLease: leaseSpy,
}));
+// One observer serves every waveform; it reports each one near as it is watched, unless a test
+// reports by hand.
+const nearScreen = {
+ auto: true,
+ watched: [] as Element[],
+ report: (_target: Element, _near: boolean) => {},
+};
+globalThis.IntersectionObserver = class {
+ constructor(callback: IntersectionObserverCallback) {
+ nearScreen.report = (target, near) =>
+ callback(
+ [{ target, isIntersecting: near } as IntersectionObserverEntry],
+ this as unknown as IntersectionObserver,
+ );
+ }
+ observe(target: Element) {
+ nearScreen.watched.push(target);
+ if (nearScreen.auto) nearScreen.report(target, true);
+ }
+ unobserve() {}
+ disconnect() {}
+} as unknown as typeof IntersectionObserver;
+
const EMPTY_STRIP = { width: 0, height: 0, inViewStart: 0, inViewEnd: 0 };
const watchGap = vi.hoisted(() => vi.fn());
const strip = vi.hoisted(() => ({
@@ -52,6 +75,7 @@ afterEach(() => {
leaseSpy.mockReset();
leaseSpy.mockImplementation(() => ({ status: "loading" as const }));
strip.size = EMPTY_STRIP;
+ nearScreen.auto = true;
watchGap.mockClear();
document.body.innerHTML = "";
});
@@ -141,6 +165,23 @@ describe("AudioWaveform", () => {
expect(watchGap.mock.calls.filter(([gap]) => gap)).toEqual([]);
});
+ it("leaves a waveform off screen undrawn through a zoom, and draws it once it comes near", () => {
+ nearScreen.auto = false;
+ const getContext = vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null);
+ setStrip({ width: 1000, height: 40, inViewStart: 0, inViewEnd: 1000 });
+ renderReadyWaveform();
+ const root = nearScreen.watched.at(-1)!;
+ expect(getContext).not.toHaveBeenCalled();
+ act(() => nearScreen.report(root, true));
+ const drawn = getContext.mock.calls.length;
+ expect(drawn).toBeGreaterThan(0);
+ act(() => nearScreen.report(root, false));
+ setStrip({ width: 2000, height: 40, inViewStart: 0, inViewEnd: 2000 });
+ expect(getContext.mock.calls.length).toBe(drawn);
+ act(() => nearScreen.report(root, true));
+ expect(getContext.mock.calls.length).toBe(drawn + 1);
+ });
+
it("redraws a short clip, drawn whole, when a zoom changes its width", () => {
const getContext = vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null);
setStrip({ width: 1000, height: 40, inViewStart: 0, inViewEnd: 1000 });
diff --git a/packages/studio/src/player/components/AudioWaveform.tsx b/packages/studio/src/player/components/AudioWaveform.tsx
index d7e226ba1b..ef5b2f5514 100644
--- a/packages/studio/src/player/components/AudioWaveform.tsx
+++ b/packages/studio/src/player/components/AudioWaveform.tsx
@@ -1,4 +1,4 @@
-import { memo, useCallback, useContext, useEffect, useMemo, useRef } from "react";
+import { memo, useCallback, useContext, useEffect, useMemo, useRef, useState } from "react";
import { fadeGain } from "@hyperframes/core/audio-fade";
import { useThumbnailLease } from "../../hooks/useThumbnailLease";
import { useThumbnailStripSize } from "../../hooks/useThumbnailStripSize";
@@ -158,6 +158,32 @@ async function decodeWaveformPeaks(url: string, signal: AbortSignal): Promise<nu
}
/** Bounded waveform subscriber; cache, cancellation and dedupe live in one scheduler. */
+const NEAR_SCREEN: IntersectionObserverInit & { scrollMargin: string } = {
+ rootMargin: "50% 100%",
+ scrollMargin: "50% 100%",
+};
+const nearScreenListeners = new Map<Element, (near: boolean) => void>();
+let nearScreen: IntersectionObserver | null = null;
+
+function useNearScreen(element: Element | null): boolean {
+ const [near, setNear] = useState(() => typeof IntersectionObserver === "undefined");
+ useEffect(() => {
+ if (!element || typeof IntersectionObserver === "undefined") return;
+ nearScreen ??= new IntersectionObserver(
+ (entries) =>
+ entries.forEach((entry) => nearScreenListeners.get(entry.target)?.(entry.isIntersecting)),
+ NEAR_SCREEN,
+ );
+ nearScreenListeners.set(element, setNear);
+ nearScreen.observe(element);
+ return () => {
+ nearScreenListeners.delete(element);
+ nearScreen?.unobserve(element);
+ };
+ }, [element]);
+ return near;
+}
+
export const AudioWaveform = memo(function AudioWaveform({
audioUrl,
waveformUrl,
@@ -171,7 +197,7 @@ export const AudioWaveform = memo(function AudioWaveform({
muted = false,
labelInset = 16,
}: AudioWaveformProps) {
- const rootRef = useRef<HTMLDivElement | null>(null);
+ const [root, setRoot] = useState<HTMLDivElement | null>(null);
const canvasRef = useRef<HTMLCanvasElement | null>(null);
const cacheKey = waveformUrl ?? audioUrl;
const request = useMemo(
@@ -218,7 +244,10 @@ export const AudioWaveform = memo(function AudioWaveform({
span,
);
}, [fades, from, muted, peaks, to, trimEndFraction, trimStartFraction]);
- useEffect(draw, [draw, strip.width, strip.height]);
+ const near = useNearScreen(root);
+ useEffect(() => {
+ if (near) draw();
+ }, [draw, near, strip.width, strip.height]);
useEffect(() => {
const root = document.documentElement;
@@ -231,15 +260,15 @@ export const AudioWaveform = memo(function AudioWaveform({
}, [draw]);
useEffect(() => {
- const clip = rootRef.current?.closest(".timeline-clip");
+ const clip = root?.closest(".timeline-clip");
if (!(clip instanceof HTMLElement)) return;
if (muted) clip.setAttribute("data-audio-muted", "true");
else clip.removeAttribute("data-audio-muted");
return () => clip.removeAttribute("data-audio-muted");
- }, [muted]);
+ }, [muted, root]);
return (
- <div ref={rootRef} className="absolute inset-0">
+ <div ref={setRoot} className="absolute inset-0">
<div
ref={setStripRef}
className="absolute inset-0 overflow-hidden"
diff --git a/packages/studio/src/player/components/VideoThumbnail.test.tsx b/packages/studio/src/player/components/VideoThumbnail.test.tsx
index 3f1801f438..d9103555f4 100644
--- a/packages/studio/src/player/components/VideoThumbnail.test.tsx
+++ b/packages/studio/src/player/components/VideoThumbnail.test.tsx
@@ -30,25 +30,35 @@ afterEach(() => {
// Registered after the reset above, so each test's strip unmounts before the cache is cleared.
const harness = createHappyDomRootHarness();
+function clip(
+ videoSrc = "/api/projects/p/preview/assets/clip.mp4",
+ priority: "visible" | "overscan" = "visible",
+) {
+ return (
+ <VideoThumbnail
+ videoSrc={videoSrc}
+ label=""
+ labelColor="#fff"
+ projectId="p"
+ sessionEpoch={1}
+ priority={priority}
+ />
+ );
+}
+
async function render(width = 0, height = 40) {
Object.defineProperty(host, "clientWidth", { configurable: true, value: width });
Object.defineProperty(host, "clientHeight", { configurable: true, value: height });
const root = harness.mount(host);
await act(async () => {
- root.render(
- <VideoThumbnail
- videoSrc="/api/projects/p/preview/assets/clip.mp4"
- label=""
- labelColor="#fff"
- projectId="p"
- sessionEpoch={1}
- priority="visible"
- />,
- );
+ root.render(clip());
await Promise.resolve();
});
+ return root;
}
+const shownSources = () => [...host.querySelectorAll("img")].map((img) => img.getAttribute("src"));
+
describe("VideoThumbnail", () => {
it("does not acquire a thumbnail lease before the clip is measured", async () => {
vi.mocked(decodeVideoThumbnail).mockResolvedValue({
@@ -71,7 +81,7 @@ describe("VideoThumbnail", () => {
expect(decodeVideoThumbnail).toHaveBeenNthCalledWith(
1,
- expect.objectContaining({ frameCount: 1 }),
+ expect.objectContaining({ frameCount: 1, contentVersion: "project=p&session=1" }),
expect.any(AbortSignal),
);
expect(decodeVideoThumbnail).toHaveBeenCalledWith(
@@ -91,7 +101,8 @@ describe("VideoThumbnail", () => {
await render(300);
const tiles = [...host.querySelectorAll("img")].map((img) => img.getAttribute("src"));
- expect(tiles).toEqual(["blob:0", "blob:2", "blob:4", "blob:5", "blob:7"]);
+ // Seven slices and the end frame: the last tile shows the end.
+ expect(tiles).toEqual(["blob:0", "blob:2", "blob:3", "blob:4", "blob:7"]);
});
describe("on a 10-minute clip at full zoom", () => {
@@ -280,6 +291,140 @@ describe("VideoThumbnail during a zoom", () => {
);
});
+ it("shows the strip it has, not the poster, while a zoom's new width decodes", async () => {
+ vi.mocked(decodeVideoThumbnail).mockImplementation(async ({ frameCount }) =>
+ frameCount === 1
+ ? { value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, weight: 1 }
+ : {
+ value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 },
+ weight: 2,
+ },
+ );
+ await render(440);
+ expect(
+ new Set([...host.querySelectorAll("img")].map((img) => img.getAttribute("src"))),
+ ).toEqual(new Set(["blob:a", "blob:b"]));
+ // The new width's decode never lands during this test.
+ vi.mocked(decodeVideoThumbnail).mockImplementation(() => new Promise(() => {}));
+ await act(async () => {
+ reportResize(880, 40);
+ await Promise.resolve();
+ });
+ await rest();
+ const shown = [...host.querySelectorAll("img")].map((img) => img.getAttribute("src"));
+ expect(shown.length).toBeGreaterThan(0);
+ expect(shown).not.toContain("blob:poster");
+ });
+
+ it("keeps holding the strip when the clip moves between on and off screen mid-decode", async () => {
+ vi.mocked(decodeVideoThumbnail).mockImplementation(async ({ frameCount }) =>
+ frameCount === 1
+ ? { value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, weight: 1 }
+ : { value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 }, weight: 2 },
+ );
+ const root = await render(440);
+ vi.mocked(decodeVideoThumbnail).mockImplementation(() => new Promise(() => {}));
+ await act(async () => {
+ reportResize(880, 40);
+ root.render(clip(undefined, "overscan"));
+ await Promise.resolve();
+ });
+ await rest();
+ expect(new Set(shownSources())).toEqual(new Set(["blob:a", "blob:b"]));
+ });
+
+ it("holds the strip, and decodes only the new width, when the cache is over its budget", async () => {
+ const disposed: number[] = [];
+ vi.mocked(decodeVideoThumbnail).mockImplementation(async ({ frameCount }) =>
+ frameCount === 1
+ ? { value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, weight: 1 }
+ : {
+ value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 },
+ weight: 2,
+ dispose: () => disposed.push(frameCount),
+ },
+ );
+ await render(440);
+ const held = vi.mocked(decodeVideoThumbnail).mock.calls.at(-1)![0].frameCount;
+ // Fill the project's cache past its entry budget, so a strip without a lease is evicted.
+ await act(async () => {
+ for (let i = 0; i < 100; i++) {
+ const lease = thumbnailScheduler.acquire(
+ {
+ key: `filler-${i}`,
+ projectId: "p",
+ sessionEpoch: 1,
+ kind: "image",
+ priority: "visible",
+ load: async () => ({
+ value: { kind: "image", url: `blob:f${i}`, aspect: 1 },
+ weight: 1,
+ }),
+ },
+ () => {},
+ );
+ for (let tick = 0; tick < 5; tick++) await Promise.resolve();
+ lease.release();
+ }
+ });
+ const decodes = vi.mocked(decodeVideoThumbnail).mock.calls.length;
+ vi.mocked(decodeVideoThumbnail).mockImplementation(() => new Promise(() => {}));
+ await act(async () => {
+ reportResize(880, 40);
+ await Promise.resolve();
+ });
+ await rest();
+ const newWidths = vi
+ .mocked(decodeVideoThumbnail)
+ .mock.calls.slice(decodes)
+ .map(([r]) => r.frameCount);
+ expect(newWidths).not.toContain(held);
+ expect(disposed).toEqual([]);
+ expect(new Set(shownSources())).toEqual(new Set(["blob:a", "blob:b"]));
+ });
+
+ it("never holds one clip's strip for another while the other decodes", async () => {
+ vi.mocked(decodeVideoThumbnail).mockImplementation(({ source }) =>
+ source.endsWith("clip.mp4")
+ ? Promise.resolve({
+ value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 },
+ weight: 2,
+ })
+ : new Promise(() => {}),
+ );
+ const root = await render(440);
+ expect(shownSources()).toContain("blob:a");
+ await act(async () => {
+ root.render(clip("/api/projects/p/preview/assets/other.mp4"));
+ await Promise.resolve();
+ });
+ await rest();
+ expect(shownSources()).toEqual([]);
+ });
+
+ it("lets go of the strip it held once the clip narrows to one frame", async () => {
+ vi.mocked(decodeVideoThumbnail).mockImplementation(async ({ frameCount }) =>
+ frameCount === 1
+ ? { value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, weight: 1 }
+ : { value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 }, weight: 2 },
+ );
+ await render(440);
+ await act(async () => {
+ reportResize(50, 40);
+ await Promise.resolve();
+ });
+ await rest();
+ expect(new Set(shownSources())).toEqual(new Set(["blob:poster"]));
+ // Widening again shows the poster until the new width lands, not the strip it once held.
+ vi.mocked(decodeVideoThumbnail).mockImplementation(() => new Promise(() => {}));
+ await act(async () => {
+ reportResize(880, 40);
+ await Promise.resolve();
+ });
+ await rest();
+ expect(new Set(shownSources())).toEqual(new Set(["blob:poster"]));
+ });
+
it("keeps its filmstrip while the zoom resizes it, and decodes the new width at rest", async () => {
await render(440);
vi.mocked(decodeVideoThumbnail).mockClear();
diff --git a/packages/studio/src/player/components/VideoThumbnail.tsx b/packages/studio/src/player/components/VideoThumbnail.tsx
index a5e00b8a4b..dcb0eb776b 100644
--- a/packages/studio/src/player/components/VideoThumbnail.tsx
+++ b/packages/studio/src/player/components/VideoThumbnail.tsx
@@ -1,4 +1,4 @@
-import { memo, useMemo } from "react";
+import { memo, useMemo, useState } from "react";
import { useThumbnailLease } from "../../hooks/useThumbnailLease";
import { useThumbnailStripSize } from "../../hooks/useThumbnailStripSize";
import {
@@ -8,7 +8,11 @@ import {
} from "../lib/thumbnailScheduler";
import { decodeVideoThumbnail } from "../lib/thumbnailVideoDecoder";
import { ThumbnailTiles } from "./ThumbnailTiles";
-import { computeThumbnailStrip, quantizeThumbnailFrameCount } from "./thumbnailUtils";
+import {
+ computeThumbnailStrip,
+ quantizeThumbnailFrameCount,
+ thumbnailFrameForTile,
+} from "./thumbnailUtils";
import { useValueAtRest } from "./timelineMotion";
interface VideoThumbnailProps {
@@ -55,6 +59,7 @@ function createVideoThumbnailRequest(
decodeVideoThumbnail(
{
source: videoSrc,
+ contentVersion: createThumbnailKey({ project: projectId, session: sessionEpoch }),
sourceStart,
sourceRangeDuration: sourceRangeDuration ?? duration,
frameCount,
@@ -68,13 +73,31 @@ function createVideoThumbnailRequest(
function selectThumbnailSnapshot(
poster: ThumbnailSnapshot,
rich: ThumbnailSnapshot,
+ shown: ThumbnailSnapshot,
): ThumbnailSnapshot {
if (rich.status === "ready") return rich;
+ if (shown.status === "ready") return shown;
if (poster.status === "ready") return poster;
if (rich.status === "loading" || poster.status === "loading") return { status: "loading" };
return poster;
}
+type VideoThumbnailRequest = ReturnType<typeof createVideoThumbnailRequest>;
+
+function useVideoThumbnailSnapshot(
+ poster: VideoThumbnailRequest | null,
+ rich: VideoThumbnailRequest | null,
+ media: string,
+): ThumbnailSnapshot {
+ const posterSnapshot = useThumbnailLease(poster);
+ const richSnapshot = useThumbnailLease(rich);
+ const [shown, setShown] = useState({ media, request: rich });
+ const settled = richSnapshot.status === "ready" || rich === null;
+ if (settled && shown.request !== rich) setShown({ media, request: rich });
+ const shownSnapshot = useThumbnailLease(shown.media === media ? shown.request : null);
+ return selectThumbnailSnapshot(posterSnapshot, richSnapshot, shownSnapshot);
+}
+
/** Sparse, bounded video frames supplied by the shared thumbnail scheduler. */
export const VideoThumbnail = memo(function VideoThumbnail({
videoSrc,
@@ -114,9 +137,11 @@ export const VideoThumbnail = memo(function VideoThumbnail({
[requestFrameCount, requestProps],
);
const measured = useValueAtRest(container.width > 0);
- const posterSnapshot = useThumbnailLease(measured ? posterRequest : null);
- const richSnapshot = useThumbnailLease(measured && requestFrameCount > 1 ? richRequest : null);
- const snapshot = selectThumbnailSnapshot(posterSnapshot, richSnapshot);
+ const snapshot = useVideoThumbnailSnapshot(
+ measured ? posterRequest : null,
+ measured && requestFrameCount > 1 ? richRequest : null,
+ posterRequest.key,
+ );
const value = snapshot.status === "ready" ? snapshot.value : null;
const urls =
value?.kind === "filmstrip" ? value.urls : value?.kind === "image" ? [value.url] : [];
@@ -133,7 +158,7 @@ export const VideoThumbnail = memo(function VideoThumbnail({
watchGap={watchGap}
>
{(index) => {
- const src = urls[Math.round((index * (urls.length - 1)) / Math.max(1, frameCount - 1))];
+ const src = urls[thumbnailFrameForTile(index, frameCount, urls.length)];
return (
<div
key={index}
diff --git a/packages/studio/src/player/components/thumbnailUtils.test.ts b/packages/studio/src/player/components/thumbnailUtils.test.ts
index e9fd1e73a4..3c67e344af 100644
--- a/packages/studio/src/player/components/thumbnailUtils.test.ts
+++ b/packages/studio/src/player/components/thumbnailUtils.test.ts
@@ -4,7 +4,9 @@ import {
encodePreviewPath,
resolveMediaPreviewUrl,
quantizeThumbnailFrameCount,
+ thumbnailFrameForTile,
} from "./thumbnailUtils";
+import { MAX_VISIBLE_THUMBNAIL_FRAMES } from "../lib/timelineViewportBudgets";
describe("computeThumbnailStrip", () => {
it("sizes tiles by aspect ratio at the clip height", () => {
@@ -51,15 +53,27 @@ describe("computeThumbnailStrip", () => {
});
});
+describe("thumbnailFrameForTile", () => {
+ it("shows the clip's last frame in the last tile, and the slice under each other tile's centre", () => {
+ // 8 slices and the end frame.
+ expect([0, 1, 2].map((tile) => thumbnailFrameForTile(tile, 3, 9))).toEqual([1, 4, 8]);
+ // 2 slices and the end frame across 4 tiles.
+ expect([0, 1, 2, 3].map((tile) => thumbnailFrameForTile(tile, 4, 3))).toEqual([0, 0, 1, 2]);
+ expect(thumbnailFrameForTile(0, 1, 9)).toBe(4);
+ });
+});
+
describe("quantizeThumbnailFrameCount", () => {
it("uses doubling buckets and never exceeds the 4K geometry ceiling", () => {
expect(quantizeThumbnailFrameCount(5)).toBe(8);
expect(quantizeThumbnailFrameCount(32)).toBe(32);
- expect(quantizeThumbnailFrameCount(34)).toBe(33);
});
- it("caps decode requests at the shared visible-frame budget", () => {
- expect(quantizeThumbnailFrameCount(124)).toBe(33);
+ it("caps decode requests at the largest step within the visible-frame budget", () => {
+ // A step that is not a power of two would share no frames with the step below it.
+ expect(MAX_VISIBLE_THUMBNAIL_FRAMES).toBeGreaterThan(32);
+ expect(quantizeThumbnailFrameCount(34)).toBe(32);
+ expect(quantizeThumbnailFrameCount(124)).toBe(32);
});
});
diff --git a/packages/studio/src/player/components/thumbnailUtils.ts b/packages/studio/src/player/components/thumbnailUtils.ts
index 0a5a335f15..65290e44fe 100644
--- a/packages/studio/src/player/components/thumbnailUtils.ts
+++ b/packages/studio/src/player/components/thumbnailUtils.ts
@@ -11,7 +11,23 @@ export interface ThumbnailStripLayout {
/** Quantize request identities so a pixel-by-pixel resize does not thrash the cache. */
export function quantizeThumbnailFrameCount(frameCount: number): number {
const safeCount = Math.max(1, Number.isFinite(frameCount) ? Math.ceil(frameCount) : 1);
- return Math.min(MAX_VISIBLE_THUMBNAIL_FRAMES, 2 ** Math.ceil(Math.log2(safeCount)));
+ const cap = 2 ** Math.floor(Math.log2(MAX_VISIBLE_THUMBNAIL_FRAMES));
+ return Math.min(cap, 2 ** Math.ceil(Math.log2(safeCount)));
+}
+
+/**
+ * The decoded frame tile `index` of `tileCount` shows, of a strip's `frameCount`: the last tile
+ * shows the clip's last frame; the others the slice (videoThumbnailTimestamps) holding their centre.
+ */
+export function thumbnailFrameForTile(
+ index: number,
+ tileCount: number,
+ frameCount: number,
+): number {
+ if (frameCount < 2) return 0;
+ if (tileCount > 1 && index >= tileCount - 1) return frameCount - 1;
+ const slices = frameCount - 1;
+ return Math.min(slices - 1, Math.floor(((index + 0.5) * slices) / tileCount));
}
/**
diff --git a/packages/studio/src/player/components/timelineMotion.test.ts b/packages/studio/src/player/components/timelineMotion.test.ts
new file mode 100644
index 0000000000..33d4c2ee65
--- /dev/null
+++ b/packages/studio/src/player/components/timelineMotion.test.ts
@@ -0,0 +1,27 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { markTimelineMotion, TIMELINE_REST_MS, whenTimelineIdle } from "./timelineMotion";
+
+afterEach(() => {
+ vi.runOnlyPendingTimers();
+ vi.useRealTimers();
+});
+
+it("waits for a zoom to rest before resolving, and resolves at once when nothing moves", async () => {
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "requestIdleCallback"] });
+ markTimelineMotion();
+ let idle = false;
+ void whenTimelineIdle().then(() => (idle = true));
+ await vi.advanceTimersByTimeAsync(TIMELINE_REST_MS - 50);
+ expect(idle).toBe(false);
+ // Another zoom step pushes the rest back.
+ markTimelineMotion();
+ await vi.advanceTimersByTimeAsync(TIMELINE_REST_MS - 50);
+ expect(idle).toBe(false);
+ await vi.advanceTimersByTimeAsync(TIMELINE_REST_MS);
+ expect(idle).toBe(true);
+
+ let again = false;
+ void whenTimelineIdle().then(() => (again = true));
+ await vi.advanceTimersByTimeAsync(1);
+ expect(again).toBe(true);
+});
diff --git a/packages/studio/src/player/components/timelineMotion.ts b/packages/studio/src/player/components/timelineMotion.ts
index 2fda7a3b52..78ebb29d82 100644
--- a/packages/studio/src/player/components/timelineMotion.ts
+++ b/packages/studio/src/player/components/timelineMotion.ts
@@ -23,6 +23,26 @@ export function markTimelineMotion(): void {
}, TIMELINE_REST_MS);
}
+const onIdle = (callback: () => void) => {
+ if (typeof requestIdleCallback === "function") requestIdleCallback(callback, { timeout: 1000 });
+ else setTimeout(callback, 0);
+};
+
+/** Resolves at an idle moment with no zoom moving the timeline: long work waits there, not mid-zoom. */
+export function whenTimelineIdle(): Promise<void> {
+ return new Promise((resolve) => {
+ const check = () => {
+ if (!moving) return resolve();
+ const stop = subscribeTimelineMotion(() => {
+ if (moving) return;
+ stop();
+ onIdle(check);
+ });
+ };
+ onIdle(check);
+ });
+}
+
export function isTimelineMoving(): boolean {
return moving;
}
diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts
index c08f4687bb..cbac2487a4 100644
--- a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts
+++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts
@@ -39,10 +39,20 @@ function recordDecodes(decoded: number[][]): void {
});
}
+const endOf = (start: number, duration: number) => start + Math.max(0, duration - 0.001);
+
+// Decoded frames are shared per source for the module's life, so each test decodes its own file.
+let sources = 0;
+let source = "";
+
beforeEach(() => {
+ source = `/clip-${++sources}.mp4`;
vi.clearAllMocks();
getKeyPacket.mockImplementation(async () => null);
- vi.spyOn(URL, "createObjectURL").mockReturnValueOnce("blob:one").mockReturnValueOnce("blob:two");
+ vi.spyOn(URL, "createObjectURL")
+ .mockReturnValueOnce("blob:one")
+ .mockReturnValueOnce("blob:two")
+ .mockReturnValueOnce("blob:end");
vi.spyOn(URL, "revokeObjectURL").mockImplementation(() => {});
HTMLCanvasElement.prototype.toBlob = function toBlob(callback) {
callback(new Blob(["frame"], { type: "image/jpeg" }));
@@ -55,9 +65,15 @@ beforeEach(() => {
});
describe("videoThumbnailTimestamps", () => {
- it("uses the midpoint for a poster and sorted sparse points for a strip", () => {
+ it("uses the midpoint for a poster, and each slice's left edge then the end for a strip", () => {
expect(videoThumbnailTimestamps(2, 6, 1)).toEqual([5]);
- expect(videoThumbnailTimestamps(2, 6, 4)).toEqual([2, 4, 6, 8]);
+ expect(videoThumbnailTimestamps(2, 6, 4)).toEqual([2, 3.5, 5, 6.5, endOf(2, 6)]);
+ });
+
+ it("nests each strip in the strip twice as long, and the poster in every strip", () => {
+ const strip = videoThumbnailTimestamps(0, 8, 4);
+ expect(videoThumbnailTimestamps(0, 8, 8).filter((_, i) => i % 2 === 0)).toEqual(strip);
+ expect(strip).toContain(videoThumbnailTimestamps(0, 8, 1)[0]);
});
it("clamps invalid source ranges", () => {
@@ -71,31 +87,31 @@ describe("decodeVideoThumbnail", () => {
const decoded: number[][] = [];
recordDecodes(decoded);
const result = await decodeVideoThumbnail(
- { source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 6, frameCount: 2 },
+ { source, sourceStart: 2, sourceRangeDuration: 6, frameCount: 2 },
new AbortController().signal,
);
- expect(decoded).toEqual([[2, 8]]);
+ expect(decoded).toEqual([[2, 5, endOf(2, 6)]]);
expect(result.value).toEqual({
kind: "filmstrip",
- urls: ["blob:one", "blob:two"],
+ urls: ["blob:one", "blob:two", "blob:end"],
aspect: 9 / 16,
});
result.dispose?.();
result.dispose?.();
- expect(URL.revokeObjectURL).toHaveBeenCalledTimes(2);
+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3);
expect(dispose).toHaveBeenCalledTimes(1);
});
it("decodes each strip frame at its keyframe unless that keyframe is before the clip's range", async () => {
- getKeyPacket.mockImplementation(async (time) => ({ timestamp: Math.floor(time / 4) * 4 }));
+ getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 0.5 }));
const decoded: number[][] = [];
recordDecodes(decoded);
await decodeVideoThumbnail(
- { source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 8, frameCount: 3 },
+ { source, sourceStart: 2, sourceRangeDuration: 8, frameCount: 4 },
new AbortController().signal,
);
- expect(decoded).toEqual([[2, 4, 8]]);
+ expect(decoded).toEqual([[2, 3.5, 5.5, 7.5, endOf(2, 8)]]);
});
it("keeps a slot's own time when its keyframe is more than half a slot earlier", async () => {
@@ -103,14 +119,10 @@ describe("decodeVideoThumbnail", () => {
const decoded: number[][] = [];
recordDecodes(decoded);
await decodeVideoThumbnail(
- { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
- new AbortController().signal,
- );
- await decodeVideoThumbnail(
- { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 },
+ { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 4 },
new AbortController().signal,
);
- expect(decoded).toEqual([[0, 5, 10], [5]]);
+ expect(decoded).toEqual([[0, 2.5, 5, 7.5, endOf(0, 10)]]);
});
it("looks up each keyframe just before decoding it", async () => {
@@ -126,10 +138,11 @@ describe("decodeVideoThumbnail", () => {
}
});
await decodeVideoThumbnail(
- { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
+ { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 2 },
new AbortController().signal,
);
- expect(events).toEqual(["key 0", "frame 0", "key 5", "frame 5", "key 10", "frame 10"]);
+ const end = endOf(0, 10);
+ expect(events).toEqual(["key 0", "frame 0", "key 5", "frame 5", `key ${end}`, `frame ${end}`]);
});
it("ends the decode times without throwing when cancelled during a keyframe lookup", async () => {
@@ -146,7 +159,7 @@ describe("decodeVideoThumbnail", () => {
});
await expect(
decodeVideoThumbnail(
- { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
+ { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },
controller.signal,
),
).rejects.toThrow("Aborted");
@@ -164,10 +177,198 @@ describe("decodeVideoThumbnail", () => {
const decoded: number[][] = [];
recordDecodes(decoded);
await decodeVideoThumbnail(
- { source: "/clip.mp4", sourceStart: 5, sourceRangeDuration: 5, frameCount: 3 },
+ { source, sourceStart: 5, sourceRangeDuration: 5, frameCount: 2 },
new AbortController().signal,
);
- expect(decoded).toEqual([[5, 7.5, 10]]);
+ expect(decoded).toEqual([[5, 7.5, endOf(5, 5)]]);
+ });
+
+ it("reuses the frames a strip of the same source shows, so a zoom decodes only new times", async () => {
+ vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`);
+ const decoded: number[][] = [];
+ recordDecodes(decoded);
+ const range = { source, sourceStart: 0, sourceRangeDuration: 8 };
+ const signal = new AbortController().signal;
+ const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal);
+ // Twice the frames decodes only the new half; half the frames decodes nothing. All share the end.
+ const eight = await decodeVideoThumbnail({ ...range, frameCount: 8 }, signal);
+ const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);
+ expect(decoded).toEqual([
+ [0, 2, 4, 6, endOf(0, 8)],
+ [1, 3, 5, 7],
+ ]);
+ const urls = (result: typeof four) =>
+ result.value.kind === "filmstrip" ? result.value.urls : [];
+ expect(urls(eight).filter((_, i) => i % 2 === 0)).toEqual(urls(four));
+ expect(urls(two)).toEqual([urls(four)[0], urls(four)[2], urls(four)[4]]);
+ for (const result of [four, eight, two]) result.dispose?.();
+ });
+
+ it("decodes a frame again when the one shared for its time sits too far before it", async () => {
+ // Every keyframe is 2 s before the time asked: a 2-frame strip of 8 s accepts that (its
+ // frames stand for 4 s each), a 4-frame strip does not, or its frames would run backwards.
+ getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 2 }));
+ const decoded: number[][] = [];
+ recordDecodes(decoded);
+ const range = { source, sourceStart: 0, sourceRangeDuration: 8 };
+ const signal = new AbortController().signal;
+ const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);
+ const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal);
+ expect(decoded).toEqual([
+ [0, 2, endOf(0, 8)],
+ [2, 4, 6],
+ ]);
+ two.dispose?.();
+ four.dispose?.();
+ });
+
+ it("returns a strip whose frames are all shared without opening the file again", async () => {
+ recordDecodes([]);
+ const range = { source, sourceStart: 0, sourceRangeDuration: 8 };
+ const signal = new AbortController().signal;
+ const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal);
+ const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);
+ expect(input.getPrimaryVideoTrack).toHaveBeenCalledTimes(1);
+ four.dispose?.();
+ two.dispose?.();
+ });
+
+ it("reopens the same asset after a project round trip and keeps new-session frames separate", async () => {
+ let frame = 0;
+ vi.mocked(URL.createObjectURL)
+ .mockReset()
+ .mockImplementation(() => `blob:version-${++frame}`);
+ const decoded: number[][] = [];
+ recordDecodes(decoded);
+ const signal = new AbortController().signal;
+ const original = await decodeVideoThumbnail(
+ { source, contentVersion: "A:1", frameCount: 1 },
+ signal,
+ );
+ const other = await decodeVideoThumbnail(
+ { source: `${source}?B`, contentVersion: "B:2", frameCount: 1 },
+ signal,
+ );
+ input.getPrimaryVideoTrack.mockResolvedValue({
+ getDisplayWidth: vi.fn(async () => 1280),
+ getDisplayHeight: vi.fn(async () => 360),
+ getDurationFromMetadata: vi.fn(async () => 10),
+ });
+ const reopened = await decodeVideoThumbnail(
+ { source, contentVersion: "A:3", frameCount: 1 },
+ signal,
+ );
+ const shared = await decodeVideoThumbnail(
+ { source, contentVersion: "A:3", frameCount: 1 },
+ signal,
+ );
+ expect(reopened.value).toEqual({ kind: "image", url: "blob:version-3", aspect: 32 / 9 });
+ expect(shared.value).toEqual(reopened.value);
+ expect(decoded).toEqual([[5], [5], [5]]);
+ expect(input.getPrimaryVideoTrack).toHaveBeenCalledTimes(3);
+ original.dispose?.();
+ expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:version-1");
+ expect(URL.revokeObjectURL).not.toHaveBeenCalledWith("blob:version-3");
+ other.dispose?.();
+ reopened.dispose?.();
+ expect(URL.revokeObjectURL).not.toHaveBeenCalledWith("blob:version-3");
+ shared.dispose?.();
+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3);
+ });
+
+ it("keeps one copy of a frame two strips decode at once, and revokes the other", async () => {
+ vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`);
+ // The first strip holds its first frame until the second is decoding too.
+ let release!: () => void;
+ const bothDecoding = new Promise<void>((resolve) => (release = resolve));
+ canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {
+ const first = canvasesAtTimestamps.mock.calls.length === 1;
+ for await (const _time of timestamps) {
+ if (first) await bothDecoding;
+ else release();
+ yield { canvas: document.createElement("canvas") };
+ }
+ });
+ const range = { source, sourceStart: 0, sourceRangeDuration: 8 };
+ const signal = new AbortController().signal;
+ const decodingTwo = decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);
+ await vi.waitFor(() => expect(canvasesAtTimestamps).toHaveBeenCalledOnce());
+ const [two, four] = await Promise.all([
+ decodingTwo,
+ decodeVideoThumbnail({ ...range, frameCount: 4 }, signal),
+ ]);
+ // Both decoded the frames at 0 s, 4 s and the end; the second copy of each goes at once.
+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3);
+ two.dispose?.();
+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3);
+ four.dispose?.();
+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(8);
+ });
+
+ it("fills a frame that failed to decode from the one before it, so tiles stay on their times", async () => {
+ canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {
+ let slot = 0;
+ for await (const _time of timestamps) {
+ yield slot++ === 2 ? null : { canvas: document.createElement("canvas") };
+ }
+ });
+ const signal = new AbortController().signal;
+ const result = await decodeVideoThumbnail(
+ { source, sourceStart: 0, sourceRangeDuration: 8, frameCount: 2 },
+ signal,
+ );
+ expect(result.value).toMatchObject({
+ kind: "filmstrip",
+ urls: ["blob:one", "blob:two", "blob:two"],
+ });
+ // The cache is charged for the two decoded frames, not the filled copy.
+ recordDecodes([]);
+ const whole = await decodeVideoThumbnail(
+ { source: `${source}?whole`, sourceStart: 0, sourceRangeDuration: 8, frameCount: 2 },
+ signal,
+ );
+ expect(result.weight * 3).toBe(whole.weight * 2);
+ result.dispose?.();
+ whole.dispose?.();
+ });
+
+ it("snaps a poster up to a quarter of its range and a strip frame up to half a slice", async () => {
+ const decoded: number[][] = [];
+ recordDecodes(decoded);
+ const signal = new AbortController().signal;
+ getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 2.4 }));
+ await decodeVideoThumbnail(
+ { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 },
+ signal,
+ );
+ getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 2.6 }));
+ await decodeVideoThumbnail(
+ { source: `${source}?b`, sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 },
+ signal,
+ );
+ // Four slices of 8 s: a keyframe 0.9 s early snaps, one 1.1 s early does not.
+ getKeyPacket.mockImplementation(async (time) => ({
+ timestamp: time - (time === 4 ? 0.9 : 1.1),
+ }));
+ await decodeVideoThumbnail(
+ { source: `${source}?c`, sourceStart: 0, sourceRangeDuration: 8, frameCount: 4 },
+ signal,
+ );
+ expect(decoded).toEqual([[5 - 2.4], [5], [0, 2, 4 - 0.9, 6, endOf(0, 8)]]);
+ });
+
+ it("revokes a shared frame only once no strip shows it", async () => {
+ vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`);
+ recordDecodes([]);
+ const range = { source, sourceStart: 0, sourceRangeDuration: 8 };
+ const signal = new AbortController().signal;
+ const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal);
+ const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);
+ four.dispose?.();
+ // The three frames the 2-slice strip still shows stay; the other two go.
+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(2);
+ two.dispose?.();
+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(5);
});
it("releases input and degrades when the source has no video track", async () => {
@@ -181,13 +382,15 @@ describe("decodeVideoThumbnail", () => {
it("revokes partial results when cancellation lands during extraction", async () => {
const controller = new AbortController();
const canvas = document.createElement("canvas");
- canvasesAtTimestamps.mockImplementation(async function* () {
- yield { canvas, timestamp: 1, duration: 1 };
- controller.abort();
- yield { canvas, timestamp: 2, duration: 1 };
+ canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {
+ let frames = 0;
+ for await (const timestamp of timestamps) {
+ if (frames++ === 1) controller.abort();
+ yield { canvas, timestamp, duration: 1 };
+ }
});
await expect(
- decodeVideoThumbnail({ source: "/clip.mp4", frameCount: 2 }, controller.signal),
+ decodeVideoThumbnail({ source, frameCount: 2 }, controller.signal),
).rejects.toMatchObject({ name: "AbortError" });
expect(URL.revokeObjectURL).toHaveBeenCalledTimes(1);
expect(dispose).toHaveBeenCalledTimes(1);
@@ -207,10 +410,7 @@ describe("decodeVideoThumbnail", () => {
getDurationFromMetadata,
});
- const decoding = decodeVideoThumbnail(
- { source: "/clip.mp4", frameCount: 1 },
- controller.signal,
- );
+ const decoding = decodeVideoThumbnail({ source, frameCount: 1 }, controller.signal);
await vi.waitFor(() => expect(getDisplayWidth).toHaveBeenCalledOnce());
controller.abort();
resolveWidth(1080);
diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts
index b892c6cc94..0bf417dd56 100644
--- a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts
+++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts
@@ -1,14 +1,21 @@
import { TIMELINE_VIEWPORT_BUDGETS, type TimelineViewportBudgets } from "./timelineViewportBudgets";
-import type { ThumbnailLoadedResult, ThumbnailValue } from "./thumbnailScheduler";
+import {
+ createThumbnailKey,
+ type ThumbnailLoadedResult,
+ type ThumbnailValue,
+} from "./thumbnailScheduler";
export interface VideoThumbnailDecodeRequest {
source: string;
+ contentVersion?: string;
sourceStart?: number;
sourceRangeDuration?: number;
frameCount: number;
fit?: "contain" | "cover";
}
+const LAST_FRAME_INSET_S = 0.001;
+
export function videoThumbnailTimestamps(
start: number,
duration: number,
@@ -18,10 +25,63 @@ export function videoThumbnailTimestamps(
const safeDuration = Math.max(0, Number.isFinite(duration) ? duration : 0);
const count = Math.max(1, Number.isFinite(frameCount) ? Math.floor(frameCount) : 1);
if (count === 1) return [safeStart + safeDuration / 2];
- return Array.from(
+ const edges = Array.from(
{ length: count },
- (_, index) => safeStart + (safeDuration * index) / (count - 1),
+ (_, index) => safeStart + (safeDuration * index) / count,
);
+ return [...edges, safeStart + Math.max(0, safeDuration - LAST_FRAME_INSET_S)];
+}
+
+interface SharedFrame {
+ url: string;
+ decodedAt: number;
+ users: number;
+}
+
+interface HeldFrame {
+ key: string;
+ frame: SharedFrame;
+}
+
+interface FrameWindow {
+ sourceStart: number;
+ lead: number;
+ end: number;
+}
+
+const sharedFrames = new Map<string, SharedFrame[]>();
+const sourceInfos = new Map<string, { version: string | undefined; info: SourceInfo }>();
+
+const accepts = (window: FrameWindow, time: number, decodedAt: number) =>
+ decodedAt >= window.sourceStart && time - decodedAt <= (time >= window.end ? 0 : window.lead);
+
+function takeSharedFrame(key: string, time: number, window: FrameWindow): SharedFrame | undefined {
+ const frame = sharedFrames.get(key)?.find((shared) => accepts(window, time, shared.decodedAt));
+ if (frame) frame.users += 1;
+ return frame;
+}
+
+function shareFrame(key: string, decodedAt: number, url: string): SharedFrame {
+ const frames = sharedFrames.get(key) ?? [];
+ const existing = frames.find((shared) => shared.decodedAt === decodedAt);
+ if (existing) {
+ existing.users += 1;
+ URL.revokeObjectURL(url);
+ return existing;
+ }
+ const frame = { url, decodedAt, users: 1 };
+ sharedFrames.set(key, [...frames, frame]);
+ return frame;
+}
+
+function releaseSharedFrames(held: HeldFrame[]): void {
+ for (const { key, frame } of held.splice(0)) {
+ if (--frame.users > 0) continue;
+ const rest = sharedFrames.get(key)?.filter((shared) => shared !== frame) ?? [];
+ if (rest.length > 0) sharedFrames.set(key, rest);
+ else sharedFrames.delete(key);
+ URL.revokeObjectURL(frame.url);
+ }
}
async function canvasToBlob(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<Blob> {
@@ -38,7 +98,8 @@ async function canvasToBlob(canvas: HTMLCanvasElement | OffscreenCanvas): Promis
}
interface DecodedResources {
- urls: string[];
+ urls: (string | undefined)[];
+ held: HeldFrame[];
canvases: Set<HTMLCanvasElement | OffscreenCanvas>;
}
@@ -48,12 +109,18 @@ interface ThumbnailCanvasSink {
): AsyncIterable<{ canvas: HTMLCanvasElement | OffscreenCanvas } | null>;
}
+interface SourceInfo {
+ aspect: number;
+ metadataDuration: number | null;
+}
+
function throwIfAborted(signal: AbortSignal): void {
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
}
function releaseDecodedResources(resources: DecodedResources): void {
- for (const url of resources.urls.splice(0)) URL.revokeObjectURL(url);
+ resources.urls.length = 0;
+ releaseSharedFrames(resources.held);
for (const canvas of resources.canvases) {
canvas.width = 0;
canvas.height = 0;
@@ -75,19 +142,71 @@ function targetDimensions(
};
}
+function stripTimes(
+ request: VideoThumbnailDecodeRequest,
+ info: SourceInfo,
+ budgets: Readonly<TimelineViewportBudgets>,
+) {
+ const requestedStart = Math.max(0, request.sourceStart ?? 0);
+ const sourceDuration = Math.max(
+ 0,
+ info.metadataDuration ?? requestedStart + (request.sourceRangeDuration ?? 0),
+ );
+ const sourceStart = Math.min(requestedStart, sourceDuration);
+ const requestedDuration =
+ request.sourceRangeDuration ?? Math.max(0, sourceDuration - sourceStart);
+ const duration = Math.min(
+ Math.max(0, requestedDuration),
+ Math.max(0, sourceDuration - sourceStart),
+ );
+ const timestamps = videoThumbnailTimestamps(
+ sourceStart,
+ duration,
+ Math.min(request.frameCount, budgets.richPreviewFrameCount),
+ );
+ const slices = Math.max(1, timestamps.length - 1);
+ const window: FrameWindow = {
+ sourceStart,
+ lead: duration / Math.max(2, slices) / 2,
+ end: timestamps.length > 1 ? timestamps.at(-1)! : Infinity,
+ };
+ return { window, timestamps };
+}
+
+function takeDecodedFrames(
+ { timestamps, window }: ReturnType<typeof stripTimes>,
+ keyOf: (time: number) => string,
+ resources: DecodedResources,
+): number[] {
+ const missing: number[] = [];
+ timestamps.forEach((time, slot) => {
+ const key = keyOf(time);
+ const frame = takeSharedFrame(key, time, window);
+ if (!frame) return void missing.push(slot);
+ resources.urls[slot] = frame.url;
+ resources.held.push({ key, frame });
+ });
+ return missing;
+}
+
async function decodeFrames(
sink: ThumbnailCanvasSink,
- timestamps: AsyncIterable<number>,
+ times: AsyncIterable<number>,
+ slots: { slot: number; key: string; decodedAt?: number }[],
signal: AbortSignal,
resources: DecodedResources,
): Promise<void> {
- for await (const wrapped of sink.canvasesAtTimestamps(timestamps)) {
+ let next = 0;
+ for await (const wrapped of sink.canvasesAtTimestamps(times)) {
throwIfAborted(signal);
- if (!wrapped) continue;
+ const target = slots[next++];
+ if (!wrapped || target?.decodedAt === undefined) continue;
resources.canvases.add(wrapped.canvas);
const blob = await canvasToBlob(wrapped.canvas);
throwIfAborted(signal);
- resources.urls.push(URL.createObjectURL(blob));
+ const frame = shareFrame(target.key, target.decodedAt, URL.createObjectURL(blob));
+ resources.urls[target.slot] = frame.url;
+ resources.held.push({ key: target.key, frame });
}
throwIfAborted(signal);
}
@@ -95,18 +214,25 @@ async function decodeFrames(
function loadedResult(
resources: DecodedResources,
aspect: number,
- width: number,
- height: number,
+ budgets: Readonly<TimelineViewportBudgets>,
+ frameCount = resources.urls.length,
): ThumbnailLoadedResult {
- const firstUrl = resources.urls[0];
+ const decoded = resources.urls.filter((url): url is string => url !== undefined);
+ const firstUrl = decoded[0];
if (!firstUrl) throw new Error("Video source returned no thumbnail frames");
+ let nearest = firstUrl;
+ const urls = Array.from(
+ { length: frameCount },
+ (_, slot) => (nearest = resources.urls[slot] ?? nearest),
+ );
+ const { width, height } = targetDimensions(aspect, budgets);
const value: ThumbnailValue =
- resources.urls.length === 1
+ urls.length === 1
? { kind: "image", url: firstUrl, aspect }
- : { kind: "filmstrip", urls: [...resources.urls], aspect };
+ : { kind: "filmstrip", urls, aspect };
return {
value,
- weight: width * height * 4 * resources.urls.length,
+ weight: width * height * 4 * decoded.length,
dispose: () => releaseDecodedResources(resources),
};
}
@@ -116,6 +242,34 @@ export async function decodeVideoThumbnail(
request: VideoThumbnailDecodeRequest,
signal: AbortSignal,
budgets: Readonly<TimelineViewportBudgets> = TIMELINE_VIEWPORT_BUDGETS,
+): Promise<ThumbnailLoadedResult> {
+ const fit = request.fit ?? "cover";
+ const sourceKey = createThumbnailKey({ source: request.source, version: request.contentVersion });
+ const keyOf = (time: number) => `${sourceKey}\u0000${fit}\u0000${time}`;
+ const resources: DecodedResources = { urls: [], held: [], canvases: new Set() };
+ const entry = sourceInfos.get(request.source);
+ const known = entry?.version === request.contentVersion ? entry?.info : undefined;
+ try {
+ if (known) {
+ if (takeDecodedFrames(stripTimes(request, known, budgets), keyOf, resources).length === 0) {
+ throwIfAborted(signal);
+ return loadedResult(resources, known.aspect, budgets);
+ }
+ releaseDecodedResources(resources);
+ }
+ return await decodeMissingFrames(request, signal, budgets, keyOf, resources);
+ } catch (error) {
+ releaseDecodedResources(resources);
+ throw error;
+ }
+}
+
+async function decodeMissingFrames(
+ request: VideoThumbnailDecodeRequest,
+ signal: AbortSignal,
+ budgets: Readonly<TimelineViewportBudgets>,
+ keyOf: (time: number) => string,
+ resources: DecodedResources,
): Promise<ThumbnailLoadedResult> {
const mediabunny = await import("mediabunny");
throwIfAborted(signal);
@@ -124,7 +278,6 @@ export async function decodeVideoThumbnail(
source: new mediabunny.UrlSource(request.source),
formats: mediabunny.ALL_FORMATS,
});
- const resources: DecodedResources = { urls: [], canvases: new Set() };
try {
const track = await input.getPrimaryVideoTrack();
throwIfAborted(signal);
@@ -139,46 +292,36 @@ export async function decodeVideoThumbnail(
}
const metadataDuration = await track.getDurationFromMetadata({ skipLiveWait: true });
throwIfAborted(signal);
- const requestedStart = Math.max(0, request.sourceStart ?? 0);
- const sourceDuration = Math.max(
- 0,
- metadataDuration ?? requestedStart + (request.sourceRangeDuration ?? 0),
- );
- const sourceStart = Math.min(requestedStart, sourceDuration);
- const requestedDuration =
- request.sourceRangeDuration ?? Math.max(0, sourceDuration - sourceStart);
- const duration = Math.min(
- Math.max(0, requestedDuration),
- Math.max(0, sourceDuration - sourceStart),
- );
- const timestamps = videoThumbnailTimestamps(
- sourceStart,
- duration,
- Math.min(request.frameCount, budgets.richPreviewFrameCount),
- );
- const keys = new mediabunny.EncodedPacketSink(track);
- const maxKeyframeLead = duration / Math.max(2, timestamps.length - 1) / 2;
- async function* decodeTimesAtNearbyKeyframes() {
- for (const time of timestamps) {
- const key = await keys.getKeyPacket(time, { metadataOnly: true });
- if (signal.aborted) return;
- const near = key && key.timestamp >= sourceStart && time - key.timestamp <= maxKeyframeLead;
- yield near ? key.timestamp : time;
+ const info: SourceInfo = { aspect: displayWidth / displayHeight, metadataDuration };
+ sourceInfos.set(request.source, { version: request.contentVersion, info });
+ const strip = stripTimes(request, info, budgets);
+ const { timestamps, window } = strip;
+ const slots: { slot: number; key: string; decodedAt?: number }[] = takeDecodedFrames(
+ strip,
+ keyOf,
+ resources,
+ ).map((slot) => ({ slot, key: keyOf(timestamps[slot]!) }));
+ if (slots.length > 0) {
+ const keys = new mediabunny.EncodedPacketSink(track);
+ async function* decodeTimesAtNearbyKeyframes() {
+ for (const target of slots) {
+ const time = timestamps[target.slot]!;
+ const key = await keys.getKeyPacket(time, { metadataOnly: true });
+ if (signal.aborted) return;
+ target.decodedAt = key && accepts(window, time, key.timestamp) ? key.timestamp : time;
+ yield target.decodedAt;
+ }
}
+ const target = targetDimensions(info.aspect, budgets);
+ const sink = new mediabunny.CanvasSink(track, {
+ width: target.width,
+ height: target.height,
+ fit: request.fit ?? "cover",
+ poolSize: 1,
+ });
+ await decodeFrames(sink, decodeTimesAtNearbyKeyframes(), slots, signal, resources);
}
- const aspect = displayWidth / displayHeight;
- const target = targetDimensions(aspect, budgets);
- const sink = new mediabunny.CanvasSink(track, {
- width: target.width,
- height: target.height,
- fit: request.fit ?? "cover",
- poolSize: 1,
- });
- await decodeFrames(sink, decodeTimesAtNearbyKeyframes(), signal, resources);
- return loadedResult(resources, aspect, target.width, target.height);
- } catch (error) {
- releaseDecodedResources(resources);
- throw error;
+ return loadedResult(resources, info.aspect, budgets, timestamps.length);
} finally {
input.dispose();
}