Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
perf(studio): zooming a long timeline re-decodes and redraws less (#5142)
Miguel Ángel9f09861d2e78
diff --git a/packages/core/src/beats/beatDetection.test.ts b/packages/core/src/beats/beatDetection.test.tsindex 31bd167535..f0a6ec72fc 100644--- a/packages/core/src/beats/beatDetection.test.ts+++ b/packages/core/src/beats/beatDetection.test.ts@@ -1,6 +1,6 @@-import { describe, expect, it } from "vitest";+import { describe, expect, it, vi } from "vitest";-import { loadBpmDetective } from "./beatDetection";+import { analyzeMusicFromBuffer, loadBpmDetective } from "./beatDetection";describe("loadBpmDetective", () => {it("resolves to null when the import is unavailable", async () => {@@ -39,3 +39,72 @@ describe("loadBpmDetective", () => {expect(detect).toBe(fn);});});++describe("analyzeMusicFromBuffer", () => {+ it("waits on the caller's pause before each long stage", async () => {+ const sampleRate = 8000;+ const channel = new Float32Array(sampleRate * 2);+ for (let beat = 0; beat < 4; beat++) channel.fill(0.9, beat * 4000, beat * 4000 + 80);+ const audio = {+ getChannelData: () => channel,+ sampleRate,+ duration: 2,+ } as unknown as AudioBuffer;+ const held: (() => void)[] = [];+ const pause = () => new Promise<void>((resolve) => held.push(resolve));+ let done = false;+ const analysis = analyzeMusicFromBuffer(audio, { pause }).then((result) => {+ done = true;+ return result;+ });+ for (let stage = 1; stage <= 3; stage++) {+ await vi.waitFor(() => expect(held).toHaveLength(stage));+ expect(done).toBe(false);+ held[stage - 1]!();+ }+ await expect(analysis).resolves.toMatchObject({ sampleRate });+ });++ it("rechecks pause after a slow tempo-detector import", async () => {+ let releaseImport!: () => void;+ const importGate = new Promise<void>((resolve) => (releaseImport = resolve));+ let importStarted = false;+ let moving = false;+ const movingAtDetect: boolean[] = [];+ const detect = vi.fn(() => {+ movingAtDetect.push(moving);+ return 120;+ });+ vi.resetModules();+ vi.doMock("bpm-detective", async () => {+ importStarted = true;+ await importGate;+ return { default: detect };+ });+ try {+ const { analyzeMusicFromBuffer: analyze } = await import("./beatDetection");+ const sampleRate = 8000;+ const audio = {+ getChannelData: () => new Float32Array(sampleRate * 2),+ sampleRate,+ duration: 2,+ } as unknown as AudioBuffer;+ const waiters: (() => void)[] = [];+ const pause = () =>+ moving ? new Promise<void>((resolve) => waiters.push(resolve)) : Promise.resolve();+ const analysis = analyze(audio, { pause });+ await vi.waitFor(() => expect(importStarted).toBe(true));+ moving = true;+ releaseImport();+ await vi.waitFor(() => expect(waiters).toHaveLength(1));+ expect(detect).not.toHaveBeenCalled();+ moving = false;+ waiters.shift()!();+ await analysis;+ expect(movingAtDetect).toEqual([false]);+ } finally {+ vi.doUnmock("bpm-detective");+ vi.resetModules();+ }+ });+});diff --git a/packages/core/src/beats/beatDetection.ts b/packages/core/src/beats/beatDetection.tsindex 59bc109f6b..db2eb2399a 100644--- a/packages/core/src/beats/beatDetection.ts+++ b/packages/core/src/beats/beatDetection.ts@@ -212,18 +212,27 @@ function gateBeatsBySilence(return { times, strengths, peak };}+export interface MusicAnalysisOptions {+ pause?: () => Promise<void>;+}+// fallow-ignore-next-line complexity-export async function analyzeMusicFromBuffer(audioBuffer: AudioBuffer): Promise<MusicBeatAnalysis> {+export async function analyzeMusicFromBuffer(+ audioBuffer: AudioBuffer,+ { pause }: MusicAnalysisOptions = {},+): Promise<MusicBeatAnalysis> {const channelData = audioBuffer.getChannelData(0);const sampleRate = audioBuffer.sampleRate;const duration = audioBuffer.duration;+ await pause?.();const rawBeats = await detectBeats(audioBuffer);const onsetBpm = computeBpmFromBeats(rawBeats);let detectiveBpm: number | null = null;+ const detect = await loadBpmDetective();+ await pause?.();try {- const detect = await loadBpmDetective();if (detect) detectiveBpm = detect(audioBuffer);} catch {// Not enough peaks or browser context unavailable@@ -261,6 +270,7 @@ export async function analyzeMusicFromBuffer(audioBuffer: AudioBuffer): Promise<regularizeBpm = detectiveBpm;}+ await pause?.();const gridBeats =regularizeBpm !== null ? regularizeBeats(rawBeats, regularizeBpm, duration) : rawBeats;const gated = gateBeatsBySilence(gridBeats, channelData, sampleRate);@@ -288,13 +298,16 @@ export async function detectBeatsFromUrl(url: string): Promise<number[]> {}}-export async function analyzeMusicFromUrl(url: string): Promise<MusicBeatAnalysis> {+export async function analyzeMusicFromUrl(+ url: string,+ options: MusicAnalysisOptions = {},+): Promise<MusicBeatAnalysis> {const audioContext = new AudioContext();try {const response = await fetch(url);const arrayBuffer = await response.arrayBuffer();const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);- return analyzeMusicFromBuffer(audioBuffer);+ return analyzeMusicFromBuffer(audioBuffer, options);} finally {await audioContext.close();}diff --git a/packages/studio/src/hooks/useMusicBeatAnalysis.ts b/packages/studio/src/hooks/useMusicBeatAnalysis.tsindex 8d0f2dc585..e71339ba99 100644--- a/packages/studio/src/hooks/useMusicBeatAnalysis.ts+++ b/packages/studio/src/hooks/useMusicBeatAnalysis.ts@@ -2,6 +2,7 @@ import { useEffect, useMemo, useRef } from "react";import { usePlayerStore } from "../player/store/playerStore";import { resolveBeatSourceTrack } from "../utils/timelineInspector";import { analyzeMusicFromUrl } from "@hyperframes/core/beats";+import { whenTimelineIdle } from "../player/components/timelineMotion";import { useFileManagerContextOptional } from "../contexts/FileManagerContext";import { mergeUserBeats } from "../utils/beatEditing";import { getTimelineElementIndexes } from "../player/lib/timelineElementIndexes";@@ -79,7 +80,7 @@ async function loadBeatAnalysis(): Promise<{ analysis: MusicAnalysis; times: number[]; strengths: number[] } | null> {let promise = analysisCache.get(musicSrc);if (!promise) {- promise = analyzeMusicFromUrl(musicSrc);+ promise = analyzeMusicFromUrl(musicSrc, { pause: whenTimelineIdle });cacheAnalysis(musicSrc, promise);}try {diff --git a/packages/studio/src/player/components/AudioWaveform.test.tsx b/packages/studio/src/player/components/AudioWaveform.test.tsxindex 1652e04b9a..20819a8c97 100644--- a/packages/studio/src/player/components/AudioWaveform.test.tsx+++ b/packages/studio/src/player/components/AudioWaveform.test.tsx@@ -11,6 +11,29 @@ vi.mock("../../hooks/useThumbnailLease", () => ({useThumbnailLease: leaseSpy,}));+// One observer serves every waveform; it reports each one near as it is watched, unless a test+// reports by hand.+const nearScreen = {+ auto: true,+ watched: [] as Element[],+ report: (_target: Element, _near: boolean) => {},+};+globalThis.IntersectionObserver = class {+ constructor(callback: IntersectionObserverCallback) {+ nearScreen.report = (target, near) =>+ callback(+ [{ target, isIntersecting: near } as IntersectionObserverEntry],+ this as unknown as IntersectionObserver,+ );+ }+ observe(target: Element) {+ nearScreen.watched.push(target);+ if (nearScreen.auto) nearScreen.report(target, true);+ }+ unobserve() {}+ disconnect() {}+} as unknown as typeof IntersectionObserver;+const EMPTY_STRIP = { width: 0, height: 0, inViewStart: 0, inViewEnd: 0 };const watchGap = vi.hoisted(() => vi.fn());const strip = vi.hoisted(() => ({@@ -52,6 +75,7 @@ afterEach(() => {leaseSpy.mockReset();leaseSpy.mockImplementation(() => ({ status: "loading" as const }));strip.size = EMPTY_STRIP;+ nearScreen.auto = true;watchGap.mockClear();document.body.innerHTML = "";});@@ -141,6 +165,23 @@ describe("AudioWaveform", () => {expect(watchGap.mock.calls.filter(([gap]) => gap)).toEqual([]);});+ it("leaves a waveform off screen undrawn through a zoom, and draws it once it comes near", () => {+ nearScreen.auto = false;+ const getContext = vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null);+ setStrip({ width: 1000, height: 40, inViewStart: 0, inViewEnd: 1000 });+ renderReadyWaveform();+ const root = nearScreen.watched.at(-1)!;+ expect(getContext).not.toHaveBeenCalled();+ act(() => nearScreen.report(root, true));+ const drawn = getContext.mock.calls.length;+ expect(drawn).toBeGreaterThan(0);+ act(() => nearScreen.report(root, false));+ setStrip({ width: 2000, height: 40, inViewStart: 0, inViewEnd: 2000 });+ expect(getContext.mock.calls.length).toBe(drawn);+ act(() => nearScreen.report(root, true));+ expect(getContext.mock.calls.length).toBe(drawn + 1);+ });+it("redraws a short clip, drawn whole, when a zoom changes its width", () => {const getContext = vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null);setStrip({ width: 1000, height: 40, inViewStart: 0, inViewEnd: 1000 });diff --git a/packages/studio/src/player/components/AudioWaveform.tsx b/packages/studio/src/player/components/AudioWaveform.tsxindex d7e226ba1b..ef5b2f5514 100644--- a/packages/studio/src/player/components/AudioWaveform.tsx+++ b/packages/studio/src/player/components/AudioWaveform.tsx@@ -1,4 +1,4 @@-import { memo, useCallback, useContext, useEffect, useMemo, useRef } from "react";+import { memo, useCallback, useContext, useEffect, useMemo, useRef, useState } from "react";import { fadeGain } from "@hyperframes/core/audio-fade";import { useThumbnailLease } from "../../hooks/useThumbnailLease";import { useThumbnailStripSize } from "../../hooks/useThumbnailStripSize";@@ -158,6 +158,32 @@ async function decodeWaveformPeaks(url: string, signal: AbortSignal): Promise<nu}/** Bounded waveform subscriber; cache, cancellation and dedupe live in one scheduler. */+const NEAR_SCREEN: IntersectionObserverInit & { scrollMargin: string } = {+ rootMargin: "50% 100%",+ scrollMargin: "50% 100%",+};+const nearScreenListeners = new Map<Element, (near: boolean) => void>();+let nearScreen: IntersectionObserver | null = null;++function useNearScreen(element: Element | null): boolean {+ const [near, setNear] = useState(() => typeof IntersectionObserver === "undefined");+ useEffect(() => {+ if (!element || typeof IntersectionObserver === "undefined") return;+ nearScreen ??= new IntersectionObserver(+ (entries) =>+ entries.forEach((entry) => nearScreenListeners.get(entry.target)?.(entry.isIntersecting)),+ NEAR_SCREEN,+ );+ nearScreenListeners.set(element, setNear);+ nearScreen.observe(element);+ return () => {+ nearScreenListeners.delete(element);+ nearScreen?.unobserve(element);+ };+ }, [element]);+ return near;+}+export const AudioWaveform = memo(function AudioWaveform({audioUrl,waveformUrl,@@ -171,7 +197,7 @@ export const AudioWaveform = memo(function AudioWaveform({muted = false,labelInset = 16,}: AudioWaveformProps) {- const rootRef = useRef<HTMLDivElement | null>(null);+ const [root, setRoot] = useState<HTMLDivElement | null>(null);const canvasRef = useRef<HTMLCanvasElement | null>(null);const cacheKey = waveformUrl ?? audioUrl;const request = useMemo(@@ -218,7 +244,10 @@ export const AudioWaveform = memo(function AudioWaveform({span,);}, [fades, from, muted, peaks, to, trimEndFraction, trimStartFraction]);- useEffect(draw, [draw, strip.width, strip.height]);+ const near = useNearScreen(root);+ useEffect(() => {+ if (near) draw();+ }, [draw, near, strip.width, strip.height]);useEffect(() => {const root = document.documentElement;@@ -231,15 +260,15 @@ export const AudioWaveform = memo(function AudioWaveform({}, [draw]);useEffect(() => {- const clip = rootRef.current?.closest(".timeline-clip");+ const clip = root?.closest(".timeline-clip");if (!(clip instanceof HTMLElement)) return;if (muted) clip.setAttribute("data-audio-muted", "true");else clip.removeAttribute("data-audio-muted");return () => clip.removeAttribute("data-audio-muted");- }, [muted]);+ }, [muted, root]);return (- <div ref={rootRef} className="absolute inset-0">+ <div ref={setRoot} className="absolute inset-0"><divref={setStripRef}className="absolute inset-0 overflow-hidden"diff --git a/packages/studio/src/player/components/VideoThumbnail.test.tsx b/packages/studio/src/player/components/VideoThumbnail.test.tsxindex 3f1801f438..d9103555f4 100644--- a/packages/studio/src/player/components/VideoThumbnail.test.tsx+++ b/packages/studio/src/player/components/VideoThumbnail.test.tsx@@ -30,25 +30,35 @@ afterEach(() => {// Registered after the reset above, so each test's strip unmounts before the cache is cleared.const harness = createHappyDomRootHarness();+function clip(+ videoSrc = "/api/projects/p/preview/assets/clip.mp4",+ priority: "visible" | "overscan" = "visible",+) {+ return (+ <VideoThumbnail+ videoSrc={videoSrc}+ label=""+ labelColor="#fff"+ projectId="p"+ sessionEpoch={1}+ priority={priority}+ />+ );+}+async function render(width = 0, height = 40) {Object.defineProperty(host, "clientWidth", { configurable: true, value: width });Object.defineProperty(host, "clientHeight", { configurable: true, value: height });const root = harness.mount(host);await act(async () => {- root.render(- <VideoThumbnail- videoSrc="/api/projects/p/preview/assets/clip.mp4"- label=""- labelColor="#fff"- projectId="p"- sessionEpoch={1}- priority="visible"- />,- );+ root.render(clip());await Promise.resolve();});+ return root;}+const shownSources = () => [...host.querySelectorAll("img")].map((img) => img.getAttribute("src"));+describe("VideoThumbnail", () => {it("does not acquire a thumbnail lease before the clip is measured", async () => {vi.mocked(decodeVideoThumbnail).mockResolvedValue({@@ -71,7 +81,7 @@ describe("VideoThumbnail", () => {expect(decodeVideoThumbnail).toHaveBeenNthCalledWith(1,- expect.objectContaining({ frameCount: 1 }),+ expect.objectContaining({ frameCount: 1, contentVersion: "project=p&session=1" }),expect.any(AbortSignal),);expect(decodeVideoThumbnail).toHaveBeenCalledWith(@@ -91,7 +101,8 @@ describe("VideoThumbnail", () => {await render(300);const tiles = [...host.querySelectorAll("img")].map((img) => img.getAttribute("src"));- expect(tiles).toEqual(["blob:0", "blob:2", "blob:4", "blob:5", "blob:7"]);+ // Seven slices and the end frame: the last tile shows the end.+ expect(tiles).toEqual(["blob:0", "blob:2", "blob:3", "blob:4", "blob:7"]);});describe("on a 10-minute clip at full zoom", () => {@@ -280,6 +291,140 @@ describe("VideoThumbnail during a zoom", () => {);});+ it("shows the strip it has, not the poster, while a zoom's new width decodes", async () => {+ vi.mocked(decodeVideoThumbnail).mockImplementation(async ({ frameCount }) =>+ frameCount === 1+ ? { value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, weight: 1 }+ : {+ value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 },+ weight: 2,+ },+ );+ await render(440);+ expect(+ new Set([...host.querySelectorAll("img")].map((img) => img.getAttribute("src"))),+ ).toEqual(new Set(["blob:a", "blob:b"]));+ // The new width's decode never lands during this test.+ vi.mocked(decodeVideoThumbnail).mockImplementation(() => new Promise(() => {}));+ await act(async () => {+ reportResize(880, 40);+ await Promise.resolve();+ });+ await rest();+ const shown = [...host.querySelectorAll("img")].map((img) => img.getAttribute("src"));+ expect(shown.length).toBeGreaterThan(0);+ expect(shown).not.toContain("blob:poster");+ });++ it("keeps holding the strip when the clip moves between on and off screen mid-decode", async () => {+ vi.mocked(decodeVideoThumbnail).mockImplementation(async ({ frameCount }) =>+ frameCount === 1+ ? { value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, weight: 1 }+ : { value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 }, weight: 2 },+ );+ const root = await render(440);+ vi.mocked(decodeVideoThumbnail).mockImplementation(() => new Promise(() => {}));+ await act(async () => {+ reportResize(880, 40);+ root.render(clip(undefined, "overscan"));+ await Promise.resolve();+ });+ await rest();+ expect(new Set(shownSources())).toEqual(new Set(["blob:a", "blob:b"]));+ });++ it("holds the strip, and decodes only the new width, when the cache is over its budget", async () => {+ const disposed: number[] = [];+ vi.mocked(decodeVideoThumbnail).mockImplementation(async ({ frameCount }) =>+ frameCount === 1+ ? { value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, weight: 1 }+ : {+ value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 },+ weight: 2,+ dispose: () => disposed.push(frameCount),+ },+ );+ await render(440);+ const held = vi.mocked(decodeVideoThumbnail).mock.calls.at(-1)![0].frameCount;+ // Fill the project's cache past its entry budget, so a strip without a lease is evicted.+ await act(async () => {+ for (let i = 0; i < 100; i++) {+ const lease = thumbnailScheduler.acquire(+ {+ key: `filler-${i}`,+ projectId: "p",+ sessionEpoch: 1,+ kind: "image",+ priority: "visible",+ load: async () => ({+ value: { kind: "image", url: `blob:f${i}`, aspect: 1 },+ weight: 1,+ }),+ },+ () => {},+ );+ for (let tick = 0; tick < 5; tick++) await Promise.resolve();+ lease.release();+ }+ });+ const decodes = vi.mocked(decodeVideoThumbnail).mock.calls.length;+ vi.mocked(decodeVideoThumbnail).mockImplementation(() => new Promise(() => {}));+ await act(async () => {+ reportResize(880, 40);+ await Promise.resolve();+ });+ await rest();+ const newWidths = vi+ .mocked(decodeVideoThumbnail)+ .mock.calls.slice(decodes)+ .map(([r]) => r.frameCount);+ expect(newWidths).not.toContain(held);+ expect(disposed).toEqual([]);+ expect(new Set(shownSources())).toEqual(new Set(["blob:a", "blob:b"]));+ });++ it("never holds one clip's strip for another while the other decodes", async () => {+ vi.mocked(decodeVideoThumbnail).mockImplementation(({ source }) =>+ source.endsWith("clip.mp4")+ ? Promise.resolve({+ value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 },+ weight: 2,+ })+ : new Promise(() => {}),+ );+ const root = await render(440);+ expect(shownSources()).toContain("blob:a");+ await act(async () => {+ root.render(clip("/api/projects/p/preview/assets/other.mp4"));+ await Promise.resolve();+ });+ await rest();+ expect(shownSources()).toEqual([]);+ });++ it("lets go of the strip it held once the clip narrows to one frame", async () => {+ vi.mocked(decodeVideoThumbnail).mockImplementation(async ({ frameCount }) =>+ frameCount === 1+ ? { value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, weight: 1 }+ : { value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 }, weight: 2 },+ );+ await render(440);+ await act(async () => {+ reportResize(50, 40);+ await Promise.resolve();+ });+ await rest();+ expect(new Set(shownSources())).toEqual(new Set(["blob:poster"]));+ // Widening again shows the poster until the new width lands, not the strip it once held.+ vi.mocked(decodeVideoThumbnail).mockImplementation(() => new Promise(() => {}));+ await act(async () => {+ reportResize(880, 40);+ await Promise.resolve();+ });+ await rest();+ expect(new Set(shownSources())).toEqual(new Set(["blob:poster"]));+ });+it("keeps its filmstrip while the zoom resizes it, and decodes the new width at rest", async () => {await render(440);vi.mocked(decodeVideoThumbnail).mockClear();diff --git a/packages/studio/src/player/components/VideoThumbnail.tsx b/packages/studio/src/player/components/VideoThumbnail.tsxindex a5e00b8a4b..dcb0eb776b 100644--- a/packages/studio/src/player/components/VideoThumbnail.tsx+++ b/packages/studio/src/player/components/VideoThumbnail.tsx@@ -1,4 +1,4 @@-import { memo, useMemo } from "react";+import { memo, useMemo, useState } from "react";import { useThumbnailLease } from "../../hooks/useThumbnailLease";import { useThumbnailStripSize } from "../../hooks/useThumbnailStripSize";import {@@ -8,7 +8,11 @@ import {} from "../lib/thumbnailScheduler";import { decodeVideoThumbnail } from "../lib/thumbnailVideoDecoder";import { ThumbnailTiles } from "./ThumbnailTiles";-import { computeThumbnailStrip, quantizeThumbnailFrameCount } from "./thumbnailUtils";+import {+ computeThumbnailStrip,+ quantizeThumbnailFrameCount,+ thumbnailFrameForTile,+} from "./thumbnailUtils";import { useValueAtRest } from "./timelineMotion";interface VideoThumbnailProps {@@ -55,6 +59,7 @@ function createVideoThumbnailRequest(decodeVideoThumbnail({source: videoSrc,+ contentVersion: createThumbnailKey({ project: projectId, session: sessionEpoch }),sourceStart,sourceRangeDuration: sourceRangeDuration ?? duration,frameCount,@@ -68,13 +73,31 @@ function createVideoThumbnailRequest(function selectThumbnailSnapshot(poster: ThumbnailSnapshot,rich: ThumbnailSnapshot,+ shown: ThumbnailSnapshot,): ThumbnailSnapshot {if (rich.status === "ready") return rich;+ if (shown.status === "ready") return shown;if (poster.status === "ready") return poster;if (rich.status === "loading" || poster.status === "loading") return { status: "loading" };return poster;}+type VideoThumbnailRequest = ReturnType<typeof createVideoThumbnailRequest>;++function useVideoThumbnailSnapshot(+ poster: VideoThumbnailRequest | null,+ rich: VideoThumbnailRequest | null,+ media: string,+): ThumbnailSnapshot {+ const posterSnapshot = useThumbnailLease(poster);+ const richSnapshot = useThumbnailLease(rich);+ const [shown, setShown] = useState({ media, request: rich });+ const settled = richSnapshot.status === "ready" || rich === null;+ if (settled && shown.request !== rich) setShown({ media, request: rich });+ const shownSnapshot = useThumbnailLease(shown.media === media ? shown.request : null);+ return selectThumbnailSnapshot(posterSnapshot, richSnapshot, shownSnapshot);+}+/** Sparse, bounded video frames supplied by the shared thumbnail scheduler. */export const VideoThumbnail = memo(function VideoThumbnail({videoSrc,@@ -114,9 +137,11 @@ export const VideoThumbnail = memo(function VideoThumbnail({[requestFrameCount, requestProps],);const measured = useValueAtRest(container.width > 0);- const posterSnapshot = useThumbnailLease(measured ? posterRequest : null);- const richSnapshot = useThumbnailLease(measured && requestFrameCount > 1 ? richRequest : null);- const snapshot = selectThumbnailSnapshot(posterSnapshot, richSnapshot);+ const snapshot = useVideoThumbnailSnapshot(+ measured ? posterRequest : null,+ measured && requestFrameCount > 1 ? richRequest : null,+ posterRequest.key,+ );const value = snapshot.status === "ready" ? snapshot.value : null;const urls =value?.kind === "filmstrip" ? value.urls : value?.kind === "image" ? [value.url] : [];@@ -133,7 +158,7 @@ export const VideoThumbnail = memo(function VideoThumbnail({watchGap={watchGap}>{(index) => {- const src = urls[Math.round((index * (urls.length - 1)) / Math.max(1, frameCount - 1))];+ const src = urls[thumbnailFrameForTile(index, frameCount, urls.length)];return (<divkey={index}diff --git a/packages/studio/src/player/components/thumbnailUtils.test.ts b/packages/studio/src/player/components/thumbnailUtils.test.tsindex e9fd1e73a4..3c67e344af 100644--- a/packages/studio/src/player/components/thumbnailUtils.test.ts+++ b/packages/studio/src/player/components/thumbnailUtils.test.ts@@ -4,7 +4,9 @@ import {encodePreviewPath,resolveMediaPreviewUrl,quantizeThumbnailFrameCount,+ thumbnailFrameForTile,} from "./thumbnailUtils";+import { MAX_VISIBLE_THUMBNAIL_FRAMES } from "../lib/timelineViewportBudgets";describe("computeThumbnailStrip", () => {it("sizes tiles by aspect ratio at the clip height", () => {@@ -51,15 +53,27 @@ describe("computeThumbnailStrip", () => {});});+describe("thumbnailFrameForTile", () => {+ it("shows the clip's last frame in the last tile, and the slice under each other tile's centre", () => {+ // 8 slices and the end frame.+ expect([0, 1, 2].map((tile) => thumbnailFrameForTile(tile, 3, 9))).toEqual([1, 4, 8]);+ // 2 slices and the end frame across 4 tiles.+ expect([0, 1, 2, 3].map((tile) => thumbnailFrameForTile(tile, 4, 3))).toEqual([0, 0, 1, 2]);+ expect(thumbnailFrameForTile(0, 1, 9)).toBe(4);+ });+});+describe("quantizeThumbnailFrameCount", () => {it("uses doubling buckets and never exceeds the 4K geometry ceiling", () => {expect(quantizeThumbnailFrameCount(5)).toBe(8);expect(quantizeThumbnailFrameCount(32)).toBe(32);- expect(quantizeThumbnailFrameCount(34)).toBe(33);});- it("caps decode requests at the shared visible-frame budget", () => {- expect(quantizeThumbnailFrameCount(124)).toBe(33);+ it("caps decode requests at the largest step within the visible-frame budget", () => {+ // A step that is not a power of two would share no frames with the step below it.+ expect(MAX_VISIBLE_THUMBNAIL_FRAMES).toBeGreaterThan(32);+ expect(quantizeThumbnailFrameCount(34)).toBe(32);+ expect(quantizeThumbnailFrameCount(124)).toBe(32);});});diff --git a/packages/studio/src/player/components/thumbnailUtils.ts b/packages/studio/src/player/components/thumbnailUtils.tsindex 0a5a335f15..65290e44fe 100644--- a/packages/studio/src/player/components/thumbnailUtils.ts+++ b/packages/studio/src/player/components/thumbnailUtils.ts@@ -11,7 +11,23 @@ export interface ThumbnailStripLayout {/** Quantize request identities so a pixel-by-pixel resize does not thrash the cache. */export function quantizeThumbnailFrameCount(frameCount: number): number {const safeCount = Math.max(1, Number.isFinite(frameCount) ? Math.ceil(frameCount) : 1);- return Math.min(MAX_VISIBLE_THUMBNAIL_FRAMES, 2 ** Math.ceil(Math.log2(safeCount)));+ const cap = 2 ** Math.floor(Math.log2(MAX_VISIBLE_THUMBNAIL_FRAMES));+ return Math.min(cap, 2 ** Math.ceil(Math.log2(safeCount)));+}++/**+ * The decoded frame tile `index` of `tileCount` shows, of a strip's `frameCount`: the last tile+ * shows the clip's last frame; the others the slice (videoThumbnailTimestamps) holding their centre.+ */+export function thumbnailFrameForTile(+ index: number,+ tileCount: number,+ frameCount: number,+): number {+ if (frameCount < 2) return 0;+ if (tileCount > 1 && index >= tileCount - 1) return frameCount - 1;+ const slices = frameCount - 1;+ return Math.min(slices - 1, Math.floor(((index + 0.5) * slices) / tileCount));}/**diff --git a/packages/studio/src/player/components/timelineMotion.test.ts b/packages/studio/src/player/components/timelineMotion.test.tsnew file mode 100644index 0000000000..33d4c2ee65--- /dev/null+++ b/packages/studio/src/player/components/timelineMotion.test.ts@@ -0,0 +1,27 @@+import { afterEach, expect, it, vi } from "vitest";+import { markTimelineMotion, TIMELINE_REST_MS, whenTimelineIdle } from "./timelineMotion";++afterEach(() => {+ vi.runOnlyPendingTimers();+ vi.useRealTimers();+});++it("waits for a zoom to rest before resolving, and resolves at once when nothing moves", async () => {+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "requestIdleCallback"] });+ markTimelineMotion();+ let idle = false;+ void whenTimelineIdle().then(() => (idle = true));+ await vi.advanceTimersByTimeAsync(TIMELINE_REST_MS - 50);+ expect(idle).toBe(false);+ // Another zoom step pushes the rest back.+ markTimelineMotion();+ await vi.advanceTimersByTimeAsync(TIMELINE_REST_MS - 50);+ expect(idle).toBe(false);+ await vi.advanceTimersByTimeAsync(TIMELINE_REST_MS);+ expect(idle).toBe(true);++ let again = false;+ void whenTimelineIdle().then(() => (again = true));+ await vi.advanceTimersByTimeAsync(1);+ expect(again).toBe(true);+});diff --git a/packages/studio/src/player/components/timelineMotion.ts b/packages/studio/src/player/components/timelineMotion.tsindex 2fda7a3b52..78ebb29d82 100644--- a/packages/studio/src/player/components/timelineMotion.ts+++ b/packages/studio/src/player/components/timelineMotion.ts@@ -23,6 +23,26 @@ export function markTimelineMotion(): void {}, TIMELINE_REST_MS);}+const onIdle = (callback: () => void) => {+ if (typeof requestIdleCallback === "function") requestIdleCallback(callback, { timeout: 1000 });+ else setTimeout(callback, 0);+};++/** Resolves at an idle moment with no zoom moving the timeline: long work waits there, not mid-zoom. */+export function whenTimelineIdle(): Promise<void> {+ return new Promise((resolve) => {+ const check = () => {+ if (!moving) return resolve();+ const stop = subscribeTimelineMotion(() => {+ if (moving) return;+ stop();+ onIdle(check);+ });+ };+ onIdle(check);+ });+}+export function isTimelineMoving(): boolean {return moving;}diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.tsindex c08f4687bb..cbac2487a4 100644--- a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts+++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts@@ -39,10 +39,20 @@ function recordDecodes(decoded: number[][]): void {});}+const endOf = (start: number, duration: number) => start + Math.max(0, duration - 0.001);++// Decoded frames are shared per source for the module's life, so each test decodes its own file.+let sources = 0;+let source = "";+beforeEach(() => {+ source = `/clip-${++sources}.mp4`;vi.clearAllMocks();getKeyPacket.mockImplementation(async () => null);- vi.spyOn(URL, "createObjectURL").mockReturnValueOnce("blob:one").mockReturnValueOnce("blob:two");+ vi.spyOn(URL, "createObjectURL")+ .mockReturnValueOnce("blob:one")+ .mockReturnValueOnce("blob:two")+ .mockReturnValueOnce("blob:end");vi.spyOn(URL, "revokeObjectURL").mockImplementation(() => {});HTMLCanvasElement.prototype.toBlob = function toBlob(callback) {callback(new Blob(["frame"], { type: "image/jpeg" }));@@ -55,9 +65,15 @@ beforeEach(() => {});describe("videoThumbnailTimestamps", () => {- it("uses the midpoint for a poster and sorted sparse points for a strip", () => {+ it("uses the midpoint for a poster, and each slice's left edge then the end for a strip", () => {expect(videoThumbnailTimestamps(2, 6, 1)).toEqual([5]);- expect(videoThumbnailTimestamps(2, 6, 4)).toEqual([2, 4, 6, 8]);+ expect(videoThumbnailTimestamps(2, 6, 4)).toEqual([2, 3.5, 5, 6.5, endOf(2, 6)]);+ });++ it("nests each strip in the strip twice as long, and the poster in every strip", () => {+ const strip = videoThumbnailTimestamps(0, 8, 4);+ expect(videoThumbnailTimestamps(0, 8, 8).filter((_, i) => i % 2 === 0)).toEqual(strip);+ expect(strip).toContain(videoThumbnailTimestamps(0, 8, 1)[0]);});it("clamps invalid source ranges", () => {@@ -71,31 +87,31 @@ describe("decodeVideoThumbnail", () => {const decoded: number[][] = [];recordDecodes(decoded);const result = await decodeVideoThumbnail(- { source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 6, frameCount: 2 },+ { source, sourceStart: 2, sourceRangeDuration: 6, frameCount: 2 },new AbortController().signal,);- expect(decoded).toEqual([[2, 8]]);+ expect(decoded).toEqual([[2, 5, endOf(2, 6)]]);expect(result.value).toEqual({kind: "filmstrip",- urls: ["blob:one", "blob:two"],+ urls: ["blob:one", "blob:two", "blob:end"],aspect: 9 / 16,});result.dispose?.();result.dispose?.();- expect(URL.revokeObjectURL).toHaveBeenCalledTimes(2);+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3);expect(dispose).toHaveBeenCalledTimes(1);});it("decodes each strip frame at its keyframe unless that keyframe is before the clip's range", async () => {- getKeyPacket.mockImplementation(async (time) => ({ timestamp: Math.floor(time / 4) * 4 }));+ getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 0.5 }));const decoded: number[][] = [];recordDecodes(decoded);await decodeVideoThumbnail(- { source: "/clip.mp4", sourceStart: 2, sourceRangeDuration: 8, frameCount: 3 },+ { source, sourceStart: 2, sourceRangeDuration: 8, frameCount: 4 },new AbortController().signal,);- expect(decoded).toEqual([[2, 4, 8]]);+ expect(decoded).toEqual([[2, 3.5, 5.5, 7.5, endOf(2, 8)]]);});it("keeps a slot's own time when its keyframe is more than half a slot earlier", async () => {@@ -103,14 +119,10 @@ describe("decodeVideoThumbnail", () => {const decoded: number[][] = [];recordDecodes(decoded);await decodeVideoThumbnail(- { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },- new AbortController().signal,- );- await decodeVideoThumbnail(- { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 },+ { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 4 },new AbortController().signal,);- expect(decoded).toEqual([[0, 5, 10], [5]]);+ expect(decoded).toEqual([[0, 2.5, 5, 7.5, endOf(0, 10)]]);});it("looks up each keyframe just before decoding it", async () => {@@ -126,10 +138,11 @@ describe("decodeVideoThumbnail", () => {}});await decodeVideoThumbnail(- { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },+ { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 2 },new AbortController().signal,);- expect(events).toEqual(["key 0", "frame 0", "key 5", "frame 5", "key 10", "frame 10"]);+ const end = endOf(0, 10);+ expect(events).toEqual(["key 0", "frame 0", "key 5", "frame 5", `key ${end}`, `frame ${end}`]);});it("ends the decode times without throwing when cancelled during a keyframe lookup", async () => {@@ -146,7 +159,7 @@ describe("decodeVideoThumbnail", () => {});await expect(decodeVideoThumbnail(- { source: "/clip.mp4", sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },+ { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 3 },controller.signal,),).rejects.toThrow("Aborted");@@ -164,10 +177,198 @@ describe("decodeVideoThumbnail", () => {const decoded: number[][] = [];recordDecodes(decoded);await decodeVideoThumbnail(- { source: "/clip.mp4", sourceStart: 5, sourceRangeDuration: 5, frameCount: 3 },+ { source, sourceStart: 5, sourceRangeDuration: 5, frameCount: 2 },new AbortController().signal,);- expect(decoded).toEqual([[5, 7.5, 10]]);+ expect(decoded).toEqual([[5, 7.5, endOf(5, 5)]]);+ });++ it("reuses the frames a strip of the same source shows, so a zoom decodes only new times", async () => {+ vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`);+ const decoded: number[][] = [];+ recordDecodes(decoded);+ const range = { source, sourceStart: 0, sourceRangeDuration: 8 };+ const signal = new AbortController().signal;+ const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal);+ // Twice the frames decodes only the new half; half the frames decodes nothing. All share the end.+ const eight = await decodeVideoThumbnail({ ...range, frameCount: 8 }, signal);+ const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);+ expect(decoded).toEqual([+ [0, 2, 4, 6, endOf(0, 8)],+ [1, 3, 5, 7],+ ]);+ const urls = (result: typeof four) =>+ result.value.kind === "filmstrip" ? result.value.urls : [];+ expect(urls(eight).filter((_, i) => i % 2 === 0)).toEqual(urls(four));+ expect(urls(two)).toEqual([urls(four)[0], urls(four)[2], urls(four)[4]]);+ for (const result of [four, eight, two]) result.dispose?.();+ });++ it("decodes a frame again when the one shared for its time sits too far before it", async () => {+ // Every keyframe is 2 s before the time asked: a 2-frame strip of 8 s accepts that (its+ // frames stand for 4 s each), a 4-frame strip does not, or its frames would run backwards.+ getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 2 }));+ const decoded: number[][] = [];+ recordDecodes(decoded);+ const range = { source, sourceStart: 0, sourceRangeDuration: 8 };+ const signal = new AbortController().signal;+ const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);+ const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal);+ expect(decoded).toEqual([+ [0, 2, endOf(0, 8)],+ [2, 4, 6],+ ]);+ two.dispose?.();+ four.dispose?.();+ });++ it("returns a strip whose frames are all shared without opening the file again", async () => {+ recordDecodes([]);+ const range = { source, sourceStart: 0, sourceRangeDuration: 8 };+ const signal = new AbortController().signal;+ const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal);+ const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);+ expect(input.getPrimaryVideoTrack).toHaveBeenCalledTimes(1);+ four.dispose?.();+ two.dispose?.();+ });++ it("reopens the same asset after a project round trip and keeps new-session frames separate", async () => {+ let frame = 0;+ vi.mocked(URL.createObjectURL)+ .mockReset()+ .mockImplementation(() => `blob:version-${++frame}`);+ const decoded: number[][] = [];+ recordDecodes(decoded);+ const signal = new AbortController().signal;+ const original = await decodeVideoThumbnail(+ { source, contentVersion: "A:1", frameCount: 1 },+ signal,+ );+ const other = await decodeVideoThumbnail(+ { source: `${source}?B`, contentVersion: "B:2", frameCount: 1 },+ signal,+ );+ input.getPrimaryVideoTrack.mockResolvedValue({+ getDisplayWidth: vi.fn(async () => 1280),+ getDisplayHeight: vi.fn(async () => 360),+ getDurationFromMetadata: vi.fn(async () => 10),+ });+ const reopened = await decodeVideoThumbnail(+ { source, contentVersion: "A:3", frameCount: 1 },+ signal,+ );+ const shared = await decodeVideoThumbnail(+ { source, contentVersion: "A:3", frameCount: 1 },+ signal,+ );+ expect(reopened.value).toEqual({ kind: "image", url: "blob:version-3", aspect: 32 / 9 });+ expect(shared.value).toEqual(reopened.value);+ expect(decoded).toEqual([[5], [5], [5]]);+ expect(input.getPrimaryVideoTrack).toHaveBeenCalledTimes(3);+ original.dispose?.();+ expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:version-1");+ expect(URL.revokeObjectURL).not.toHaveBeenCalledWith("blob:version-3");+ other.dispose?.();+ reopened.dispose?.();+ expect(URL.revokeObjectURL).not.toHaveBeenCalledWith("blob:version-3");+ shared.dispose?.();+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3);+ });++ it("keeps one copy of a frame two strips decode at once, and revokes the other", async () => {+ vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`);+ // The first strip holds its first frame until the second is decoding too.+ let release!: () => void;+ const bothDecoding = new Promise<void>((resolve) => (release = resolve));+ canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {+ const first = canvasesAtTimestamps.mock.calls.length === 1;+ for await (const _time of timestamps) {+ if (first) await bothDecoding;+ else release();+ yield { canvas: document.createElement("canvas") };+ }+ });+ const range = { source, sourceStart: 0, sourceRangeDuration: 8 };+ const signal = new AbortController().signal;+ const decodingTwo = decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);+ await vi.waitFor(() => expect(canvasesAtTimestamps).toHaveBeenCalledOnce());+ const [two, four] = await Promise.all([+ decodingTwo,+ decodeVideoThumbnail({ ...range, frameCount: 4 }, signal),+ ]);+ // Both decoded the frames at 0 s, 4 s and the end; the second copy of each goes at once.+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3);+ two.dispose?.();+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(3);+ four.dispose?.();+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(8);+ });++ it("fills a frame that failed to decode from the one before it, so tiles stay on their times", async () => {+ canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {+ let slot = 0;+ for await (const _time of timestamps) {+ yield slot++ === 2 ? null : { canvas: document.createElement("canvas") };+ }+ });+ const signal = new AbortController().signal;+ const result = await decodeVideoThumbnail(+ { source, sourceStart: 0, sourceRangeDuration: 8, frameCount: 2 },+ signal,+ );+ expect(result.value).toMatchObject({+ kind: "filmstrip",+ urls: ["blob:one", "blob:two", "blob:two"],+ });+ // The cache is charged for the two decoded frames, not the filled copy.+ recordDecodes([]);+ const whole = await decodeVideoThumbnail(+ { source: `${source}?whole`, sourceStart: 0, sourceRangeDuration: 8, frameCount: 2 },+ signal,+ );+ expect(result.weight * 3).toBe(whole.weight * 2);+ result.dispose?.();+ whole.dispose?.();+ });++ it("snaps a poster up to a quarter of its range and a strip frame up to half a slice", async () => {+ const decoded: number[][] = [];+ recordDecodes(decoded);+ const signal = new AbortController().signal;+ getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 2.4 }));+ await decodeVideoThumbnail(+ { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 },+ signal,+ );+ getKeyPacket.mockImplementation(async (time) => ({ timestamp: time - 2.6 }));+ await decodeVideoThumbnail(+ { source: `${source}?b`, sourceStart: 0, sourceRangeDuration: 10, frameCount: 1 },+ signal,+ );+ // Four slices of 8 s: a keyframe 0.9 s early snaps, one 1.1 s early does not.+ getKeyPacket.mockImplementation(async (time) => ({+ timestamp: time - (time === 4 ? 0.9 : 1.1),+ }));+ await decodeVideoThumbnail(+ { source: `${source}?c`, sourceStart: 0, sourceRangeDuration: 8, frameCount: 4 },+ signal,+ );+ expect(decoded).toEqual([[5 - 2.4], [5], [0, 2, 4 - 0.9, 6, endOf(0, 8)]]);+ });++ it("revokes a shared frame only once no strip shows it", async () => {+ vi.mocked(URL.createObjectURL).mockImplementation(() => `blob:${Math.random()}`);+ recordDecodes([]);+ const range = { source, sourceStart: 0, sourceRangeDuration: 8 };+ const signal = new AbortController().signal;+ const four = await decodeVideoThumbnail({ ...range, frameCount: 4 }, signal);+ const two = await decodeVideoThumbnail({ ...range, frameCount: 2 }, signal);+ four.dispose?.();+ // The three frames the 2-slice strip still shows stay; the other two go.+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(2);+ two.dispose?.();+ expect(URL.revokeObjectURL).toHaveBeenCalledTimes(5);});it("releases input and degrades when the source has no video track", async () => {@@ -181,13 +382,15 @@ describe("decodeVideoThumbnail", () => {it("revokes partial results when cancellation lands during extraction", async () => {const controller = new AbortController();const canvas = document.createElement("canvas");- canvasesAtTimestamps.mockImplementation(async function* () {- yield { canvas, timestamp: 1, duration: 1 };- controller.abort();- yield { canvas, timestamp: 2, duration: 1 };+ canvasesAtTimestamps.mockImplementation(async function* (timestamps: AsyncIterable<number>) {+ let frames = 0;+ for await (const timestamp of timestamps) {+ if (frames++ === 1) controller.abort();+ yield { canvas, timestamp, duration: 1 };+ }});await expect(- decodeVideoThumbnail({ source: "/clip.mp4", frameCount: 2 }, controller.signal),+ decodeVideoThumbnail({ source, frameCount: 2 }, controller.signal),).rejects.toMatchObject({ name: "AbortError" });expect(URL.revokeObjectURL).toHaveBeenCalledTimes(1);expect(dispose).toHaveBeenCalledTimes(1);@@ -207,10 +410,7 @@ describe("decodeVideoThumbnail", () => {getDurationFromMetadata,});- const decoding = decodeVideoThumbnail(- { source: "/clip.mp4", frameCount: 1 },- controller.signal,- );+ const decoding = decodeVideoThumbnail({ source, frameCount: 1 }, controller.signal);await vi.waitFor(() => expect(getDisplayWidth).toHaveBeenCalledOnce());controller.abort();resolveWidth(1080);diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.tsindex b892c6cc94..0bf417dd56 100644--- a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts+++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts@@ -1,14 +1,21 @@import { TIMELINE_VIEWPORT_BUDGETS, type TimelineViewportBudgets } from "./timelineViewportBudgets";-import type { ThumbnailLoadedResult, ThumbnailValue } from "./thumbnailScheduler";+import {+ createThumbnailKey,+ type ThumbnailLoadedResult,+ type ThumbnailValue,+} from "./thumbnailScheduler";export interface VideoThumbnailDecodeRequest {source: string;+ contentVersion?: string;sourceStart?: number;sourceRangeDuration?: number;frameCount: number;fit?: "contain" | "cover";}+const LAST_FRAME_INSET_S = 0.001;+export function videoThumbnailTimestamps(start: number,duration: number,@@ -18,10 +25,63 @@ export function videoThumbnailTimestamps(const safeDuration = Math.max(0, Number.isFinite(duration) ? duration : 0);const count = Math.max(1, Number.isFinite(frameCount) ? Math.floor(frameCount) : 1);if (count === 1) return [safeStart + safeDuration / 2];- return Array.from(+ const edges = Array.from({ length: count },- (_, index) => safeStart + (safeDuration * index) / (count - 1),+ (_, index) => safeStart + (safeDuration * index) / count,);+ return [...edges, safeStart + Math.max(0, safeDuration - LAST_FRAME_INSET_S)];+}++interface SharedFrame {+ url: string;+ decodedAt: number;+ users: number;+}++interface HeldFrame {+ key: string;+ frame: SharedFrame;+}++interface FrameWindow {+ sourceStart: number;+ lead: number;+ end: number;+}++const sharedFrames = new Map<string, SharedFrame[]>();+const sourceInfos = new Map<string, { version: string | undefined; info: SourceInfo }>();++const accepts = (window: FrameWindow, time: number, decodedAt: number) =>+ decodedAt >= window.sourceStart && time - decodedAt <= (time >= window.end ? 0 : window.lead);++function takeSharedFrame(key: string, time: number, window: FrameWindow): SharedFrame | undefined {+ const frame = sharedFrames.get(key)?.find((shared) => accepts(window, time, shared.decodedAt));+ if (frame) frame.users += 1;+ return frame;+}++function shareFrame(key: string, decodedAt: number, url: string): SharedFrame {+ const frames = sharedFrames.get(key) ?? [];+ const existing = frames.find((shared) => shared.decodedAt === decodedAt);+ if (existing) {+ existing.users += 1;+ URL.revokeObjectURL(url);+ return existing;+ }+ const frame = { url, decodedAt, users: 1 };+ sharedFrames.set(key, [...frames, frame]);+ return frame;+}++function releaseSharedFrames(held: HeldFrame[]): void {+ for (const { key, frame } of held.splice(0)) {+ if (--frame.users > 0) continue;+ const rest = sharedFrames.get(key)?.filter((shared) => shared !== frame) ?? [];+ if (rest.length > 0) sharedFrames.set(key, rest);+ else sharedFrames.delete(key);+ URL.revokeObjectURL(frame.url);+ }}async function canvasToBlob(canvas: HTMLCanvasElement | OffscreenCanvas): Promise<Blob> {@@ -38,7 +98,8 @@ async function canvasToBlob(canvas: HTMLCanvasElement | OffscreenCanvas): Promis}interface DecodedResources {- urls: string[];+ urls: (string | undefined)[];+ held: HeldFrame[];canvases: Set<HTMLCanvasElement | OffscreenCanvas>;}@@ -48,12 +109,18 @@ interface ThumbnailCanvasSink {): AsyncIterable<{ canvas: HTMLCanvasElement | OffscreenCanvas } | null>;}+interface SourceInfo {+ aspect: number;+ metadataDuration: number | null;+}+function throwIfAborted(signal: AbortSignal): void {if (signal.aborted) throw new DOMException("Aborted", "AbortError");}function releaseDecodedResources(resources: DecodedResources): void {- for (const url of resources.urls.splice(0)) URL.revokeObjectURL(url);+ resources.urls.length = 0;+ releaseSharedFrames(resources.held);for (const canvas of resources.canvases) {canvas.width = 0;canvas.height = 0;@@ -75,19 +142,71 @@ function targetDimensions(};}+function stripTimes(+ request: VideoThumbnailDecodeRequest,+ info: SourceInfo,+ budgets: Readonly<TimelineViewportBudgets>,+) {+ const requestedStart = Math.max(0, request.sourceStart ?? 0);+ const sourceDuration = Math.max(+ 0,+ info.metadataDuration ?? requestedStart + (request.sourceRangeDuration ?? 0),+ );+ const sourceStart = Math.min(requestedStart, sourceDuration);+ const requestedDuration =+ request.sourceRangeDuration ?? Math.max(0, sourceDuration - sourceStart);+ const duration = Math.min(+ Math.max(0, requestedDuration),+ Math.max(0, sourceDuration - sourceStart),+ );+ const timestamps = videoThumbnailTimestamps(+ sourceStart,+ duration,+ Math.min(request.frameCount, budgets.richPreviewFrameCount),+ );+ const slices = Math.max(1, timestamps.length - 1);+ const window: FrameWindow = {+ sourceStart,+ lead: duration / Math.max(2, slices) / 2,+ end: timestamps.length > 1 ? timestamps.at(-1)! : Infinity,+ };+ return { window, timestamps };+}++function takeDecodedFrames(+ { timestamps, window }: ReturnType<typeof stripTimes>,+ keyOf: (time: number) => string,+ resources: DecodedResources,+): number[] {+ const missing: number[] = [];+ timestamps.forEach((time, slot) => {+ const key = keyOf(time);+ const frame = takeSharedFrame(key, time, window);+ if (!frame) return void missing.push(slot);+ resources.urls[slot] = frame.url;+ resources.held.push({ key, frame });+ });+ return missing;+}+async function decodeFrames(sink: ThumbnailCanvasSink,- timestamps: AsyncIterable<number>,+ times: AsyncIterable<number>,+ slots: { slot: number; key: string; decodedAt?: number }[],signal: AbortSignal,resources: DecodedResources,): Promise<void> {- for await (const wrapped of sink.canvasesAtTimestamps(timestamps)) {+ let next = 0;+ for await (const wrapped of sink.canvasesAtTimestamps(times)) {throwIfAborted(signal);- if (!wrapped) continue;+ const target = slots[next++];+ if (!wrapped || target?.decodedAt === undefined) continue;resources.canvases.add(wrapped.canvas);const blob = await canvasToBlob(wrapped.canvas);throwIfAborted(signal);- resources.urls.push(URL.createObjectURL(blob));+ const frame = shareFrame(target.key, target.decodedAt, URL.createObjectURL(blob));+ resources.urls[target.slot] = frame.url;+ resources.held.push({ key: target.key, frame });}throwIfAborted(signal);}@@ -95,18 +214,25 @@ async function decodeFrames(function loadedResult(resources: DecodedResources,aspect: number,- width: number,- height: number,+ budgets: Readonly<TimelineViewportBudgets>,+ frameCount = resources.urls.length,): ThumbnailLoadedResult {- const firstUrl = resources.urls[0];+ const decoded = resources.urls.filter((url): url is string => url !== undefined);+ const firstUrl = decoded[0];if (!firstUrl) throw new Error("Video source returned no thumbnail frames");+ let nearest = firstUrl;+ const urls = Array.from(+ { length: frameCount },+ (_, slot) => (nearest = resources.urls[slot] ?? nearest),+ );+ const { width, height } = targetDimensions(aspect, budgets);const value: ThumbnailValue =- resources.urls.length === 1+ urls.length === 1? { kind: "image", url: firstUrl, aspect }- : { kind: "filmstrip", urls: [...resources.urls], aspect };+ : { kind: "filmstrip", urls, aspect };return {value,- weight: width * height * 4 * resources.urls.length,+ weight: width * height * 4 * decoded.length,dispose: () => releaseDecodedResources(resources),};}@@ -116,6 +242,34 @@ export async function decodeVideoThumbnail(request: VideoThumbnailDecodeRequest,signal: AbortSignal,budgets: Readonly<TimelineViewportBudgets> = TIMELINE_VIEWPORT_BUDGETS,+): Promise<ThumbnailLoadedResult> {+ const fit = request.fit ?? "cover";+ const sourceKey = createThumbnailKey({ source: request.source, version: request.contentVersion });+ const keyOf = (time: number) => `${sourceKey}\u0000${fit}\u0000${time}`;+ const resources: DecodedResources = { urls: [], held: [], canvases: new Set() };+ const entry = sourceInfos.get(request.source);+ const known = entry?.version === request.contentVersion ? entry?.info : undefined;+ try {+ if (known) {+ if (takeDecodedFrames(stripTimes(request, known, budgets), keyOf, resources).length === 0) {+ throwIfAborted(signal);+ return loadedResult(resources, known.aspect, budgets);+ }+ releaseDecodedResources(resources);+ }+ return await decodeMissingFrames(request, signal, budgets, keyOf, resources);+ } catch (error) {+ releaseDecodedResources(resources);+ throw error;+ }+}++async function decodeMissingFrames(+ request: VideoThumbnailDecodeRequest,+ signal: AbortSignal,+ budgets: Readonly<TimelineViewportBudgets>,+ keyOf: (time: number) => string,+ resources: DecodedResources,): Promise<ThumbnailLoadedResult> {const mediabunny = await import("mediabunny");throwIfAborted(signal);@@ -124,7 +278,6 @@ export async function decodeVideoThumbnail(source: new mediabunny.UrlSource(request.source),formats: mediabunny.ALL_FORMATS,});- const resources: DecodedResources = { urls: [], canvases: new Set() };try {const track = await input.getPrimaryVideoTrack();throwIfAborted(signal);@@ -139,46 +292,36 @@ export async function decodeVideoThumbnail(}const metadataDuration = await track.getDurationFromMetadata({ skipLiveWait: true });throwIfAborted(signal);- const requestedStart = Math.max(0, request.sourceStart ?? 0);- const sourceDuration = Math.max(- 0,- metadataDuration ?? requestedStart + (request.sourceRangeDuration ?? 0),- );- const sourceStart = Math.min(requestedStart, sourceDuration);- const requestedDuration =- request.sourceRangeDuration ?? Math.max(0, sourceDuration - sourceStart);- const duration = Math.min(- Math.max(0, requestedDuration),- Math.max(0, sourceDuration - sourceStart),- );- const timestamps = videoThumbnailTimestamps(- sourceStart,- duration,- Math.min(request.frameCount, budgets.richPreviewFrameCount),- );- const keys = new mediabunny.EncodedPacketSink(track);- const maxKeyframeLead = duration / Math.max(2, timestamps.length - 1) / 2;- async function* decodeTimesAtNearbyKeyframes() {- for (const time of timestamps) {- const key = await keys.getKeyPacket(time, { metadataOnly: true });- if (signal.aborted) return;- const near = key && key.timestamp >= sourceStart && time - key.timestamp <= maxKeyframeLead;- yield near ? key.timestamp : time;+ const info: SourceInfo = { aspect: displayWidth / displayHeight, metadataDuration };+ sourceInfos.set(request.source, { version: request.contentVersion, info });+ const strip = stripTimes(request, info, budgets);+ const { timestamps, window } = strip;+ const slots: { slot: number; key: string; decodedAt?: number }[] = takeDecodedFrames(+ strip,+ keyOf,+ resources,+ ).map((slot) => ({ slot, key: keyOf(timestamps[slot]!) }));+ if (slots.length > 0) {+ const keys = new mediabunny.EncodedPacketSink(track);+ async function* decodeTimesAtNearbyKeyframes() {+ for (const target of slots) {+ const time = timestamps[target.slot]!;+ const key = await keys.getKeyPacket(time, { metadataOnly: true });+ if (signal.aborted) return;+ target.decodedAt = key && accepts(window, time, key.timestamp) ? key.timestamp : time;+ yield target.decodedAt;+ }}+ const target = targetDimensions(info.aspect, budgets);+ const sink = new mediabunny.CanvasSink(track, {+ width: target.width,+ height: target.height,+ fit: request.fit ?? "cover",+ poolSize: 1,+ });+ await decodeFrames(sink, decodeTimesAtNearbyKeyframes(), slots, signal, resources);}- const aspect = displayWidth / displayHeight;- const target = targetDimensions(aspect, budgets);- const sink = new mediabunny.CanvasSink(track, {- width: target.width,- height: target.height,- fit: request.fit ?? "cover",- poolSize: 1,- });- await decodeFrames(sink, decodeTimesAtNearbyKeyframes(), signal, resources);- return loadedResult(resources, aspect, target.width, target.height);- } catch (error) {- releaseDecodedResources(resources);- throw error;+ return loadedResult(resources, info.aspect, budgets, timestamps.length);} finally {input.dispose();}