Skip to content

Write HTML. Render video. Built for agents.

perf(studio): long clips mount only the picture tiles in view (#4993)

Miguel Ángel1bdb5edd93bb
diff --git a/packages/studio/src/components/sidebar/CompositionsTab.thumbnails.test.tsx b/packages/studio/src/components/sidebar/CompositionsTab.thumbnails.test.tsx
index be28743272..efe71972c8 100644
--- a/packages/studio/src/components/sidebar/CompositionsTab.thumbnails.test.tsx
+++ b/packages/studio/src/components/sidebar/CompositionsTab.thumbnails.test.tsx
@@ -39,6 +39,7 @@ class ViewportObserver {
const entry = { isIntersecting: true } as IntersectionObserverEntry;
this.callback([entry], this as unknown as IntersectionObserver);
}
+ unobserve() {}
disconnect() {}
}
diff --git a/packages/studio/src/hooks/intersectionObserverTestUtils.ts b/packages/studio/src/hooks/intersectionObserverTestUtils.ts
new file mode 100644
index 0000000000..31e01509b7
--- /dev/null
+++ b/packages/studio/src/hooks/intersectionObserverTestUtils.ts
@@ -0,0 +1,16 @@
+// Test-only IntersectionObserver stand-in: everything observed is reported near the screen, asynchronously.
+export function NearScreenIntersectionObserver(callback: IntersectionObserverCallback) {
+ const observer = {
+ observe(target: Element) {
+ queueMicrotask(() =>
+ callback(
+ [{ isIntersecting: true, target } as IntersectionObserverEntry],
+ observer as unknown as IntersectionObserver,
+ ),
+ );
+ },
+ unobserve() {},
+ disconnect() {},
+ };
+ return observer;
+}
diff --git a/packages/studio/src/hooks/resizeObserverTestUtils.ts b/packages/studio/src/hooks/resizeObserverTestUtils.ts
index 896922269d..fa4d63be79 100644
--- a/packages/studio/src/hooks/resizeObserverTestUtils.ts
+++ b/packages/studio/src/hooks/resizeObserverTestUtils.ts
@@ -1,14 +1,21 @@
// Test-only ResizeObserver stand-in: a test delivers a size by hand with `reportResize`.
let resizeCallback: ResizeObserverCallback | null = null;
+let observed: Element | null = null;
export function MockResizeObserver(callback: ResizeObserverCallback) {
resizeCallback = callback;
- return { observe() {}, disconnect() {}, unobserve() {} };
+ return {
+ observe(target: Element) {
+ observed = target;
+ },
+ disconnect() {},
+ unobserve() {},
+ };
}
export function reportResize(width: number, height: number): void {
resizeCallback!(
- [{ contentRect: { width, height } } as ResizeObserverEntry],
+ [{ contentRect: { width, height }, target: observed } as ResizeObserverEntry],
{} as ResizeObserver,
);
}
diff --git a/packages/studio/src/hooks/useThumbnailStripSize.test.tsx b/packages/studio/src/hooks/useThumbnailStripSize.test.tsx
index 1e138b1c96..5c6709c519 100644
--- a/packages/studio/src/hooks/useThumbnailStripSize.test.tsx
+++ b/packages/studio/src/hooks/useThumbnailStripSize.test.tsx
@@ -1,7 +1,9 @@
// @vitest-environment happy-dom
import { act } from "react";
import { createRoot } from "react-dom/client";
-import { expect, it } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { createHappyDomRootHarness } from "../player/components/testRootHarness";
+import { NearScreenIntersectionObserver } from "./intersectionObserverTestUtils";
import { MockResizeObserver, reportResize } from "./resizeObserverTestUtils";
import { useThumbnailStripSize } from "./useThumbnailStripSize";
@@ -45,3 +47,236 @@ it("does not re-render the strip when the observer reports the size it already h
globalThis.ResizeObserver = originalResizeObserver;
}
});
+
+describe("on a scroll", () => {
+ const originalResizeObserver = globalThis.ResizeObserver;
+ const originalIntersectionObserver = globalThis.IntersectionObserver;
+ const frames: FrameRequestCallback[] = [];
+ const harness = createHappyDomRootHarness();
+ let host: HTMLDivElement;
+ let root: ReturnType<typeof createRoot>;
+ let left = 0;
+ let top = 0;
+ let scrolled = 0;
+ let scrolledDown = 0;
+ let stripWidth = 1_000_000;
+ let renders = 0;
+
+ function Strip() {
+ renders += 1;
+ const [size, ref] = useThumbnailStripSize();
+ return (
+ <div>
+ <div ref={ref} data-in-view={`${size.inViewStart}-${size.inViewEnd}`} />
+ </div>
+ );
+ }
+
+ const mountStrips = (count: number) =>
+ act(async () =>
+ root.render(Array.from({ length: count }, (_, index) => <Strip key={index} />)),
+ );
+
+ const spans = () =>
+ [...host.querySelectorAll("[data-in-view]")].map((strip) => strip.getAttribute("data-in-view"));
+
+ /** The timeline scrolls so the strips' left edge lands at `nextLeft`, then one frame runs. */
+ const scrollTo = (nextLeft: number) => {
+ scrolled += left - nextLeft;
+ left = nextLeft;
+ act(() => host.dispatchEvent(new Event("scroll")));
+ expect(window.requestAnimationFrame).toHaveBeenCalledTimes(1);
+ act(() => frames.splice(0).forEach((frame) => frame(0)));
+ };
+
+ const neverReportedNear = () => {
+ globalThis.IntersectionObserver = class {
+ observe() {}
+ unobserve() {}
+ disconnect() {}
+ } as unknown as typeof IntersectionObserver;
+ };
+
+ beforeEach(() => {
+ globalThis.ResizeObserver = MockResizeObserver as unknown as typeof ResizeObserver;
+ globalThis.IntersectionObserver =
+ NearScreenIntersectionObserver as unknown as typeof IntersectionObserver;
+ vi.spyOn(window, "requestAnimationFrame").mockImplementation((frame) => frames.push(frame));
+ vi.spyOn(Element.prototype, "getBoundingClientRect").mockImplementation(
+ () => ({ left, top, width: stripWidth, height: 40 }) as DOMRect,
+ );
+ vi.spyOn(HTMLElement.prototype, "clientWidth", "get").mockImplementation(() => stripWidth);
+ left = 0;
+ top = 0;
+ scrolled = 0;
+ scrolledDown = 0;
+ stripWidth = 1_000_000;
+ renders = 0;
+ host = document.body.appendChild(document.createElement("div"));
+ host.setAttribute("data-timeline-scroll-viewport", "");
+ Object.defineProperty(host, "scrollLeft", { configurable: true, get: () => scrolled });
+ Object.defineProperty(host, "scrollTop", { configurable: true, get: () => scrolledDown });
+ root = harness.mount(host);
+ });
+
+ afterEach(() => {
+ frames.length = 0;
+ vi.restoreAllMocks();
+ globalThis.ResizeObserver = originalResizeObserver;
+ globalThis.IntersectionObserver = originalIntersectionObserver;
+ });
+
+ it("re-measures every near strip in one shared frame, so a full timeline lays out once", async () => {
+ await mountStrips(50);
+
+ scrollTo(-10_000);
+
+ expect(spans()).toEqual(Array(50).fill("9216-11776"));
+ });
+
+ it("keeps a visible strip measured when it is read after a scroll but before that scroll's frame", async () => {
+ stripWidth = 5_000;
+ left = -1_000;
+ await mountStrips(1);
+
+ scrolled += 3_000;
+ left = -4_000;
+ act(() => reportResize(stripWidth, 40));
+ act(() => host.dispatchEvent(new Event("scroll")));
+ act(() => frames.splice(0).forEach((frame) => frame(0)));
+
+ expect(spans()).toEqual(["3072-5000"]);
+ });
+
+ it("keeps a strip on screen measured after a move without a scroll, then a scroll", async () => {
+ await mountStrips(1);
+ expect(spans()).toEqual(["0-1536"]);
+
+ left = -3_000;
+ scrollTo(-1_000);
+ expect(spans()).toEqual(["0-2560"]);
+ });
+
+ it("measures a strip a vertical scroll brings on screen in that same frame", async () => {
+ neverReportedNear();
+ top = 5_000;
+ await mountStrips(1);
+ expect(spans()).toEqual(["0-0"]);
+
+ scrolledDown += 4_900;
+ top = 100;
+ act(() => host.dispatchEvent(new Event("scroll")));
+ act(() => frames.splice(0).forEach((frame) => frame(0)));
+
+ expect(spans()).toEqual(["0-1536"]);
+ });
+
+ it("keeps the tiles of a strip that ends just off screen", async () => {
+ await mountStrips(1);
+
+ scrollTo(-(stripWidth + 100));
+
+ expect(spans()).toEqual(["999424-1000000"]);
+ });
+
+ it("measures the strips a jump brings on screen in that same frame", async () => {
+ neverReportedNear();
+ left = 50_000;
+ await mountStrips(50);
+ expect(spans()).toEqual(Array(50).fill("0-0"));
+
+ scrollTo(0);
+
+ expect(spans()).toEqual(Array(50).fill("0-1536"));
+ });
+
+ it("does not re-render a strip wholly on screen when it moves", async () => {
+ stripWidth = 300;
+ left = 100;
+ await mountStrips(1);
+ const settled = renders;
+
+ scrollTo(700);
+
+ expect(renders).toBe(settled);
+ });
+
+ it("does not re-render strips that a jump carries from one side of the screen to the other", async () => {
+ left = 50_000;
+ await mountStrips(50);
+ const settled = renders;
+
+ scrollTo(-2_000_000);
+
+ expect(renders).toBe(settled);
+ });
+
+ it("keeps every tile of a short clip, even far from the screen", async () => {
+ stripWidth = 4_096;
+ top = 5_000;
+ await mountStrips(1);
+
+ expect(spans()).toEqual(["0-4096"]);
+ });
+
+ it("never re-measures a short clip on a scroll", async () => {
+ stripWidth = 300;
+ left = 100;
+ await mountStrips(1);
+ const reads = vi.mocked(Element.prototype.getBoundingClientRect);
+ reads.mockClear();
+
+ scrollTo(50);
+
+ expect(reads).not.toHaveBeenCalled();
+ });
+
+ it("mounts nothing for a strip far above or below the screen", async () => {
+ top = 5_000;
+ await mountStrips(1);
+
+ expect(spans()).toEqual(["0-0"]);
+ });
+
+ it("measures a strip as it comes near the screen, without waiting for a frame", async () => {
+ const reports: IntersectionObserverCallback[] = [];
+ globalThis.IntersectionObserver = class {
+ constructor(callback: IntersectionObserverCallback) {
+ reports.push(callback);
+ }
+ observe() {}
+ unobserve() {}
+ disconnect() {}
+ } as unknown as typeof IntersectionObserver;
+ await mountStrips(1);
+ const strip = host.querySelector("[data-in-view]")!;
+
+ left = -10_000;
+ const [presence] = reports;
+ act(() =>
+ presence!(
+ [
+ {
+ isIntersecting: true,
+ target: strip.parentElement!,
+ } as unknown as IntersectionObserverEntry,
+ ],
+ {} as IntersectionObserver,
+ ),
+ );
+
+ expect(frames).toHaveLength(0);
+ expect(strip.getAttribute("data-in-view")).toBe("9216-11776");
+ });
+
+ it("reads no strip far from the screen, however many clips the timeline mounts", async () => {
+ left = 50_000;
+ await mountStrips(50);
+ const reads = vi.mocked(Element.prototype.getBoundingClientRect);
+ reads.mockClear();
+
+ scrollTo(49_900);
+
+ expect(reads).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/studio/src/hooks/useThumbnailStripSize.ts b/packages/studio/src/hooks/useThumbnailStripSize.ts
index 35f646e354..d000c90ea9 100644
--- a/packages/studio/src/hooks/useThumbnailStripSize.ts
+++ b/packages/studio/src/hooks/useThumbnailStripSize.ts
@@ -1,31 +1,220 @@
-import { useCallback, useRef, useState } from "react";
-import { useMountEffect } from "./useMountEffect";
+import { useCallback, useState } from "react";
+import { flushSync } from "react-dom";
-interface StripSize {
+export interface StripSize {
width: number;
height: number;
+ inViewStart: number;
+ inViewEnd: number;
}
-const resize = (width: number, height: number) => (prev: StripSize) =>
- prev.width === width && prev.height === height ? prev : { width, height };
+const IN_VIEW_CHUNK_PX = 512;
-/** Size of the thumbnail's parent, kept current by a ResizeObserver; 0x0 until measured. */
+const chunk = (px: number, round: (n: number) => number) =>
+ Math.max(0, round(px / IN_VIEW_CHUNK_PX) * IN_VIEW_CHUNK_PX);
+
+const spanAt = (left: number) => ({
+ inViewStart: chunk(-left - IN_VIEW_CHUNK_PX, Math.floor),
+ inViewEnd: chunk(window.innerWidth - left + IN_VIEW_CHUNK_PX, Math.ceil),
+});
+
+// scrollMargin widens the band inside the timeline's own scroller too; TypeScript's DOM types lack it.
+const GAP_WARNING: IntersectionObserverInit & { scrollMargin: string } = {
+ rootMargin: `0px ${IN_VIEW_CHUNK_PX / 2}px`,
+ scrollMargin: `0px ${IN_VIEW_CHUNK_PX / 2}px`,
+};
+
+// A short strip keeps every tile, as before, so only long clips do work on a scroll.
+const SHORT_STRIP_MAX_PX = 8 * IN_VIEW_CHUNK_PX;
+const isShort = (width: number) => width <= SHORT_STRIP_MAX_PX;
+
+const EMPTY_STRIP: StripSize = { width: 0, height: 0, inViewStart: 0, inViewEnd: 0 };
+
+// Clamped to a measured strip, so a strip changes only when it crosses the screen's edge.
+const merge = (prev: StripSize, patch: Partial<StripSize>): StripSize => {
+ const next = { ...prev, ...patch };
+ const width = next.width > 0 ? Math.ceil(next.width) : Infinity;
+ next.inViewStart = isShort(width) ? 0 : Math.min(next.inViewStart, width);
+ next.inViewEnd = isShort(width) ? width : Math.min(next.inViewEnd, width);
+ return (Object.keys(next) as (keyof StripSize)[]).every((key) => next[key] === prev[key])
+ ? prev
+ : next;
+};
+
+type Apply = (patch: Partial<StripSize>) => void;
+
+interface Strip {
+ apply: Apply;
+ scroller: Element | null;
+ box: { left: number; top: number; width: number; height: number };
+ readAt: { x: number; y: number };
+ showing: boolean;
+}
+
+const NEAR_PX = IN_VIEW_CHUNK_PX / 2;
+
+// Each frame reads the strips showing tiles, which a move without a scroll may have carried, and the
+// strips whose last box, moved by the scroll since, lands near the screen; reads precede one commit.
+const strips = new Map<Element, Strip>();
+let users = 0;
+let frame = 0;
+let shared: {
+ resize: ResizeObserver;
+ presence: IntersectionObserver | null;
+ gaps: IntersectionObserver | null;
+} | null = null;
+
+const offsetOf = (scroller: Element | null) =>
+ scroller ? { x: scroller.scrollLeft, y: scroller.scrollTop } : { x: scrollX, y: scrollY };
+
+const isNear = ({
+ left,
+ top,
+ width,
+ height,
+}: {
+ left: number;
+ top: number;
+ width: number;
+ height: number;
+}) =>
+ left < innerWidth + NEAR_PX &&
+ left + width > -NEAR_PX &&
+ top < innerHeight + NEAR_PX &&
+ top + height > -NEAR_PX;
+
+const NOTHING_IN_VIEW = { inViewStart: 0, inViewEnd: 0 };
+
+const spanOf = (box: Strip["box"]) => (isNear(box) ? spanAt(box.left) : NOTHING_IN_VIEW);
+
+const read = (target: Element, strip: Strip) => {
+ const { left, top, width, height } = target.getBoundingClientRect();
+ strip.box = { left, top, width, height };
+ strip.readAt = offsetOf(strip.scroller);
+ return spanOf(strip.box);
+};
+
+const commit = (updates: (readonly [Apply, Partial<StripSize>])[]) =>
+ flushSync(() => updates.forEach(([apply, patch]) => apply(patch)));
+
+const refresh = () => {
+ frame = 0;
+ const offsetsNow = new Map<Element | null, { x: number; y: number }>();
+ const updates: (readonly [Apply, Partial<StripSize>])[] = [];
+ for (const [target, strip] of strips) {
+ if (isShort(strip.box.width)) continue;
+ let now = offsetsNow.get(strip.scroller);
+ if (!now) offsetsNow.set(strip.scroller, (now = offsetOf(strip.scroller)));
+ const { box, readAt } = strip;
+ const moved = {
+ left: box.left - (now.x - readAt.x),
+ top: box.top - (now.y - readAt.y),
+ width: box.width,
+ height: box.height,
+ };
+ if (strip.showing || isNear(moved)) updates.push([strip.apply, read(target, strip)]);
+ }
+ commit(updates);
+};
+
+const scheduleRefresh = () => {
+ if (!frame) frame = requestAnimationFrame(refresh);
+};
+
+const measure = (entries: { target: Element; size?: { width: number; height: number } }[]) =>
+ commit(
+ entries.flatMap(({ target, size }) => {
+ const strip = strips.get(target);
+ return strip ? [[strip.apply, { ...size, ...read(target, strip) }] as const] : [];
+ }),
+ );
+
+const onPresence = (entries: IntersectionObserverEntry[]) =>
+ measure(entries.filter((entry) => entry.isIntersecting));
+
+const onResize = (entries: ResizeObserverEntry[]) =>
+ measure(
+ entries.map(({ target, contentRect: { width, height } }) => ({
+ target,
+ size: { width, height },
+ })),
+ );
+
+const observeIntersections = (callback: IntersectionObserverCallback) =>
+ typeof IntersectionObserver === "undefined"
+ ? null
+ : new IntersectionObserver(callback, GAP_WARNING);
+
+function acquire() {
+ if (users++ === 0) {
+ shared = {
+ resize: new ResizeObserver(onResize),
+ presence: observeIntersections(onPresence),
+ gaps: observeIntersections(
+ (entries) => entries.some((entry) => entry.isIntersecting) && scheduleRefresh(),
+ ),
+ };
+ window.addEventListener("scroll", scheduleRefresh, { capture: true, passive: true });
+ }
+ return shared!;
+}
+
+function release() {
+ if (--users > 0) return;
+ shared?.resize.disconnect();
+ shared?.presence?.disconnect();
+ shared?.gaps?.disconnect();
+ shared = null;
+ window.removeEventListener("scroll", scheduleRefresh, { capture: true });
+ cancelAnimationFrame(frame);
+ frame = 0;
+}
+
+const watchGap = (gap: HTMLDivElement | null) => {
+ if (!gap) return;
+ const { gaps } = acquire();
+ gaps?.observe(gap);
+ return () => {
+ gaps?.unobserve(gap);
+ release();
+ };
+};
+
+/** Size of the thumbnail's parent and its span in the window, kept current on resize, scroll and moves. */
export function useThumbnailStripSize() {
- const [size, setSize] = useState<StripSize>({ width: 0, height: 0 });
- const observerRef = useRef<ResizeObserver | null>(null);
+ const [size, setSize] = useState(EMPTY_STRIP);
const ref = useCallback((element: HTMLDivElement | null) => {
- observerRef.current?.disconnect();
if (!element) return;
const target = element.parentElement ?? element;
- setSize(resize(target.clientWidth, target.clientHeight));
- observerRef.current = new ResizeObserver(([entry]) =>
- setSize(resize(entry.contentRect.width, entry.contentRect.height)),
- );
- observerRef.current.observe(target);
+ const { resize, presence } = acquire();
+ let current = EMPTY_STRIP;
+ const apply: Apply = (patch) => {
+ const next = merge(current, patch);
+ if (next === current) return;
+ current = next;
+ strip.showing = next.inViewEnd > 0;
+ setSize(next);
+ };
+ const scroller = target.closest("[data-timeline-scroll-viewport]");
+ const strip: Strip = {
+ apply,
+ scroller,
+ box: { left: 0, top: 0, width: 0, height: 0 },
+ readAt: { x: 0, y: 0 },
+ showing: false,
+ };
+ strips.set(target, strip);
+ apply({ width: target.clientWidth, height: target.clientHeight, ...read(target, strip) });
+ resize.observe(target);
+ presence?.observe(target);
+ return () => {
+ resize.unobserve(target);
+ presence?.unobserve(target);
+ strips.delete(target);
+ release();
+ };
}, []);
- useMountEffect(() => () => observerRef.current?.disconnect());
-
- return [size, ref] as const;
+ return [size, ref, watchGap] as const;
}
diff --git a/packages/studio/src/player/components/CompositionThumbnail.tsx b/packages/studio/src/player/components/CompositionThumbnail.tsx
index d0da0f9eb6..fe7844a009 100644
--- a/packages/studio/src/player/components/CompositionThumbnail.tsx
+++ b/packages/studio/src/player/components/CompositionThumbnail.tsx
@@ -7,6 +7,7 @@ import {
type ThumbnailRequest,
} from "../lib/thumbnailScheduler";
import { TIMELINE_VIEWPORT_BUDGETS } from "../lib/timelineViewportBudgets";
+import { ThumbnailTiles } from "./ThumbnailTiles";
import { computeThumbnailStrip, probeImageAspect } from "./thumbnailUtils";
import { studioApiFetch } from "../../utils/studioApiFetch";
@@ -142,7 +143,7 @@ export const CompositionThumbnail = memo(function CompositionThumbnail({
contentRevision = 0,
priority = "visible",
}: CompositionThumbnailProps) {
- const [container, setContainerRef] = useThumbnailStripSize();
+ const [container, setContainerRef, watchGap] = useThumbnailStripSize();
const url = buildCompositionThumbnailUrl({
previewUrl,
seekTime,
@@ -169,15 +170,18 @@ export const CompositionThumbnail = memo(function CompositionThumbnail({
return (
<div ref={setContainerRef} className="absolute inset-0 overflow-hidden">
{value && (
- <div
- className="absolute inset-0 flex"
+ <ThumbnailTiles
+ strip={container}
+ frameW={frameW}
+ frameCount={frameCount}
+ watchGap={watchGap}
style={{
animation: "hf-thumb-fade 200ms ease-out",
mixBlendMode:
"var(--timeline-composition-thumbnail-blend)" as CSSProperties["mixBlendMode"],
}}
>
- {Array.from({ length: frameCount }, (_, index) => (
+ {(index) => (
<div
key={index}
className="relative h-full shrink-0 overflow-hidden"
@@ -191,8 +195,8 @@ export const CompositionThumbnail = memo(function CompositionThumbnail({
style={{ opacity: "var(--timeline-composition-thumbnail-opacity)" }}
/>
</div>
- ))}
- </div>
+ )}
+ </ThumbnailTiles>
)}
{snapshot.status === "loading" && (
<div className="absolute inset-0 animate-pulse bg-text-0/[0.035]" />
diff --git a/packages/studio/src/player/components/ImageThumbnail.tsx b/packages/studio/src/player/components/ImageThumbnail.tsx
index b4528472e6..c3b8706088 100644
--- a/packages/studio/src/player/components/ImageThumbnail.tsx
+++ b/packages/studio/src/player/components/ImageThumbnail.tsx
@@ -3,6 +3,7 @@ import { useThumbnailLease } from "../../hooks/useThumbnailLease";
import { useThumbnailStripSize } from "../../hooks/useThumbnailStripSize";
import { createThumbnailKey, type ThumbnailPriority } from "../lib/thumbnailScheduler";
import { decodeImageThumbnail } from "../lib/thumbnailImageDecoder";
+import { ThumbnailTiles } from "./ThumbnailTiles";
import { computeThumbnailStrip } from "./thumbnailUtils";
export interface ImageThumbnailProps {
@@ -25,7 +26,7 @@ export const ImageThumbnail = memo(function ImageThumbnail({
priority = "visible",
rich = false,
}: ImageThumbnailProps) {
- const [container, setContainerRef] = useThumbnailStripSize();
+ const [container, setContainerRef, watchGap] = useThumbnailStripSize();
const request = useMemo(
() => ({
key: createThumbnailKey({ kind: "image", source: imageSrc, rich: Number(rich) }),
@@ -46,8 +47,13 @@ export const ImageThumbnail = memo(function ImageThumbnail({
return (
<div ref={setContainerRef} className="absolute inset-0 overflow-hidden">
{value?.kind === "image" && (
- <div className="absolute inset-0 flex">
- {Array.from({ length: frameCount }, (_, index) => (
+ <ThumbnailTiles
+ strip={container}
+ frameW={frameW}
+ frameCount={frameCount}
+ watchGap={watchGap}
+ >
+ {(index) => (
<div
key={index}
className="relative h-full shrink-0 overflow-hidden bg-neutral-900"
@@ -61,8 +67,8 @@ export const ImageThumbnail = memo(function ImageThumbnail({
className="absolute inset-0 h-full w-full object-cover"
/>
</div>
- ))}
- </div>
+ )}
+ </ThumbnailTiles>
)}
{snapshot.status === "loading" && (
<div
diff --git a/packages/studio/src/player/components/ThumbnailTiles.test.tsx b/packages/studio/src/player/components/ThumbnailTiles.test.tsx
new file mode 100644
index 0000000000..1ed4a466c9
--- /dev/null
+++ b/packages/studio/src/player/components/ThumbnailTiles.test.tsx
@@ -0,0 +1,30 @@
+// @vitest-environment happy-dom
+import { act } from "react";
+import { expect, it } from "vitest";
+import { createHappyDomRootHarness } from "./testRootHarness";
+import { ThumbnailTiles } from "./ThumbnailTiles";
+
+Reflect.set(globalThis, "IS_REACT_ACT_ENVIRONMENT", true);
+
+const harness = createHappyDomRootHarness();
+
+it("places the first mounted tile where the full strip would have it", () => {
+ const host = document.body.appendChild(document.createElement("div"));
+ act(() =>
+ harness.mount(host).render(
+ <ThumbnailTiles
+ strip={{ width: 10_000, height: 40, inViewStart: 1_024, inViewEnd: 2_048 }}
+ frameW={100}
+ frameCount={100}
+ watchGap={() => {}}
+ >
+ {(index) => <i key={index} data-index={index} />}
+ </ThumbnailTiles>,
+ ),
+ );
+
+ const row = host.firstElementChild as HTMLElement;
+ const first = Number(row.querySelector("i")!.dataset.index);
+ expect(first).toBe(10);
+ expect(row.style.paddingLeft).toBe(`${first * 100}px`);
+});
diff --git a/packages/studio/src/player/components/ThumbnailTiles.tsx b/packages/studio/src/player/components/ThumbnailTiles.tsx
new file mode 100644
index 0000000000..2c7d146ba2
--- /dev/null
+++ b/packages/studio/src/player/components/ThumbnailTiles.tsx
@@ -0,0 +1,41 @@
+import type { CSSProperties, ReactNode } from "react";
+import type { StripSize } from "../../hooks/useThumbnailStripSize";
+
+interface ThumbnailTilesProps {
+ strip: StripSize;
+ frameW: number;
+ frameCount: number;
+ watchGap: (gap: HTMLDivElement | null) => void | (() => void);
+ style?: CSSProperties;
+ children: (index: number) => ReactNode;
+}
+
+/** A clip's tile row, mounting only the tiles in the strip's on-screen span, each at its own place. */
+export function ThumbnailTiles({
+ strip,
+ frameW,
+ frameCount,
+ watchGap,
+ style,
+ children,
+}: ThumbnailTilesProps) {
+ const end = Math.min(frameCount, Math.ceil(strip.inViewEnd / frameW));
+ const first = Math.min(end, Math.floor(strip.inViewStart / frameW));
+ return (
+ <div className="absolute inset-0 flex" style={{ ...style, paddingLeft: first * frameW }}>
+ <div
+ ref={watchGap}
+ data-thumbnail-gap="start"
+ className="pointer-events-none absolute inset-y-0 left-0"
+ style={{ width: first * frameW }}
+ />
+ {Array.from({ length: end - first }, (_, offset) => children(first + offset))}
+ <div
+ ref={watchGap}
+ data-thumbnail-gap="end"
+ className="pointer-events-none absolute inset-y-0 right-0"
+ style={{ left: end * frameW }}
+ />
+ </div>
+ );
+}
diff --git a/packages/studio/src/player/components/VideoThumbnail.test.tsx b/packages/studio/src/player/components/VideoThumbnail.test.tsx
index 8e8cb34803..4dcd72ca15 100644
--- a/packages/studio/src/player/components/VideoThumbnail.test.tsx
+++ b/packages/studio/src/player/components/VideoThumbnail.test.tsx
@@ -1,10 +1,10 @@
// @vitest-environment happy-dom
import React, { act } from "react";
-import { createRoot, type Root } from "react-dom/client";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { MockResizeObserver, reportResize } from "../../hooks/resizeObserverTestUtils";
import { thumbnailScheduler } from "../lib/thumbnailScheduler";
import { decodeVideoThumbnail } from "../lib/thumbnailVideoDecoder";
+import { createHappyDomRootHarness } from "./testRootHarness";
import { VideoThumbnail } from "./VideoThumbnail";
vi.mock("../lib/thumbnailVideoDecoder", () => ({ decodeVideoThumbnail: vi.fn() }));
@@ -15,28 +15,26 @@ Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", {
});
let host: HTMLDivElement;
-let root: Root | null = null;
beforeEach(() => {
globalThis.ResizeObserver = MockResizeObserver as unknown as typeof ResizeObserver;
- host = document.createElement("div");
- document.body.append(host);
+ host = document.body.appendChild(document.createElement("div"));
});
afterEach(() => {
- act(() => root?.unmount());
- root = null;
thumbnailScheduler.invalidateProject("p");
vi.clearAllMocks();
- document.body.innerHTML = "";
});
+// Registered after the reset above, so each test's strip unmounts before the cache is cleared.
+const harness = createHappyDomRootHarness();
+
async function render(width = 0, height = 40) {
Object.defineProperty(host, "clientWidth", { configurable: true, value: width });
Object.defineProperty(host, "clientHeight", { configurable: true, value: height });
- root = createRoot(host);
+ const root = harness.mount(host);
await act(async () => {
- root!.render(
+ root.render(
<VideoThumbnail
videoSrc="/api/projects/p/preview/assets/clip.mp4"
label=""
@@ -95,6 +93,115 @@ describe("VideoThumbnail", () => {
expect(tiles).toEqual(["blob:0", "blob:2", "blob:4", "blob:5", "blob:7"]);
});
+ describe("on a 10-minute clip at full zoom", () => {
+ const frames: FrameRequestCallback[] = [];
+ const originalIntersectionObserver = globalThis.IntersectionObserver;
+ let left = -432_000;
+ let scrolled = 0;
+ let reportGapNearScreen: () => void = () => {};
+
+ beforeEach(() => {
+ vi.mocked(decodeVideoThumbnail).mockResolvedValue({
+ value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 },
+ weight: 256,
+ });
+ vi.spyOn(window, "requestAnimationFrame").mockImplementation((frame) => frames.push(frame));
+ globalThis.IntersectionObserver = class {
+ observed = new Set<Element>();
+ constructor(private readonly callback: IntersectionObserverCallback) {
+ reportGapNearScreen = () =>
+ callback(
+ [...this.observed].map(
+ (target) => ({ isIntersecting: true, target }) as IntersectionObserverEntry,
+ ),
+ this as unknown as IntersectionObserver,
+ );
+ }
+ observe(target: Element) {
+ this.observed.add(target);
+ queueMicrotask(() =>
+ this.callback(
+ [{ isIntersecting: true, target } as IntersectionObserverEntry],
+ this as unknown as IntersectionObserver,
+ ),
+ );
+ }
+ unobserve(target: Element) {
+ this.observed.delete(target);
+ }
+ disconnect() {
+ this.observed.clear();
+ }
+ } as unknown as typeof IntersectionObserver;
+ left = -432_000;
+ scrolled = 0;
+ host.getBoundingClientRect = () =>
+ ({ left, top: 0, width: 600 * 1440, height: 40 }) as DOMRect;
+ const scroller = document.body.appendChild(document.createElement("div"));
+ scroller.setAttribute("data-timeline-scroll-viewport", "");
+ Object.defineProperty(scroller, "scrollLeft", { configurable: true, get: () => scrolled });
+ scroller.append(host);
+ });
+
+ const scrollTo = (nextLeft: number) => {
+ scrolled += left - nextLeft;
+ left = nextLeft;
+ host.dispatchEvent(new Event("scroll"));
+ };
+
+ afterEach(() => {
+ frames.length = 0;
+ vi.restoreAllMocks();
+ globalThis.IntersectionObserver = originalIntersectionObserver;
+ });
+
+ const settle = () =>
+ act(async () => {
+ await Promise.resolve();
+ for (const frame of frames.splice(0)) frame(0);
+ });
+
+ const expectTilesCoverTheWindow = () => {
+ const tiles = host.querySelectorAll("img").length;
+ const skipped = parseFloat(
+ (host.querySelector("img")!.closest(".flex") as HTMLElement).style.paddingLeft,
+ );
+ expect(tiles).toBeLessThan(60);
+ expect(skipped).toBeLessThanOrEqual(-left);
+ expect(skipped + tiles * 71).toBeGreaterThanOrEqual(-left + window.innerWidth);
+ };
+
+ it("mounts only the tiles in view, and follows each scroll", async () => {
+ await render(600 * 1440, 40);
+ expectTilesCoverTheWindow();
+
+ for (const scrolledTo of [100_000, 300_000]) {
+ scrollTo(-scrolledTo);
+ await settle();
+ expectTilesCoverTheWindow();
+ }
+ });
+
+ it("mounts no tiles once the clip is wholly off screen", async () => {
+ await render(600 * 1440, 40);
+
+ scrollTo(-(600 * 1440 + 5_000));
+ await settle();
+
+ expect(host.querySelectorAll("img")).toHaveLength(0);
+ });
+
+ it("follows the strip when something else moves it, as a drag moves its ghost", async () => {
+ await render(600 * 1440, 40);
+
+ left = -430_500;
+ reportGapNearScreen();
+ await settle();
+
+ expectTilesCoverTheWindow();
+ });
+ });
+
it("issues a single decode job for a narrow clip", async () => {
vi.mocked(decodeVideoThumbnail).mockResolvedValue({
value: { kind: "image", url: "blob:poster", aspect: 16 / 9 },
diff --git a/packages/studio/src/player/components/VideoThumbnail.tsx b/packages/studio/src/player/components/VideoThumbnail.tsx
index ccfb6bc62b..9d42c11421 100644
--- a/packages/studio/src/player/components/VideoThumbnail.tsx
+++ b/packages/studio/src/player/components/VideoThumbnail.tsx
@@ -7,6 +7,7 @@ import {
type ThumbnailSnapshot,
} from "../lib/thumbnailScheduler";
import { decodeVideoThumbnail } from "../lib/thumbnailVideoDecoder";
+import { ThumbnailTiles } from "./ThumbnailTiles";
import { computeThumbnailStrip, quantizeThumbnailFrameCount } from "./thumbnailUtils";
interface VideoThumbnailProps {
@@ -85,7 +86,7 @@ export const VideoThumbnail = memo(function VideoThumbnail({
sessionEpoch = 0,
priority = "visible",
}: VideoThumbnailProps) {
- const [container, setContainerRef] = useThumbnailStripSize();
+ const [container, setContainerRef, watchGap] = useThumbnailStripSize();
const requestFrameCount = quantizeThumbnailFrameCount(
computeThumbnailStrip(container.width, 16 / 9, container.height).frameCount,
);
@@ -122,8 +123,13 @@ export const VideoThumbnail = memo(function VideoThumbnail({
return (
<div ref={setContainerRef} className="absolute inset-0 overflow-hidden">
{urls.length > 0 && (
- <div className="absolute inset-0 flex">
- {Array.from({ length: frameCount }, (_, index) => {
+ <ThumbnailTiles
+ strip={container}
+ frameW={frameW}
+ frameCount={frameCount}
+ watchGap={watchGap}
+ >
+ {(index) => {
const src = urls[Math.round((index * (urls.length - 1)) / Math.max(1, frameCount - 1))];
return (
<div
@@ -139,8 +145,8 @@ export const VideoThumbnail = memo(function VideoThumbnail({
/>
</div>
);
- })}
- </div>
+ }}
+ </ThumbnailTiles>
)}
{snapshot.status === "loading" && urls.length === 0 && (
<div