Skip to content

Open source CapCut alternative.

TypeScript

feat(site): filmstrip tiles show true frames and split at scene cuts [rebuild]

David Le8ea06e1137d5
diff --git a/site/src/cut/components/Timeline.tsx b/site/src/cut/components/Timeline.tsx
index 9461ab56..eb1a22a8 100644
--- a/site/src/cut/components/Timeline.tsx
+++ b/site/src/cut/components/Timeline.tsx
@@ -43,6 +43,7 @@ import { useExports } from "@/cut/lib/exportStore";
import { isDragActive, startDrag, subscribeDragActive } from "@/cut/lib/drag";
import { CLIP_GAP, startLaneMove, startLaneTrim, type LaneDrag, type LaneKind } from "@/cut/lib/laneTracks";
import { downloadMedia, ensurePeaks, importImage, importStockMusic, importStockVideo, peekEdgeFrame, requestEdgeFrame, revealMedia, stripFailedFor, subscribeStripStatus } from "@/cut/lib/media";
+import { planFilmstrip, type FilmTile } from "@/cut/lib/filmstrip";
import { track0Clips, laneGapAt, sameLane, type LaneRef, clipLen, clipSpeed, getClipSpans, overlayLaneOrder, overlayLayers, projectDuration, resolveTransitions, rippleInsert, useEditor } from "@/cut/lib/store";
import type { VideoTrackPlacement } from "@/cut/lib/store";
import { playheadAt, setSkim, skimAt, subscribePlayhead, usePlayhead, useSkim } from "@/cut/lib/playhead";
@@ -3512,14 +3513,19 @@ function ClipView({
() => stripFailedFor(asset.id),
() => false
);
+ const boxRef = useRef<HTMLDivElement>(null);
+ // Bumped as tile captures land, so the memo re-plans and picks them up.
+ const [filmGen, onTileFrame] = useReducer((x: number) => x + 1, 0);
const filmstrip = useMemo(
() =>
filmstripFrames(asset, filmIn, w, pps, speed, VIDEO_H - 4, 26, {
start: startFrame,
end: endFrame,
}),
- [asset, filmIn, w, pps, speed, startFrame, endFrame]
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ [asset, filmIn, w, pps, speed, startFrame, endFrame, filmGen]
);
+ useTileFrames(boxRef, clip.id, asset, filmstrip, filmIn, w, pps, speed, onTileFrame);
// The move gesture is the shared lane behavior (parting, snapping); its
// verticality is the video placement system — the other tracks and the new
@@ -3561,6 +3567,7 @@ function ClipView({
// Inline so it beats SELECTED_SHADOW's z-10 class on the same element.
zIndex: drag ? 20 : undefined,
}}
+ ref={boxRef}
data-tl-sel={`clip:${clip.id}`}
onPointerDown={(e) => startLaneMove(e, lane, clip.id, ui)}
onContextMenu={(e) => {
@@ -3923,7 +3930,9 @@ function DropGhostFilm({
* tiles at each end). Tiles keep the asset's aspect until the box would blow
* the tile cap; past that they widen by whole grid cells, still on the grid.
* The first and last tiles pin to the segment's exact boundary frames once
- * captured; middle tiles keep the nearest pre-sampled thumb. */
+ * captured; middle tiles show the nearest pre-sampled thumb until the zoom
+ * outruns the strip's sampling, when each carries a true frame at its own
+ * moment as its capture lands (see `planFilmstrip`). */
function filmstripFrames(
asset: MediaAsset | undefined,
filmIn: number,
@@ -3933,24 +3942,21 @@ function filmstripFrames(
tileH: number,
minTileW: number,
edges?: { start: string | null; end: string | null }
-) {
+): FilmTile[] {
if (!asset?.thumbs?.length || !asset.thumbStep) return [];
- const aspect = (asset.width ?? 16) / Math.max(1, asset.height ?? 9);
- const natural = Math.max(minTileW, Math.round(tileH * aspect));
- const cells = Math.max(1, Math.ceil(w / natural));
- const imgW = natural * Math.ceil(cells / 120);
- const stepT = (imgW / pps) * speed;
- const filmOut = filmIn + (w / pps) * speed;
- const first = Math.max(0, Math.floor(filmIn / stepT));
- const last = Math.max(first, Math.ceil(filmOut / stepT) - 1);
- const frames = Array.from({ length: last - first + 1 }, (_, i) => {
- const k = first + i;
- const timeAt = (k + 0.5) * stepT;
- const idx = Math.min(
- asset.thumbs!.length - 1,
- Math.max(0, Math.floor(timeAt / asset.thumbStep!))
- );
- return { src: asset.thumbs![idx], left: ((k * stepT - filmIn) / speed) * pps, width: imgW };
+ const frames = planFilmstrip({
+ thumbs: asset.thumbs,
+ thumbStep: asset.thumbStep,
+ duration: asset.duration,
+ aspect: (asset.width ?? 16) / Math.max(1, asset.height ?? 9),
+ filmIn,
+ w,
+ pps,
+ speed,
+ tileH,
+ minTileW,
+ cuts: asset.sceneCuts,
+ exactFrame: asset.type === "video" ? (t) => peekEdgeFrame(asset.url, t) : undefined,
});
if (edges?.start) frames[0] = { ...frames[0], src: edges.start };
if (edges?.end && frames.length > 1) {
@@ -3959,6 +3965,93 @@ function filmstripFrames(
return frames;
}
+/** How far past the scroller's view a tile still gets its true frame, so a
+ * small scroll shows captured tiles at once. */
+const TILE_CAPTURE_MARGIN = 240;
+
+/** Capture the true frames a zoomed strip's tiles ask for. The plan names a
+ * capture time on each tile the pre-sampled strip is too coarse for; this
+ * effect requests the ones inside the scroller's view (plus a margin) through
+ * the edge reader's background lane and re-renders as they land. Scroll fills
+ * in newly visible tiles; a zoom or trim re-aims the same per-tile slots, so
+ * superseded captures fall away on their own. */
+function useTileFrames(
+ boxRef: React.RefObject<HTMLDivElement | null>,
+ clipId: string,
+ asset: MediaAsset | undefined,
+ tiles: FilmTile[],
+ filmIn: number,
+ w: number,
+ pps: number,
+ speed: number,
+ onFrame: () => void
+) {
+ // The landed-frame re-render hands this hook a fresh `tiles` array with the
+ // same geometry; the ref keeps that from re-running the capture effect,
+ // while scroll and the effect body always read the newest plan.
+ const tilesRef = useRef(tiles);
+ useEffect(() => {
+ tilesRef.current = tiles;
+ });
+ const wanted = tiles.some((t) => t.wantT !== null);
+ useEffect(() => {
+ const box = boxRef.current;
+ const url = asset?.type === "video" ? asset.url : null;
+ if (!box || !url || !wanted) return;
+ const scroller = box.closest("[data-tl-scroll]") as HTMLElement | null;
+ let live = true;
+ let scheduled = false;
+ let bumping = false;
+ // One request per capture time per plan: a failed read stays quiet until
+ // a zoom or trim re-plans, and a scroll walk re-asks for nothing.
+ const asked = new Set<string>();
+ const landed = () => {
+ if (bumping) return;
+ bumping = true;
+ requestAnimationFrame(() => {
+ bumping = false;
+ if (live) onFrame();
+ });
+ };
+ const request = () => {
+ scheduled = false;
+ if (!live) return;
+ let lo = 0;
+ let hi = w;
+ if (scroller) {
+ const boxR = box.getBoundingClientRect();
+ const view = scroller.getBoundingClientRect();
+ lo = view.left - boxR.left - TILE_CAPTURE_MARGIN;
+ hi = view.right - boxR.left + TILE_CAPTURE_MARGIN;
+ }
+ tilesRef.current.forEach((t, i) => {
+ if (t.wantT === null || t.exact) return;
+ if (t.left + t.width < lo || t.left > hi) return;
+ const key = `${i}:${t.wantT.toFixed(2)}`;
+ if (asked.has(key)) return;
+ asked.add(key);
+ void requestEdgeFrame(`${clipId}:tile:${i}`, url, t.wantT, undefined, {
+ background: true,
+ }).then((src) => {
+ if (live && src) landed();
+ });
+ });
+ };
+ const schedule = () => {
+ if (scheduled) return;
+ scheduled = true;
+ requestAnimationFrame(request);
+ };
+ schedule();
+ scroller?.addEventListener("scroll", schedule, { passive: true });
+ return () => {
+ live = false;
+ scroller?.removeEventListener("scroll", schedule);
+ };
+ // Geometry names the plan; `tiles` itself rides the ref above.
+ }, [boxRef, clipId, asset, wanted, filmIn, w, pps, speed, onFrame]);
+}
+
/** The exact source frame at a clip edge. Returns null (the nearest sampled
* thumb shows instead) until the capture lands; a changed edge time falls back
* immediately so a trim drag never shows a stale exact frame. */
diff --git a/site/src/cut/lib/devHooks.ts b/site/src/cut/lib/devHooks.ts
index 80ea064e..dcb6afe6 100644
--- a/site/src/cut/lib/devHooks.ts
+++ b/site/src/cut/lib/devHooks.ts
@@ -10,7 +10,7 @@
import { renderProjectToMp4 } from "./exportRender";
import { useGenerate } from "./generate";
import { useGenScene } from "./genScene";
-import { enrichAsset, importFileToProject } from "./media";
+import { edgeFramesPending, enrichAsset, importFileToProject } from "./media";
import { awaitingFrame, startTrace, stopTrace } from "./perfTrace";
import { playheadAt } from "./playhead";
import { projectDuration, useEditor } from "./store";
@@ -23,6 +23,9 @@ export function installDevHooks(): void {
useGenScene,
importFileToProject,
enrichAsset,
+ // The filmstrip eval waits on zero pending captures before it reads the
+ // strip's tiles back out of the DOM.
+ edgeFramesPending,
playheadAt,
// The export eval renders a doc through the tab's own pipeline and reads
// the bytes back out.
diff --git a/site/src/cut/lib/filmstrip.test.ts b/site/src/cut/lib/filmstrip.test.ts
new file mode 100644
index 00000000..1baf4e30
--- /dev/null
+++ b/site/src/cut/lib/filmstrip.test.ts
@@ -0,0 +1,231 @@
+import { describe, expect, test } from "bun:test";
+import {
+ FILM_TILE_CAP,
+ MIN_SUB_TILE_PX,
+ pickThumbTimes,
+ planFilmstrip,
+ WANT_GRID_S,
+} from "./filmstrip";
+
+/** The user's case: a ~24s clip with fast scene cuts carries 12 pre-sampled
+ * thumbs, one every ~2s. */
+const DURATION = 23.66;
+const THUMB_STEP = DURATION / 12;
+const thumbs = Array.from({ length: 12 }, (_, i) => `thumb-${i}`);
+
+/** A whole-clip box drawn at `pps`, its width following the zoom the way a
+ * real clip's does. */
+const plan = (over: Partial<Parameters<typeof planFilmstrip>[0]> = {}) =>
+ planFilmstrip({
+ thumbs,
+ thumbStep: THUMB_STEP,
+ duration: DURATION,
+ aspect: 16 / 9,
+ filmIn: 0,
+ w: (over.duration ?? DURATION) * (over.pps ?? 60),
+ pps: 60,
+ speed: 1,
+ tileH: 60,
+ minTileW: 26,
+ ...over,
+ });
+
+describe("planFilmstrip", () => {
+ test("tiles cover the drawn width on a source-time grid", () => {
+ const tiles = plan();
+ expect(tiles.length).toBeGreaterThan(0);
+ expect(tiles[0].left).toBeLessThanOrEqual(0);
+ const end = tiles[tiles.length - 1];
+ expect(end.left + end.width).toBeGreaterThanOrEqual(DURATION * 60);
+ for (let i = 1; i < tiles.length; i++) {
+ expect(tiles[i].left).toBeCloseTo(tiles[i - 1].left + tiles[i - 1].width, 6);
+ }
+ });
+
+ test("each tile shows the pre-sampled thumb nearest its own moment", () => {
+ for (const t of plan()) {
+ const idx = Number(t.src.slice("thumb-".length));
+ const sampledAt = (idx + 0.5) * THUMB_STEP;
+ // The end tiles overhang the source and clamp to the last thumb.
+ const inSource = Math.min(t.srcT, DURATION);
+ expect(Math.abs(sampledAt - inSource)).toBeLessThanOrEqual(THUMB_STEP / 2 + 1e-9);
+ }
+ });
+
+ test("zoomed out past the strip's sampling, no tile asks for a capture", () => {
+ const tiles = plan({ pps: 30 });
+ expect(tiles.every((t) => t.wantT === null && !t.exact)).toBe(true);
+ });
+
+ test("zoomed in, every tile asks for a capture at its own midpoint", () => {
+ // Half the capture grid is the whole allowance: a capture pulled further
+ // toward a tile's edge lands on the neighboring scene whenever a cut sits
+ // inside the tile, and the strip flips a whole tile early.
+ for (const pps of [120, 200, 300, 480, 800]) {
+ const tiles = plan({ pps });
+ for (const t of tiles) {
+ expect(t.wantT === null).toBe(false);
+ if (t.wantT! <= 0 || t.wantT! >= DURATION - 0.05) continue;
+ expect(Math.abs(t.wantT! - t.srcT)).toBeLessThanOrEqual(WANT_GRID_S / 2 + 1e-9);
+ expect(t.exact).toBe(false);
+ }
+ }
+ });
+
+ test("capture times land on fine-grid centers, clear of frame boundaries", () => {
+ // Grid times keep cache keys stable while the strip scrolls, trims, and
+ // re-plans; centers keep captures off the whole- and half-second moments
+ // where scene cuts live, so a straddling tile stays on its midpoint's
+ // side of the cut.
+ for (const t of plan({ pps: 300 })) {
+ if (t.wantT === null || t.wantT <= 0 || t.wantT >= DURATION - 0.05) continue;
+ const cells = t.wantT / WANT_GRID_S - 0.5;
+ expect(Math.abs(cells - Math.round(cells))).toBeLessThan(1e-6);
+ }
+ });
+
+ test("a tile whose midpoint sits a hair before a cut stays before it", () => {
+ // The regression the timeline showed: a midpoint at ~2.97 must capture
+ // below 3.0, never at it — the frame at 3.0 is the next scene's first.
+ const pps = 270; // stepT ≈ 0.396 puts a tile midpoint at ~2.972
+ const nearCut = plan({ pps }).filter((t) => t.srcT > 2.95 && t.srcT < 3);
+ expect(nearCut.length).toBeGreaterThan(0);
+ for (const t of nearCut) expect(t.wantT!).toBeLessThan(3);
+ });
+
+ test("capture times stay inside the source", () => {
+ for (const t of plan({ pps: 800 })) {
+ expect(t.wantT!).toBeGreaterThanOrEqual(0);
+ expect(t.wantT!).toBeLessThanOrEqual(DURATION - 0.05 + 1e-9);
+ }
+ });
+
+ test("a captured frame replaces the thumb and marks the tile exact", () => {
+ const tiles = plan({ pps: 300, exactFrame: (t) => (t < 12 ? `frame-${t.toFixed(3)}` : null) });
+ for (const t of tiles) {
+ if (t.wantT! < 12) {
+ expect(t.exact).toBe(true);
+ expect(t.src).toBe(`frame-${t.wantT!.toFixed(3)}`);
+ } else {
+ expect(t.exact).toBe(false);
+ expect(t.src.startsWith("thumb-")).toBe(true);
+ }
+ }
+ });
+
+ test("a trimmed left edge keeps tiles on the same source grid", () => {
+ const pps = 300;
+ const whole = plan({ pps });
+ const trimmed = plan({ pps, filmIn: 3.2, w: (DURATION - 3.2) * pps });
+ const wholeTimes = new Set(whole.map((t) => t.srcT.toFixed(4)));
+ for (const t of trimmed) expect(wholeTimes.has(t.srcT.toFixed(4))).toBe(true);
+ });
+
+ test("past the tile cap, tiles widen by whole cells and stay capped", () => {
+ const tiles = plan({ pps: 800, duration: 600, w: 600 * 800, thumbStep: 600 / 24 });
+ expect(tiles.length).toBeLessThanOrEqual(FILM_TILE_CAP + 1);
+ });
+
+ test("an empty strip plans nothing", () => {
+ expect(plan({ thumbs: [] })).toEqual([]);
+ });
+
+ test("a scene cut inside a refining tile splits it at the cut", () => {
+ const pps = 300;
+ const cut = 3.0;
+ const tiles = plan({ pps, cuts: [cut] });
+ // Some tile boundary sits exactly on the cut...
+ const edges = tiles.map((t) => t.left);
+ expect(edges.some((x) => Math.abs(x - cut * pps) < 1e-6)).toBe(true);
+ // ...and each side captures strictly on its own side of it.
+ for (const t of tiles) {
+ const a = t.left / pps;
+ const b = (t.left + t.width) / pps;
+ if (b <= cut + 1e-9) expect(t.wantT!).toBeLessThan(cut);
+ if (a >= cut - 1e-9) expect(t.wantT!).toBeGreaterThan(cut);
+ }
+ });
+
+ test("split tiles still tile the strip without gaps", () => {
+ const tiles = plan({ pps: 300, cuts: [1.7, 3.0, 9.13] });
+ for (let i = 1; i < tiles.length; i++) {
+ expect(tiles[i].left).toBeCloseTo(tiles[i - 1].left + tiles[i - 1].width, 6);
+ }
+ });
+
+ test("a cut a sliver from a tile boundary does not split off a sub-pixel tile", () => {
+ const pps = 300;
+ const stepT = plan({ pps })[1].width / pps;
+ const sliver = stepT + MIN_SUB_TILE_PX / pps / 2; // just inside tile 1
+ const tiles = plan({ pps, cuts: [sliver] });
+ for (const t of tiles) expect(t.width).toBeGreaterThanOrEqual(MIN_SUB_TILE_PX - 1e-6);
+ });
+
+ test("zoomed out, cuts split nothing", () => {
+ const zoomedOut = plan({ pps: 30, cuts: [3.0, 6.0, 9.0] });
+ expect(zoomedOut).toEqual(plan({ pps: 30 }));
+ });
+
+ test("no cuts leaves the plan unchanged", () => {
+ expect(plan({ pps: 300, cuts: [] })).toEqual(plan({ pps: 300 }));
+ });
+});
+
+describe("pickThumbTimes", () => {
+ /** Probes every `at` seconds with scene cuts at the given times. */
+ const probesWithCuts = (duration: number, at: number, cuts: number[]) => {
+ const out = [];
+ for (let t = at / 2; t < duration; t += at) {
+ const cut = cuts.some((c) => t - at < c && c <= t);
+ out.push({ t, cut });
+ }
+ return out;
+ };
+
+ test("no probes: every bucket grabs its midpoint", () => {
+ const times = pickThumbTimes(12, 2, 24, []);
+ times.forEach((t, k) => expect(t).toBeCloseTo(Math.min(24 - 0.05, (k + 0.5) * 2), 6));
+ });
+
+ test("no cuts: every bucket grabs near its midpoint", () => {
+ const times = pickThumbTimes(12, 2, 24, probesWithCuts(24, 0.25, []));
+ times.forEach((t, k) => {
+ expect(Math.abs(t - (k + 0.5) * 2)).toBeLessThanOrEqual(0.25);
+ });
+ });
+
+ test("a cut late in a bucket: the grab stays with the dominant scene", () => {
+ // The regression the timeline showed: a scene cut at 3.0s inside the
+ // bucket [2, 4) — most of the bucket is the first scene, so the thumb
+ // must come from before the cut... with the cut at the bucket's own
+ // 3/4 point, the run before it is the longer one.
+ const times = pickThumbTimes(12, 2, 24, probesWithCuts(24, 0.25, [3.5]));
+ const bucket1 = times[1]; // bucket [2, 4)
+ expect(bucket1).toBeLessThan(3.5);
+ expect(bucket1).toBeGreaterThanOrEqual(2);
+ });
+
+ test("a cut early in a bucket: the grab jumps past it", () => {
+ const times = pickThumbTimes(12, 2, 24, probesWithCuts(24, 0.25, [2.5]));
+ const bucket1 = times[1]; // bucket [2, 4): the scene after 2.5 owns it
+ expect(bucket1).toBeGreaterThan(2.5);
+ expect(bucket1).toBeLessThan(4);
+ });
+
+ test("grabs never land right on a cut", () => {
+ const cuts = [3, 6, 9, 12, 15, 18, 21];
+ const times = pickThumbTimes(12, 2, 24, probesWithCuts(24, 0.25, cuts));
+ for (const t of times) {
+ for (const c of cuts) expect(Math.abs(t - c)).toBeGreaterThan(0.1);
+ }
+ });
+
+ test("grabs stay inside their own bucket, ascending", () => {
+ const times = pickThumbTimes(12, 2, 24, probesWithCuts(24, 0.25, [1, 4.2, 17.9]));
+ times.forEach((t, k) => {
+ expect(t).toBeGreaterThanOrEqual(k * 2);
+ expect(t).toBeLessThan((k + 1) * 2);
+ });
+ for (let i = 1; i < times.length; i++) expect(times[i]).toBeGreaterThan(times[i - 1]);
+ });
+});
diff --git a/site/src/cut/lib/filmstrip.ts b/site/src/cut/lib/filmstrip.ts
new file mode 100644
index 00000000..56fd158f
--- /dev/null
+++ b/site/src/cut/lib/filmstrip.ts
@@ -0,0 +1,173 @@
+/**
+ * Filmstrip tile planning: which frame each tile of a clip's strip shows.
+ *
+ * Tiles sit on a source-time grid, so the strip holds still while a trimmed
+ * edge sweeps across it. Each tile spans `stepT` seconds of source; the asset
+ * carries a pre-sampled strip with one thumb every `thumbStep` seconds
+ * (roughly two). While a tile spans at least that much, the nearest
+ * pre-sampled thumb is a faithful picture of the tile's whole span.
+ *
+ * Zoomed past that, a tile spans a fraction of the sampling interval, and on
+ * footage that cuts between scenes the nearest thumb can show a moment up to
+ * `thumbStep / 2` away — a visibly wrong scene under the playhead. Those
+ * tiles each name a capture time (`wantT`) for a true frame at their own
+ * moment: the tile's midpoint, on a fine fixed grid (tenths of a second) so
+ * cache keys hold still through scrolls and trims. The midpoint is the one
+ * time that keeps a tile on its own side of a scene cut — a capture pulled
+ * toward the tile's edge lands on the neighboring scene whenever a cut sits
+ * inside the tile, and the strip then flips a whole tile early. `exactFrame`
+ * answers with captures already in hand; a tile without one keeps the nearest
+ * pre-sampled thumb until its capture lands.
+ *
+ * Detected scene cuts (`cuts`, measured at import) split any refining tile
+ * they pass through: each side captures its own scene, so the strip changes
+ * picture at the cut's own pixel and the boundary area under the pointer
+ * shows what actually plays there.
+ */
+
+export type FilmTile = {
+ src: string;
+ /** Pixel offset within the clip box. */
+ left: number;
+ width: number;
+ /** The tile's midpoint in source seconds. */
+ srcT: number;
+ /** Capture time for a true frame; null while the pre-sampled strip is
+ * already finer than the tile grid. */
+ wantT: number | null;
+ /** `src` is a true frame at `wantT`. */
+ exact: boolean;
+};
+
+/** Widest a strip gets before tiles widen by whole grid cells. */
+export const FILM_TILE_CAP = 120;
+
+/** Capture times land on this grid, tenths of a second: fine enough that a
+ * capture sits imperceptibly close to its tile's midpoint, coarse enough that
+ * a strip re-planned by a scroll or a trim asks for the same times again. */
+export const WANT_GRID_S = 0.1;
+
+/** Narrowest sub-tile a scene cut may split off. */
+export const MIN_SUB_TILE_PX = 5;
+
+/** A probe of the source: a moment, and whether the picture changed scene
+ * between the previous probe and this one. */
+export type ThumbProbe = { t: number; cut: boolean };
+
+/** Where the pre-sampled strip grabs each bucket's thumb.
+ *
+ * The strip is read back by position (`thumbs[floor(t / thumbStep)]`), so
+ * each bucket owns one thumb; this picks the moment inside the bucket that
+ * thumb is grabbed at. Probes carry scene-change signals measured off small
+ * decoded frames (the watch pipeline's grid signatures): each bucket grabs
+ * the middle of its longest stretch of stable content, so a bucket crossed
+ * by a scene cut shows the scene that owns most of it — a blind midpoint
+ * grab lands right at a cut often enough to paint whole buckets with their
+ * neighbor's scene. A bucket with no probes grabs its midpoint. */
+export function pickThumbTimes(
+ count: number,
+ thumbStep: number,
+ duration: number,
+ probes: ThumbProbe[]
+): number[] {
+ const clampT = (t: number) => Math.max(0, Math.min(duration - 0.05, t));
+ return Array.from({ length: count }, (_, k) => {
+ const lo = k * thumbStep;
+ const hi = (k + 1) * thumbStep;
+ const inBucket = probes.filter((p) => p.t >= lo && p.t < hi);
+ const mid = (k + 0.5) * thumbStep;
+ if (inBucket.length === 0) return clampT(mid);
+ const runs: ThumbProbe[][] = [[inBucket[0]]];
+ for (let i = 1; i < inBucket.length; i++) {
+ if (inBucket[i].cut) runs.push([]);
+ runs[runs.length - 1].push(inBucket[i]);
+ }
+ const center = (r: ThumbProbe[]) => r[Math.floor((r.length - 1) / 2)].t;
+ const best = runs.reduce((a, b) =>
+ b.length > a.length ||
+ (b.length === a.length && Math.abs(center(b) - mid) < Math.abs(center(a) - mid))
+ ? b
+ : a
+ );
+ return clampT(center(best));
+ });
+}
+
+export function planFilmstrip(p: {
+ thumbs: string[];
+ thumbStep: number;
+ /** Source length in seconds; capture times clamp inside it. */
+ duration: number;
+ /** Source width/height ratio. */
+ aspect: number;
+ /** Source time at the clip's left edge. */
+ filmIn: number;
+ /** Drawn width of the clip box, px. */
+ w: number;
+ pps: number;
+ speed: number;
+ tileH: number;
+ minTileW: number;
+ /** Scene changes in the source, seconds. A refining tile a cut passes
+ * through splits at it, so the strip changes picture at the cut's own
+ * pixel; the boundary area then shows what actually plays there. */
+ cuts?: number[];
+ /** Synchronous lookup of a captured true frame at a `wantT`. */
+ exactFrame?: (wantT: number) => string | null;
+}): FilmTile[] {
+ if (!p.thumbs.length || !p.thumbStep) return [];
+ const natural = Math.max(p.minTileW, Math.round(p.tileH * p.aspect));
+ const cells = Math.max(1, Math.ceil(p.w / natural));
+ const imgW = natural * Math.ceil(cells / FILM_TILE_CAP);
+ const stepT = (imgW / p.pps) * p.speed;
+ const filmOut = p.filmIn + (p.w / p.pps) * p.speed;
+ const first = Math.max(0, Math.floor(p.filmIn / stepT));
+ const last = Math.max(first, Math.ceil(filmOut / stepT) - 1);
+ const refine = stepT < p.thumbStep;
+ // A sub-tile narrower than a few pixels reads as a rendering artifact; a
+ // cut that close to a tile boundary already flips within that distance.
+ const minSubT = (MIN_SUB_TILE_PX / p.pps) * p.speed;
+ const tiles: FilmTile[] = [];
+ for (let k = first; k <= last; k++) {
+ const t0 = k * stepT;
+ const t1 = (k + 1) * stepT;
+ const inner = refine
+ ? (p.cuts ?? []).filter((c) => c > t0 + minSubT && c < t1 - minSubT).slice(0, 2)
+ : [];
+ const bounds = [t0, ...inner, t1];
+ for (let s = 0; s + 1 < bounds.length; s++) {
+ const a = bounds[s];
+ const b = bounds[s + 1];
+ const mid = (a + b) / 2;
+ const idx = Math.min(p.thumbs.length - 1, Math.max(0, Math.floor(mid / p.thumbStep)));
+ let src = p.thumbs[idx];
+ let wantT: number | null = null;
+ let exact = false;
+ if (refine) {
+ // Grid centers, never grid boundaries: cuts sit on frame boundaries
+ // (whole and half seconds above all), and a capture that lands
+ // exactly on a cut decodes the next scene's first frame — a tile
+ // whose midpoint sits a hair before a cut would flip to the wrong
+ // side of it. A sub-tile too narrow for a grid center inside it
+ // captures at its own midpoint.
+ const snapped = (Math.floor(mid / WANT_GRID_S) + 0.5) * WANT_GRID_S;
+ wantT = snapped > a && snapped < b ? snapped : mid;
+ wantT = Math.max(0, Math.min(p.duration - 0.05, wantT));
+ const hit = p.exactFrame?.(wantT) ?? null;
+ if (hit) {
+ src = hit;
+ exact = true;
+ }
+ }
+ tiles.push({
+ src,
+ left: ((a - p.filmIn) / p.speed) * p.pps,
+ width: ((b - a) / p.speed) * p.pps,
+ srcT: mid,
+ wantT,
+ exact,
+ });
+ }
+ }
+ return tiles;
+}
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index 6e39e407..69876ae3 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -41,6 +41,7 @@ import {
rasterCanvasToDataUrl,
type RasterSurface,
} from "./raster";
+import { pickThumbTimes, type ThumbProbe } from "./filmstrip";
import { useEditor } from "./store";
import type { AssetType, AudioClip, MediaAsset, ProjectSummary, StoredAsset, VideoClip, WatchKeepReason } from "./types";
import { contentRect, IMAGE_CLIP_SECONDS, mediaUrl } from "./types";
@@ -1471,14 +1472,18 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {
const key = stripCacheKey(useEditor.getState().projectId, asset.fileName);
const cached = await readCachedStrip(key, asset.duration);
if (cached) {
- useEditor.getState().updateAsset(asset.id, { thumbs: cached.thumbs, thumbStep: cached.thumbStep });
+ useEditor.getState().updateAsset(asset.id, {
+ thumbs: cached.thumbs,
+ thumbStep: cached.thumbStep,
+ sceneCuts: cached.cuts ?? [],
+ });
} else {
- const { thumbs, thumbStep } = await makeThumbs(src, asset.duration);
+ const { thumbs, thumbStep, cuts } = await makeThumbs(src, asset.duration);
// An empty strip is a failure, not a result: persisting it would
// leave the asset looking permanently mid-load.
if (!thumbs.length) throw new UnreadableMediaError("No frames could be read for the filmstrip.");
- useEditor.getState().updateAsset(asset.id, { thumbs, thumbStep });
- writeCachedStrip(key, { thumbs, thumbStep, duration: asset.duration, at: Date.now() });
+ useEditor.getState().updateAsset(asset.id, { thumbs, thumbStep, sceneCuts: cuts });
+ writeCachedStrip(key, { thumbs, thumbStep, cuts, duration: asset.duration, at: Date.now() });
}
enrichRetries.delete(asset.id);
} else if (asset.type === "audio" && !asset.peaks?.length) {
@@ -1528,10 +1533,17 @@ const STRIP_DB = "cut-filmstrips";
const STRIP_STORE = "strips";
const STRIP_CAP = 500; // prune oldest beyond this many cached strips
-type CachedStrip = { thumbs: string[]; thumbStep: number; duration: number; at: number };
+type CachedStrip = {
+ thumbs: string[];
+ thumbStep: number;
+ cuts?: number[];
+ duration: number;
+ at: number;
+};
function stripCacheKey(projectId: string | null, fileName: string) {
- return `${projectId ?? ""}/${fileName}@${THUMB_H}`;
+ // "s3" marks strips that carry scene cuts; strips cached before them regenerate.
+ return `${projectId ?? ""}/${fileName}@${THUMB_H}s3`;
}
function openStripDb(): Promise<IDBDatabase> {
@@ -1594,13 +1606,123 @@ function writeCachedStrip(key: string, strip: CachedStrip) {
})();
}
+/** Probes per strip bucket for scene-change detection, and the cap on the
+ * whole probe sweep so long files stay a bounded pass. */
+const THUMB_PROBE_CAP = 120;
+/** % of 16-grid cells that must change between adjacent probes to call a
+ * scene cut. Far above the watch selector's "meaningfully new" bar (8%), so
+ * motion and pans pass under it and only a real cut splits a bucket. */
+const THUMB_PROBE_CUT_PCT = 45;
+
+/** g16 grid signatures for ascending `times`, decoded small in one sweep;
+ * null where the decoder had no frame. */
+async function g16Sweep(url: string, times: number[]): Promise<(Float32Array | null)[]> {
+ const sig = createRasterCanvas(SIGNATURE_SIZE, SIGNATURE_SIZE);
+ const ctx = sig.getContext("2d", {
+ willReadFrequently: true,
+ }) as CanvasRenderingContext2D | null;
+ if (!ctx) return times.map(() => null);
+ const out: (Float32Array | null)[] = [];
+ for await (const frame of framesAt(url, times, { height: SIGNATURE_SIZE })) {
+ if (!frame) {
+ out.push(null);
+ continue;
+ }
+ ctx.drawImage(frame.canvas as CanvasImageSource, 0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE);
+ out.push(
+ frameSig({
+ width: SIGNATURE_SIZE,
+ height: SIGNATURE_SIZE,
+ channels: 4,
+ data: ctx.getImageData(0, 0, SIGNATURE_SIZE, SIGNATURE_SIZE).data,
+ }).g16
+ );
+ }
+ return out;
+}
+
+/** Scene-change signals across the clip: one small decoded frame every
+ * fraction of a bucket, compared through the watch pipeline's grid signature.
+ * A probe is flagged `cut` when the picture changed scene since the probe
+ * before it. */
+async function thumbProbes(url: string, duration: number, count: number): Promise<ThumbProbe[]> {
+ const perBucket = Math.max(3, Math.floor(THUMB_PROBE_CAP / count));
+ const n = count * perBucket;
+ const times = Array.from({ length: n }, (_, i) =>
+ Math.max(0, Math.min(duration - 0.05, ((i + 0.5) * duration) / n))
+ );
+ const sigs = await g16Sweep(url, times);
+ return times.map((t, i) => {
+ const p = i > 0 ? sigs[i - 1] : null;
+ const q = sigs[i];
+ return {
+ t,
+ cut: !!p && !!q && diffCellPct(q, p, DEDUP_TUNING.GLOBAL_TOL) > THUMB_PROBE_CUT_PCT,
+ };
+ });
+}
+
+/** How many cut brackets get the dense second sweep, samples per bracket, and
+ * the most cuts an asset stores at all. */
+const CUT_REFINE_MAX = 32;
+const CUT_REFINE_STEPS = 8;
+const SCENE_CUTS_MAX = 256;
+
+/** The clip's scene-change moments. The probe pass brackets each cut between
+ * two probes; a denser sweep inside each bracket places the cut within an
+ * eighth of the probe spacing (tens of milliseconds on a typical clip).
+ * Brackets past the refinement cap keep their bracket midpoint. */
+async function sceneCutTimes(url: string, probes: ThumbProbe[]): Promise<number[]> {
+ const brackets: [number, number][] = [];
+ for (let i = 1; i < probes.length; i++) {
+ if (probes[i].cut) brackets.push([probes[i - 1].t, probes[i].t]);
+ }
+ const refine = brackets.slice(0, CUT_REFINE_MAX);
+ const cuts = brackets.slice(CUT_REFINE_MAX).map(([a, b]) => (a + b) / 2);
+ if (refine.length) {
+ const S = CUT_REFINE_STEPS;
+ const times = refine.flatMap(([a, b]) =>
+ Array.from({ length: S + 1 }, (_, j) => a + (j * (b - a)) / S)
+ );
+ const sigs = await g16Sweep(url, times);
+ refine.forEach(([a, b], r) => {
+ let at = (a + b) / 2;
+ let strongest = -1;
+ for (let j = 1; j <= S; j++) {
+ const p = sigs[r * (S + 1) + j - 1];
+ const q = sigs[r * (S + 1) + j];
+ if (!p || !q) continue;
+ const d = diffCellPct(q, p, DEDUP_TUNING.GLOBAL_TOL);
+ if (d > strongest) {
+ strongest = d;
+ at = a + ((j - 0.5) * (b - a)) / S;
+ }
+ }
+ cuts.push(at);
+ });
+ }
+ return cuts
+ .map((c) => Math.round(c * 100) / 100)
+ .sort((x, y) => x - y)
+ .slice(0, SCENE_CUTS_MAX);
+}
+
async function makeThumbs(url: string, duration: number) {
// One frame every ~2s (min 10, max 24) so long clips don't repeat frames.
const count = Math.min(24, Math.max(10, Math.round(duration / 2)));
const thumbStep = duration / count;
- const times = Array.from({ length: count }, (_, i) =>
- Math.max(0, Math.min(duration - 0.05, (i + 0.5) * thumbStep))
- );
+ // Scene-aware grab points: probe the clip for scene changes, then grab each
+ // bucket's thumb from its dominant scene (bucket midpoints when the probe
+ // pass yields nothing).
+ let probes: ThumbProbe[] = [];
+ let cuts: number[] = [];
+ try {
+ probes = await thumbProbes(url, duration, count);
+ cuts = await sceneCutTimes(url, probes);
+ } catch {
+ // Probing is advisory; the strip still builds from midpoints.
+ }
+ const times = pickThumbTimes(count, thumbStep, duration, probes);
// Ascending times over one decode pass — the strip is a single sweep of the
// file rather than `count` seeks into it.
//
@@ -1623,7 +1745,7 @@ async function makeThumbs(url: string, duration: number) {
thumbs.push(fill);
}
}
- return { thumbs, thumbStep };
+ return { thumbs, thumbStep, cuts };
}
// Exact edge frames: a clip's first and last filmstrip tiles show the true
@@ -1658,6 +1780,9 @@ type EdgeReader = {
const edgeCache = new Map<string, string>();
const edgeQueue = new Map<string, EdgeRequest>();
+/** Filmstrip tile captures wait here, behind every queued clip-edge capture,
+ * so a trim drag's frames land first while a zoomed strip fills in behind. */
+const edgeBackQueue = new Map<string, EdgeRequest>();
const edgePool = new Map<string, Promise<EdgeReader>>();
let edgePumping = false;
@@ -1670,30 +1795,39 @@ export function peekEdgeFrame(url: string, time: number, height = THUMB_H): stri
return edgeCache.get(edgeKey(url, time, height)) ?? null;
}
-/** Capture the frame at `time`, latest-wins per `slot` (a clip edge). Resolves
- * with the frame, or null when superseded by a newer request or on a failed
- * read. */
+/** Capture the frame at `time`, latest-wins per `slot` (a clip edge or a
+ * strip tile). Resolves with the frame, or null when superseded by a newer
+ * request or on a failed read. `background` requests wait behind every queued
+ * foreground one. */
export function requestEdgeFrame(
slot: string,
url: string,
time: number,
- height = THUMB_H
+ height = THUMB_H,
+ opts?: { background?: boolean }
): Promise<string | null> {
const key = edgeKey(url, time, height);
const hit = edgeCache.get(key);
if (hit) return Promise.resolve(hit);
+ const queue = opts?.background ? edgeBackQueue : edgeQueue;
return new Promise((resolve) => {
- const prev = edgeQueue.get(slot);
+ const prev = queue.get(slot);
if (prev?.key === key) {
prev.resolvers.push(resolve);
} else {
prev?.resolvers.forEach((r) => r(null));
- edgeQueue.set(slot, { url, time, height, key, resolvers: [resolve] });
+ queue.set(slot, { url, time, height, key, resolvers: [resolve] });
}
void pumpEdgeFrames();
});
}
+/** Captures still queued or mid-read. The filmstrip eval waits on zero to
+ * know a zoomed strip has finished filling in. */
+export function edgeFramesPending(): number {
+ return edgeQueue.size + edgeBackQueue.size + (edgePumping ? 1 : 0);
+}
+
function edgeReader(url: string): Promise<EdgeReader> {
const hit = edgePool.get(url);
if (hit) {
@@ -1736,10 +1870,11 @@ async function pumpEdgeFrames() {
edgePumping = true;
try {
for (;;) {
- const next = edgeQueue.entries().next();
+ const queue = edgeQueue.size ? edgeQueue : edgeBackQueue;
+ const next = queue.entries().next();
if (next.done) break;
const [slot, req] = next.value;
- edgeQueue.delete(slot);
+ queue.delete(slot);
let src = edgeCache.get(req.key) ?? null;
if (!src) {
try {
diff --git a/site/src/cut/lib/types.ts b/site/src/cut/lib/types.ts
index ff15e109..2736c1e6 100644
--- a/site/src/cut/lib/types.ts
+++ b/site/src/cut/lib/types.ts
@@ -183,6 +183,10 @@ export interface StoredAsset {
/** BCP-47 of the audio's spoken language, when known (stamped on voiceovers
* at synthesis) — what transcription should run its recognizer in. */
language?: string;
+ /** Scene changes detected in the source at import (seconds, near frame
+ * accurate). The timeline strip splits its tiles on these so the strip
+ * changes picture where the source does. */
+ sceneCuts?: number[];
/** For origin "chat": the chat thread that made it. Deleting that thread
* deletes the assets it still owns (see chatAssets.ts). */
chatId?: string;